Clément Denoix clemfromspace

🎯

Focusing

clemfromspace / puppeteer-middleware.py

Last active June 26, 2021 08:02

Scrapy with Puppeteer

	"""This module contains the ``SeleniumMiddleware`` scrapy middleware"""

	import asyncio

	from pyppeteer import launch
	from scrapy import signals
	from scrapy.http import HtmlResponse
	from twisted.internet import defer

	from .http import PuppeteerRequest

clemfromspace / tasks.py

Created November 24, 2017 04:34

Running a scrapy spider from a celery task

	from billiard.context import Process
	from scrapy.crawler import Crawler
	from scrapy import signals
	from scrapy.utils.project import get_project_settings
	from twisted.internet import reactor

	from celery_app import app


	class CrawlerProcess(Process):

clemfromspace / tripadvisor_spider_v2.py

Created May 19, 2016 20:26

	# -- coding: utf-8 --

	import re
	import dateparser

	from scrapy import Request
	from scrapy.spider import CrawlSpider

	from urlparse import urljoin

clemfromspace / tripadvisor_spider.py

Created May 18, 2016 22:16

	import re

	from scrapy import Request
	from scrapy.spider import CrawlSpider

	from urlparse import urljoin


	class TripAdvisorSpider(CrawlSpider):

clemfromspace / items.py

Last active March 10, 2018 17:02

Scrapy wikipédia url