A program to run the pypy version of scrapy in the same process.

run.py

from scrapy.crawler import CrawlerProcess
from scrapy.crawler import CrawlerRunner
from twisted.internet import reactor
from scrapy.utils.project import get_project_settings
from scrapy.utils.log import configure_logging
process = CrawlerProcess(get_project_settings())

# 'followall' is the name of one of the spiders of the project.
#process.crawl('followall')

configure_logging()
runner = CrawlerRunner()
runner.crawl('followall')
runner.crawl('followall')
d = runner.join()
d.addBoth(lambda _: reactor.stop())

reactor.run()
#process.start() # the script will block here until the crawling is finished