Capture scrapy crawler results to a Python list

scrape_list.py

import scrapy
from scrapy import signals
from scrapy.signalmanager import dispatcher

class MySpider(scrapy.Spider):
  ...

# gather the results, see: https://stackoverflow.com/a/40240712
nurseries = []

def crawler_results(signal, sender, item, response, spider):
    nurseries.append(item)


dispatcher.connect(crawler_results, signal=signals.item_passed)

# Run the Spider
process = CrawlerProcess()
process.crawl(MySpider)
process.start()