Slick Deals Crawler

slick_deals_spider.py

import scrapy
import csv
from datetime import datetime
from scrapy.selector import Selector


class SlickDealsSpider(scrapy.Spider):
    name = "slickdeals"

    def start_requests(self):
        url = "https://slickdeals.net/"
        yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        deal_items = Selector(response).xpath('//div[contains(@class, "fpItem")]')
        deals = []
        for index, item in enumerate(deal_items):
            store = item.xpath('.//a[contains(@class,"itemStore")]/text()').extract()
            title = item.xpath('.//a[contains(@class,"itemTitle")]/text()').extract()
            title_link = item.xpath('.//a[contains(@class,"itemTitle")]/@href').extract()
            price = item.xpath('.//div[contains(@class,"itemPrice")]/text()').extract()
            original_price = item.xpath('.//span[contains(@class, "oldListPrice")]/text()').extract()

            cnts = item.xpath('.//span[contains(@class,"count")]/text()').extract()
            if len(store)>0 and len(title)>0 and len(title_link)>0 and len(price)>0 and len(original_price)>0 and len(cnts)>0:

                likes = cnts[0]
                comments = cnts[1]
                deal = (datetime.now(),store[0],title[0],title_link[0],price[0].replace(' ', '').replace('\n',''), original_price[0],likes, comments)
                deals.append(deal)

        with open(f"/home/pi/projects/slickdealscrapy/data/data_file.csv", "a", newline="") as f:
            writer = csv.writer(f)
            writer.writerows(deals)

## schedule it with crontab
## SHELL=/bin/bash
## */15 * * * * cd /home/pi/projects/slickdealscrapy && /home/pi/.pyenv/shims/scrapy crawl slickdeals >> /tmp/mycommand.log 2>&1
## 2020-04-18 21:00:10.552354,Sam Ash ,Taylor Guitars 317e Grand Pacific Acoustic-Electric Guitar w/ Hardshell Case,/f/13990862-taylor-317e-grand-pacific-acoustic-electric-guitar-1000-off-ymmv-999?src=frontpage,$999,$1999,34,52