lcbo_sale_scraper.py

import scrapy
from twx.botapi import TelegramBot


class LCBOSpider(scrapy.Spider):
    name = 'lcbospider'
    start_urls = ['http://www.lcbo.com/lcbo/search/filter?showAll=LTO#contentBeginIndex=0&productBeginIndex=0&beginIndex=0&orderBy=&categoryPath=%2F%2F&pageView=&resultType=products&orderByContent=&searchTerm=&facet=&storeId=10151&catalogId=10001&langId=-1&fromPage=&showAll=LTO&objectId=&requesttype=ajax']

    custom_settings = {
        'CONCURRENT_REQUESTS': 2
    }

    bot = None

    def parse(self, response):
        LCBOSpider.bot = TelegramBot('<YOUR BOT TOKEN>')
        LCBOSpider.bot.update_bot_info().wait()

        count = int(response.css("span.count::text").extract()[0].split()[0])
        num_pages = int(count / 50)

        for page in range(num_pages + 1):
            start_at = page * 50
            url = "http://www.lcbo.com/webapp/wcs/stores/servlet/CategoryNavigationResultsView?pageSize=50&manufacturer=&searchType=1000&resultCatEntryType=2&catalogId=10001&categoryId=&langId=-1&storeId=10151&sType=SimpleSearch&filterFacet=&metaData=bHRvOnRydWU%3D"
            params = {'contentBeginIndex':'0',
                    'productBeginIndex':str(start_at),
                    'beginIndex':str(start_at),
                    'orderBy':'',
                    'categoryPath':'//',
                    'pageView':'',
                    'resultType':'products',
                    'orderByContent':'',
                    'searchTerm':'',
                    'facet':'',
                    'storeId':'10151',
                    'catalogId':'10001',
                    'langId':'-1',
                    'fromPage':'',
                    'showAll':'LTO',
                    'objectId':'',
                    'requesttype':'static'}
            yield scrapy.FormRequest(url, formdata=params, callback=self.parse_prices, dont_filter=True)

    def parse_prices(self, response):
        for block in response.css('div.product'):
            product_name = block.css('div.product-name a::text').extract()[0]
            price = float(block.css('div.price::text').extract()[0].replace('$', ''))
            saved = float(block.css('div.saved-price::text').extract()[0].replace('SAVE $', ''))
            percentage = saved / (price + saved)

            if percentage > 0.12:
                result = LCBOSpider.bot.send_message(int('<YOUR TELEGRAM USER ID>), '{} is on sale @ ${:0.2f} ({:2.1f}% off)'.format(product_name, price, percentage*100.0)).wait()

            yield {'product': product_name,
                   'price': price,
                   'save': saved,
                   'percentage': percentage}