isla.py

# -*- coding: utf-8 -*-
import scrapy
import sys
import re
reload(sys)
sys.setdefaultencoding('utf-8')
from w3lib.html import remove_tags
from isla.items import IslaItem


class IslaspiderSpider(scrapy.Spider):
    name = 'islaspider'
    allowed_domains = ['isla.com.br']
    start_urls = ['https://isla.com.br/']

    def parse(self, response):
        links = [
            'https://isla.com.br/produtos/categorias/Hortali%C3%A7as/1',
            'https://isla.com.br/produtos/categorias/Flores%20e%20Ornamentais/2',
            'https://isla.com.br/produtos/categorias/Ervas%20e%20Temperos/4',
            'https://isla.com.br/produtos/categorias/Frutas/8',
            'https://isla.com.br/produtos/categorias/%C3%81rvores/17'
        ]
        for url in links:
            yield response.follow(url, self.parse_category)

    def parse_category(self, response):
        for prod in response.css(".contProducts a"):
            unavailable = prod.css("button.comprar::attr(class)").get().split(" ")
            if len(unavailable) == 1:
                url = prod.css("::attr(href)").get()
                yield response.follow(url, self.parse_product)

    def parse_product(self, response):
        name = response.css(".tituloproduto::text").get()
        url = response.url
        image = 'https://' + IslaspiderSpider.allowed_domains[0] + response.css('.imagemProd img::attr(src)').get()
        body = {}
        body['sku'] = response.css(".tituloproduto::text").get().split('-')[0].strip()
        features = {}
        for line in response.css(".caracDesc table tr"):
            key = line.css("td:first-child::text").get().strip()
            val = line.css("td:last-child::text").get().strip()
            features[key] = val

        body['caracteristicas'] = features
        features = {}
        for line in response.css(".sementesPlantio + .desc .necessidade table tr"):
            key = line.css("td:first-child::text").get().strip()
            val = line.css("td:last-child::text").get().strip()
            features[key] = val

        body['necessidades_plantio'] = features
        features = {}
        for line in response.css(".modoPlantio + .desc ul img"):
            src = line.css("::attr(src)").get()
            key = re.sub(r"_|.jpg", ' ', src.split('/').pop()).strip()
            val =  'https://' + IslaspiderSpider.allowed_domains[0] + src
            features[key] = val

        body['modo_plantio'] = features
        yield {"name": name, "image": image, "url": url, "body": body}