gistfile1.py

# -*- encoding: utf-8 -*-
from datetime import datetime
from scrapy.item import Item, Field
from scrapy.contrib.loader.processor import TakeFirst
from scrapy.contrib.loader import ItemLoader as ScrapyItemLoader
from sqlalchemy import DateTime
from sqlalchemy.orm import class_mapper
from sqlalchemy.orm.properties import ColumnProperty
from kickscraper.db.models import Project, User, Location, Category


def to_date(timestamp):
    return datetime.fromtimestamp(timestamp[0])


class ItemLoader(ScrapyItemLoader):
    default_output_processor = TakeFirst()


def item_factory(name, Model):
    """Create scrapy item from sqlalquemy model"""
    d = {}
    for prop in class_mapper(Model).iterate_properties:
        if isinstance(prop, ColumnProperty):
            column = prop.columns[0]
            kwargs = {}
            if isinstance(column.type, DateTime):
                kwargs['output_processor'] = to_date
            d[column.name] = Field(**kwargs)
    return type(name, (Item,), d)


ProjectItem = item_factory('ProjectItem', Project)
UserItem = item_factory('UserItem', User)
LocationItem = item_factory('LocationItem', Location)
CategoryItem = item_factory('CategoryItem', Category)