spreadsheet.py

#!/usr/bin/env python
# coding: utf-8
from __future__ import unicode_literals
import csv
import sys
from collections import OrderedDict

import couchdbkit

BASE_URL = 'http://books.scielo.org/id/%s'
server = couchdbkit.Server(uri='')
db = server['scielobooks_1a']

_ERROR = '###ERROR###'


def bisac(data):
    try:
        return dict(data[0])['code']
    except:
        return _ERROR

FIELDS = (
    # ('csv column', 'fieldname', 'callable', 'is_mandatory')
    ('eBook ISBN', 'eisbn', lambda x: x.replace('-', ''), True),
    ('Related ISBN', 'isbn', lambda x: x.replace('-', ''), False),
    ('Title', 'title', lambda x: x, True),
    ('Subtitle', '', None, False),
    ('Edition', 'edition', lambda x: x, False),
    ('Series', '', None, False),
    ('# in series', '', None, False),
    ('Description', 'synopsis', lambda x: x.strip(), True),
    ('Publisher', 'publisher', lambda x: x.strip(), True),
    ('Imprint', '', None, False),
    ('Language', 'language', lambda x: x.strip(), True),
    ('Contributor 1', '', None, True),
    ('Contributor Type 1', '', None, True),
    ('Contributor 2', '', None, False),
    ('Contributor Type 2', '', None, False),
    ('Contributor 3', '', None, False),
    ('Contributor Type 3', '', None, False),
    ('Sales Rights', '', None, True),
    ('Publication Date', '', None, False),
    ('OnSale Date', '', None, False),
    ('Price 1', '', None, True),
    ('Currency 1', '', None, True),
    ('Price 2', '', None, False),
    ('Currency 2', '', None, False),
    ('Price 3', '', None, False),
    ('Currency 3', '', None, False),
    ('Price 4', '', None, False),
    ('Currency 4', '', None, False),
    ('Price 5', '', None, False),
    ('Currency 5', '', None, False),
    ('Categorization Code 1', 'bisac_code', bisac, True),
    ('Categorization Type 1', '', None, True),
    ('Categorization Code 2', '', None, False),
    ('Categorization Type 2', '', None, False),
    ('Categorization Code 3', '', None, False),
    ('Categorization Type 3', '', None, False),
    ('Tags for Search', '', None, False),
    ('Preview %', '', None, False),
    ('Social Sharing', '', None, False),
    ('Adult Material', '', None, False),
)

CONTRIB_TYPES = {
    'individual_author': 'A01',
    'corporate_author': 'A01',
    'editor': 'B01',
    'organizer': 'A01',
    'translator': 'B06',
    'collaborator': 'A01',
    'coordinator': 'A01',
    'other': 'A01',
}


def _creators_by_roles(creators):
        creators_by_role = OrderedDict()

        try:
            for creator in creators:
                creator_d = dict(creator)

                creators_by_role.setdefault(creator_d['role'], []).append(
                    (creator_d['full_name'], creator_d['link_resume']))
        except AttributeError:
            return {}

        return creators_by_role


def _fix_author_name(author):
    """
    Transforma 'Pretto, Nelson De Luca' em 'Nelson De Luca Pretto'
    """
    return ' '.join(author.split(',')[::-1]).strip()


class Contributor(object):
    precedence = (
        'individual_author',
        'corporate_author',
        'editor',
        'organizer',
        'translator',
        'collaborator',
        'coordinator',
        'other',
    )

    def __init__(self, creators):
        self.creators = creators
        self._ignore = set()

    def __iter__(self):
        creators = _creators_by_roles(self.creators)

        while True:
            authors = role = ''
            local_precedence = [elem for elem in self.precedence if elem not in self._ignore]

            for contrib_role in local_precedence:

                if contrib_role in creators:
                    role = contrib_role
                    break

            if role:
                authors = ', '.join([_fix_author_name(au[0]) for au in creators[role]])
                self._ignore.add(role)
            else:
                raise StopIteration()

            yield [authors, CONTRIB_TYPES.get(role, '')]


if __name__ == '__main__':

    FETCH_FIELDS = [[key, [field, func, is_mandatory]] for key, field, func, is_mandatory in FIELDS if (field or is_mandatory)]
    FIELDNAMES = [item[0] for item in FIELDS]

    out = csv.DictWriter(sys.stdout, FIELDNAMES)

    # imprime o cabeçalho
    out.writerow(dict([name, name] for name in FIELDNAMES))

    for book in db.view('books/published_books', include_docs=True):
        doc_url = BASE_URL % book['id']
        row = {}

        for label, value in FETCH_FIELDS:
            fieldname, postfunc, is_mandatory = value

            try:
                fieldval = book['value'].get(fieldname,
                    _ERROR if is_mandatory else '')

                if postfunc:
                    fieldval = postfunc(fieldval)

            except Exception as exc:
                sys.stderr.write('field: %s - %s\n' % (fieldname, exc.message))
                continue

            row.update({label: fieldval.encode('utf-8')})

        # contributors
        contributors = Contributor(book['value']['creators'])
        for i, contrib in enumerate(contributors):
            # somente 3 contribuidores
            if i > 2:
                break

            names, typ = contrib
            row.update(
                {
                'Contributor %s' % (i + 1): names.encode('utf-8'),
                'Contributor Type %s' % (i + 1): typ.encode('utf-8')
                }
            )

        # categorization code
        if 'Categorization Code 1' in row and row['Categorization Code 1'] != _ERROR:
            cat_code = 'BISAC'
        else:
            cat_code = _ERROR

        row.update({'Categorization Type 1':  cat_code})

        out.writerow(row)