booklist.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import json
import collections
from pprint import *
from peewee import *

# DB init
mysql_db = MySQLDatabase('hitode', user='root')


class BaseModel(Model):
    class Meta:
        database = mysql_db


class Booklist(BaseModel):
    uid = IntegerField(primary_key=True)
    isbn = CharField(null=False)


def convert(data):
    if isinstance(data, basestring):
        return str(data.encode('utf-8'))
    elif isinstance(data, collections.Mapping):
        return dict(map(convert, data.iteritems()))
    elif isinstance(data, collections.Iterable):
        return type(data)(map(convert, data))
    else:
        return data


if __name__ == '__main__':

    # list
    data_source = []

    # unique isbn list
    #duplication = []
    #dupf = open('uisbn.txt', "r")
    #for line in dupf:
        #duplication.append(line.split('\n')[0])
    #dupf.close()
    #pprint(duplication)

    #f = open('data.json', "r")
    f = open('data.json', "r")
    books = json.load(f)

    for k in books:
        item = {}
        item['uid'] = str(int(k) + 1)
        for inner_k in books[k]:
            book = books[k][inner_k]

            if '13' not in book['isbn']:
                if '10' not in book['isbn']:
                    continue
                else:
                    item['isbn'] = book['isbn']['10']
            else:
                item['isbn'] = book['isbn']['13']

            data_source.append(item)

        pprint(data_source)

    #pprint(convert(books['4174']))
    #pprint(books[str(4174)])
    f.close()

    #dupf = open('uisbn.txt', "w+")
    #dupf.write('\n'.join(duplication) + '\n')
    dupf.close()

    pprint(data_source)

    #f = open('output.txt', "w+")
    #f.write((repr(books)).encode('utf-8'))
    #f.write(repr(convert(books)).decode('utf-8'))
    #f.close()

    # DataBase
    mysql_db.connect()
    Booklist.create_table()

    with mysql_db.transaction():
        for idx in range(0, len(data_source), 1000):
            Booklist.insert_many(data_source[idx:idx+1000]).execute()