dchaplinsky

# -*- mode: ruby -*-
# vi: set ft=ruby :

# All Vagrant configuration is done below. The "2" in Vagrant.configure
# configures the configuration version (we support older styles for
# backwards compatibility). Please don't change it unless you know what
# you're doing.
Vagrant.configure(2) do |config|
  #Django Frontend
  config.vm.network "forwarded_port", guest: 8000, host: 8080

dchaplinsky

import requests
import json
from time import sleep

data = []

print("Fetching page #%s" % 1)

r = requests.get("http://declarations.com.ua/search?format=opendata").json()
data += r["results"]["object_list"]

dchaplinsky

In [1]: from unicodecsv import DictReader

In [2]: fp = open("problematic.csv", "r")

In [3]: r = DictReader(fp)

In [4]: r.next()  # First row is not interesting
Out[4]: 
{u'\u0406\u041f\u041d': u'',
 u'\u0414\u0430\u0442\u0430 \u0432\u0456\u0434\u043f\u043e\u0432\u0456\u0434\u0456': u'',

dchaplinsky

dchaplinsky

import bz2

with bz2.BZ2File("test.bz2", "r") as fp:
    i = 0
    for l in fp:
        print(l)

dchaplinsky

import re
import os.path
import requests
from random import sample, random
from collections import Counter
from pymongo import MongoClient
from glob2 import glob

client = MongoClient()
db = client.decl

dchaplinsky

INFO   2014-12-22 12:55:53  simplifying tags: looking for tag spellings
INFO   2014-12-22 12:56:06  simplifying tags: looking for spelling duplicates (skip_space_ambiguity: True)
DEBUG  2014-12-22 12:56:06  290 duplicate tags will be removed
INFO   2014-12-22 12:56:06  simplifying tags: fixing
INFO   2014-12-22 12:56:11  inlining lexeme derivational rules...
INFO   2014-12-22 12:56:12  building paradigms...
DEBUG  2014-12-22 12:56:12                  word    len(gramtab)      len(words)  len(paradigms)
DEBUG  2014-12-22 12:56:12          пообклеювати              15              15               1
DEBUG  2014-12-22 12:56:15              кричавши            1042          133287             667
DEBUG  2014-12-22 12:56:16              димувати            1249          269033             985

dchaplinsky

INFO   2014-12-21 17:20:27  simplifying tags: looking for tag spellings
INFO   2014-12-21 17:20:41  simplifying tags: looking for spelling duplicates (skip_space_ambiguity: True)
DEBUG  2014-12-21 17:20:41  313 duplicate tags will be removed
INFO   2014-12-21 17:20:41  simplifying tags: fixing
INFO   2014-12-21 17:20:46  inlining lexeme derivational rules...
INFO   2014-12-21 17:20:47  building paradigms...
DEBUG  2014-12-21 17:20:47                  word    len(gramtab)      len(words)  len(paradigms)
DEBUG  2014-12-21 17:20:47          пообклеювати              15              15               1
DEBUG  2014-12-21 17:20:50              кричавши            1186          133287             667
DEBUG  2014-12-21 17:20:51              димувати            1417          269033             985

dchaplinsky

# -*- coding: utf-8 -*-
from string import capwords


def title(s):
    chunks = s.split()
    chunks = map(lambda x: capwords(x, u"-"), chunks)
    return u" ".join(chunks)

if __name__ == '__main__':

dchaplinsky

>> db.tagging_speed.aggregate(
{
	$match: 
	{
		msec: {$gte: 2 * 1000, $lte: 180 * 1000}
	}
},
{
	$group: 
	{