kmike

kmike

kmike

count
fit time: 3.4s
fit memory usage: 150.2MB
dump time: 5.3s
dump memory usage: 71.5MB
transform features: 130107
load time: 2.1s
load memory usage: 94.5MB
transform time: 1.8s
transform memory leak: 35.4MB

kmike

#!/usr/bin/env python
from __future__ import division, print_function
import os
import sys
import time
import psutil
from sklearn.externals import joblib
from sklearn import datasets

# the following imports are not needed, but if we won't import them

kmike

import marisa_trie
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer


# hack to store vocabulary in MARISA Trie
class _MarisaVocabularyMixin(object):

    def fit_transform(self, raw_documents, y=None):
        super(_MarisaVocabularyMixin, self).fit_transform(raw_documents)
        self._freeze_vocabulary()

kmike

#!/usr/bin/env python
from __future__ import division, print_function
import os
import sys
import time
import resource
import psutil
from sklearn import datasets
from sklearn.externals import joblib
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer

kmike

# -*- coding: utf-8 -*-
"""
Utilities for measuring parse quality.
"""
import functools
from russian_tagsets.ruscorpora import from_opencorpora_int


# memoize results for 4x speedup
op2rnc = functools.lru_cache(maxsize=10000)(from_opencorpora_int)

kmike

import numpy as np
from scipy.misc import logsumexp

def _forward(n_observations, n_components, log_startprob, log_transmat,
             framelogprob):

    fwdlattice = np.empty((n_observations, n_components))
    fwdlattice[0] = log_startprob + framelogprob[0]

    for t in range(1, n_observations):

kmike

# -*- coding: utf-8 -*-
from __future__ import absolute_import
from libc.math cimport log2  # FIXME: can be unavailable in Windows
cimport cython

ctypedef double dtype_t

DEF _NINF = float('-inf')

def log_add(*values):

kmike

def use_speedups_if_available(original, module_name, func_name):
    try:
        mod = __import__(module_name, fromlist=[func_name])  
        return getattr(mod, func_name), original
    except ImportError:
        return original, original


# ------------- Example usage ------------
def foo():