kmike

# -*- coding: utf-8 -*-
from __future__ import absolute_import
import logging
import urlparse

from reppy.parser import Rules

from scrapy import log
from scrapy.exceptions import NotConfigured
from scrapy.http import Request

kmike

kmike

import scrapy
from scrapy.http import safeurl


class Spider(scrapy.Spider):

    name = 'loremipsum'
    start_urls = ('https://www.lipsum.com',)

    def parse(self, response):

kmike

kmike

kmike

kmike

kmike

Folder structure should be the following:

vectorizers/
    vec/
    stored/
    string_dict/
        string_dict.pyx
        setup.py
    marisa_vectorizers.py
    memusage_fit.py

kmike

import numpy as np
import scipy.sparse as sp
import hat_trie
from sklearn.feature_extraction.text import CountVectorizer, _make_int_array


class HatTrieCountVectorizer(CountVectorizer):

    def _count_vocab(self, raw_documents, fixed_vocab):
        """Create sparse feature matrix, and vocabulary where fixed_vocab=False

kmike

import numpy as np
import marisa_trie
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.externals import six

class MarisaCountVectorizer(CountVectorizer):

    # ``CountVectorizer.fit`` method calls ``fit_transform`` so
    # ``fit`` is not provided
    def fit_transform(self, raw_documents, y=None):