requirements.txt

BeautifulSoup==3.2.1
requests==2.11.1
requests-cache==0.4.12
vcrpy==1.10.2

result-%22K14___13%22.csv

result-%22K_14___13%22.csv

result-%22OTK_ZU_nr_4___A___2015%22_%22dokumentu_wewn%C4%99trznego%22.csv

result-%22P25___12%22.csv

result-%22P_25___12%22.csv

scrap.py

# -*- coding: utf-8 -*-
import hashlib
import time
from collections import namedtuple
from pprint import pprint
from urllib import quote
from urlparse import urljoin

import dataset
import requests
import vcr
from BeautifulSoup import BeautifulSoup


def print_url(r, *args, **kwargs):
    print(r.url)


class NullContext:
    """Helper for nullcontext."""

    def __enter__(self):
        pass

    def __exit__(self, *exc_info):
        pass


_nullcontext = NullContext()


def nullcontext():
    """
    No-op context manager, executes block without doing any additional processing.

    Used as a standin if a particular block of code is only sometimes
    used with a normal context manager:

      with optional_cm or nullcontext():
          # Perform operation, using the specified CM if one is given
    """
    return _nullcontext


Response = namedtuple('Response', ['link', 'soup'])

SLEEP_TIME = 3


class CaptchaException(Exception):
    pass


class Client(object):
    def __init__(self, session=None):
        self.session = session or requests.Session()
        self.last_query = {}
        self.p = 1

    def _get_query_headers(self):
        text = str(self.last_query)
        digest = hashlib.md5(text).hexdigest()
        return {'query': digest}

    def _query(self, method, url, *args, **kwargs):
        retry = 3
        while retry > 0:
            retry -= 1
            global SLEEP_TIME
            print "Sleep %d seconds before requests" % (SLEEP_TIME, )
            time.sleep(SLEEP_TIME)
            try:
                resp = self.session.request(*args,
                                            method=method, url=url,
                                            headers=self._get_query_headers(),
                                            **kwargs)
            except requests.exceptions.ConnectionError:
                print "Connection error"
                continue

            if 'Aby kontynuowa' not in resp.text:
                return Response(link=resp.url,
                                soup=BeautifulSoup(resp.text))
            SLEEP_TIME = SLEEP_TIME * 2
            print "Captcha time increased to %d" % (SLEEP_TIME, )
            raw_input("Captcha detected. Press [ENTER] to continue.")

    def search(self, **query):
        params = {'wszystkieSlowa': 'wykop',
                  'wystepowanie': 'gdziekolwiek',
                  'odmiana': 'on',
                  'sygnatura': '',
                  'sad': 'dowolny',
                  'rodzaj': 'dowolny',
                  'symbole': '',
                  'odDaty': '',
                  'doDaty': '',
                  'sedziowie': '',
                  'funkcja': 'dowolna',
                  'rodzaj_organu': '',
                  'hasla': '',
                  'akty': '',
                  'przepisy': '',
                  'publikacje': '',
                  'glosy': '',
                  'submit': 'Szukaj', }
        params.update(query)
        self.last_query = params
        print params
        return self._query(method='post',
                           url='http://orzeczenia.nsa.gov.pl/cbo/search',
                           data=params)

    def next_page(self):
        self.p += 1
        return self._query('get',
                           url='http://orzeczenia.nsa.gov.pl/cbo/find',
                           params={'p': self.p})


class SERParser(object):
    def __init__(self, response):
        self.response = response
        self.soup = response.soup

    def _get_table(self):
        return self.soup.findAll(attrs={'class': 'info-list pb-none'})

    def get_count(self):
        return len(self._get_table())

    def _get_link(self, table):
        return (table.find(attrs={'class': "info-list-value "}) or
                table.find(attrs={'class': "info-list-value nk"}))

    def _get_row_from_table(self, table):
        link = self._get_link(table).find('a')

        description = table.find(attrs={'style': 'font-size: 10pt; border: none;'})

        dsc = table.find(attrs={'style': 'font-size: 10pt; border: none;'})
        description = u" ".join(map(lambda x: unicode(x).strip(), dsc.contents))

        # <a href="/cbo/wsp/C995D7166E" title="Wszystkie orzeczenia w sprawie">
        wszystkie = table.find(attrs={'title': u"Wszystkie orzeczenia w sprawie"})
        # <a href="/cbo/pow/2C50E69C2E" title="Wszystkie orzeczenia powiązane">
        powiazane = table.find(attrs={'title': u"Wszystkie orzeczenia powiązane"})

        return {'link': urljoin(self.response.link, link.attrMap['href']),
                'name': link.text,
                'description': description,
                'wszystkie': wszystkie.attrMap['href'] if wszystkie else None,
                'powiazane': powiazane.attrMap['href'] if powiazane else None,
                'wszystkie_count': wszystkie.text.strip('()') if wszystkie else 0,
                'powiazane_count': powiazane.text.strip('()') if powiazane else 0}

    def get_rows(self):
        rows = []
        for table in self._get_table():
            row = self._get_row_from_table(table)
            rows.append(row)
        return rows

    def has_prev(self):
        return any('poprz' in btn.text
                   for btn in self.soup.findAll(attrs={'class': 'nextpage'}))

    def has_next(self):
        return any('nast' in btn.text
                   for btn in self.soup.findAll(attrs={'class': 'nextpage'}))
        return bool(self.soup.find(attrs={'class': 'nextpage'}).find('a'))

    def next_page_link(self):
        if not self.has_next():
            return None
        return self.soup.find(attrs={'class': 'nextpage'}).find('a').attrMap['href']


def merge_dict(dict_a, dict_b):
    result = dict_a.copy()
    result.update(dict_b)
    return result


session = requests.Session()
session.hooks['response'] = print_url


manager = nullcontext()
# manager = vcr.use_cassette('test.yml', decode_compressed_response=True, record_mode='new_episodes')

queries = [u'"OTK ZU nr 4/A/2015" "dokumentu wewnętrznego"'.encode('utf-8'),
           u'"K14/13"',
           u'"K 14/13"',
           u'"P25/12"',
           u'"P 25/12"']

for query in queries:
    table = dataset.connect('sqlite://')['sometable']
    with manager:
        # query = u'"K14/13"'
        client = Client(session)
        resp = client.search(wszystkieSlowa=query)
        parser = SERParser(resp)
        while True:
            for row in parser.get_rows():
                print row['name']
                table.insert(merge_dict({'query': query.decode('utf-8')}, row))
            if parser.has_next():
                resp = client.next_page()
                parser = SERParser(resp)
            else:
                break
    filename = 'result-' + quote(query.replace(' ', '_')).replace('/', '___') + '.csv'
    print "Save data to ", filename
    dataset.freeze(table.all(), format='csv', filename=filename)

stats.txt

========== Repo stats ==========
https://github.com/jayfk/statuspage - 3297
https://github.com/amperser/proselint - 2059
https://github.com/pydanny/cookiecutter-django - 1936
https://github.com/quokkaproject/quokka - 1385
https://github.com/audreyr/cookiecutter-pypackage - 736
https://github.com/kavdev/dj-stripe - 321
https://github.com/torque59/Nosql-Exploitation-Framework - 262
https://github.com/pydanny/djangopackages - 202
https://github.com/pydanny/cached-property - 177
https://github.com/pydanny/cookiecutter-djangopackage - 165
https://github.com/agconti/cookiecutter-django-rest - 135
https://github.com/SunDwarf/OWAPI - 89
https://github.com/marshmallow-code/marshmallow-sqlalchemy - 87
https://github.com/phalt/beckett - 71
https://github.com/chrisdev/wagtail-cookiecutter-foundation - 63
https://github.com/jayfk/fundingoss.com - 58
https://github.com/pyupio/pyup - 52
https://github.com/bow/volt - 46
https://github.com/bonclay7/aws-amicleaner - 38
https://github.com/ericmjl/hiveplot - 32
https://github.com/trungdong/prov - 29
https://github.com/cuducos/cunhajacaiu - 26
https://github.com/SpisTresci/SpisTresci - 26
https://github.com/pydanny/contributors - 22
https://github.com/rodrigobraga/informer - 14
https://github.com/ellisonleao/vl - 13
https://github.com/bow/crimson - 12
https://github.com/scieloorg/scielo-manager - 11
https://github.com/rdcli/rdc.etl - 11
https://github.com/bow/pytest-pipeline - 9
https://github.com/valerymelou/cookiecutter-django-gulp - 9
https://github.com/InternetSemLimites/PublicAPI - 8
https://github.com/ericmjl/Circos - 7
https://github.com/jayfk/cookiecutter-saas - 6
https://github.com/goodplay/goodplay - 6
https://github.com/keitheis/alog - 5
https://github.com/ericmjl/nxviz - 5
https://github.com/picleslivre/schemaprobe - 4
https://github.com/twindb/twindb_table_compare - 4
https://github.com/datascopeanalytics/traces - 4
https://github.com/chrisdev/django-foundation-formtags - 4
https://github.com/OpenDataScienceLab/vizpydata - 3
https://github.com/Abraxas-Biosystems/eve-neo4j - 3
https://github.com/bigchaindb/pycoalaip - 3
https://github.com/hartym/git-semver - 3
https://github.com/ericmjl/pyflatten - 3
https://github.com/python/the-knights-who-say-ni - 3
https://github.com/rdcli/cmdq - 3
https://github.com/rdcli/rdc.dic - 2
https://github.com/SunDwarf/Chiru - 2
========== Infulencers stats ==========
https://api.github.com/users/pydanny - 1806
https://api.github.com/users/audreyr - 774
https://api.github.com/users/phalt - 425
https://api.github.com/users/ellisonleao - 292
https://api.github.com/users/mozillazg - 210
https://api.github.com/users/agconti - 201
https://api.github.com/users/cuducos - 115
https://api.github.com/users/luzfcb - 100
https://api.github.com/users/glogiotatidis - 99
https://api.github.com/users/RonnyPfannschmidt - 85
https://api.github.com/users/hartym - 61
https://api.github.com/users/bow - 55
https://api.github.com/users/keitheis - 55
https://api.github.com/users/torque59 - 49
https://api.github.com/users/ericmjl - 47
https://api.github.com/users/jayfk - 41
https://api.github.com/users/SunDwarf - 39
https://api.github.com/users/urkonn - 33
https://api.github.com/users/chrisdev - 29
https://api.github.com/users/jtraulle - 20
https://api.github.com/users/mildronize - 19
https://api.github.com/users/pghalliday - 19
https://api.github.com/users/jkahn - 16
https://api.github.com/users/rodrigobraga - 16
https://api.github.com/users/yupbank - 16
https://api.github.com/users/bretth - 12
https://api.github.com/users/tartavull - 12
https://api.github.com/users/illagrenan - 10
https://api.github.com/users/paulollivier - 10
https://api.github.com/users/bradrf - 7
https://api.github.com/users/simobasso - 7
https://api.github.com/users/sebikul - 6
https://api.github.com/users/kavdev - 6
https://api.github.com/users/jimmyho - 6
https://api.github.com/users/ikarth - 6
https://api.github.com/users/mimischi - 5
https://api.github.com/users/bonclay7 - 5
https://api.github.com/users/macklenc - 3
https://api.github.com/users/mistfox - 3
https://api.github.com/users/trungdong - 3
https://api.github.com/users/poncki - 3
https://api.github.com/users/DavidHickman - 3
https://api.github.com/users/ameistad - 3
https://api.github.com/users/dm-wyncode - 2
https://api.github.com/users/gusseppe - 2
https://api.github.com/users/StuartJSquires - 2
https://api.github.com/users/amir-khakshour - 2
https://api.github.com/users/marcelluseasley - 2
https://api.github.com/users/maartenq - 2
https://api.github.com/users/rscholer - 2