cb.py

"""
Bitcoin Daemon RPC wrapper for simple calculations, top address list,
maybe block explorer verification, etc.

"""

import time
import operator
import pickle
import json
import math

from bitcoinrpc.authproxy import AuthServiceProxy
from pprint import pprint


SATOSHI = 100000000
MAX_BLOCKS = 300        # Limit the number of blocks parsed.
PERSIST_EVERY = 100      # Persist the book every n blocks. (And last block)
persist_file = "cinnibook.pkl"


svc = AuthServiceProxy("http://12345:[email protected]:2222")

# ./CinniCoind --datadir=/opt/svc1/cinni0 -conf=cinni0.conf  


# Simple Persistance Handlers
# ===========================
def persist(data, meta={}, datakey="data"):
    """Simple persist of `data` and `meta`data.
    """
    pickle.dump({'meta': meta, datakey: data}, open(persist_file, 'w'))


def load():
    """Load persisted data.
    """
    return pickle.load(open(persist_file, 'r'))


def increment(balance, increment):
    """Just increment `balance` and floor
    """
    balance += increment
    return round(balance, 9)


# Base parsers
# ============
def parse_tx_out(tx):
    """Parse transaction outputs.
    Yields (output_n, output_address, output_amount)
    """
    for out in tx['vout']:
        if 'addresses' in out['scriptPubKey']:
            addresses = out['scriptPubKey']['addresses']
            if len(addresses) > 1:
                # This never happens?? We don't know what to do here.
                raise NotImplementedError("More than one vout address.")
            yield out['n'], addresses[0], float(out['value'])
        else:
            # No addresses. Do nothing for now.
            pass


def parse_tx_in(tx, svc=svc):
    """Parse input data. Get input address from previous transaction.
    None if mined. (Not sure about PoS yet.)
    """
    for in_ in tx['vin']:       # Iterate through inputs.
        if 'scriptSig' in in_:  # Only concerned with signed inputs.
            input_tx = svc.gettransaction(in_['txid'])  # Get previous
                                                        #   transaction
            yield get_tx_out(input_tx, in_['vout'])     # Get sending address.
        elif 'coinbase' in  in_:
            # This should be handled better.
            yield 0, '0000000000000000000000000000000000', 0.0
        else:
            raise Exception("Bad transaction sent to parse_tx_in. %s" % tx)


# Filters
# =======
def get_tx_out(tx, get_n):
    """Return nth output in a tx.
    Used in discovering input specifics.
    return (output_n, output_address, output_amount)
    """

    for n, address, amount in parse_tx_out(tx):
        if get_n == n:
            return n, address, amount


def find_tx_out_address(tx, target):
    """Searches for address `target` in the outputs of a transaction.
    Yields (output_n, output_address, output_amount)
    """

    for n, address, amount in parse_tx_out(tx):
        if address == target:
            yield n, address, amount


def find_tx_in_address(tx, target):
    """Searches for address `target` in the inputs of a transaction.
    Yields (input_n, input_address, input_amount)
    """

    for n, address, amount in parse_tx_in(tx):
        if address == target:
            yield n, address, amount


# Iterators
# =========
def iter_tx(blk_data):
    """Iterate through block transactions.
    Yields (transaction_hash, transaction)
    """
    for tx in blk_data['tx']:
        yield tx['txid'], tx


def iter_blks(max, min=0, svc=svc):
    """Iterate through blocks yielding the block data and number.
    Yields (block_number, block_hash, block)
    """
    for blk_n in range(min, max):
        blk_hash = svc.getblockhash(blk_n)          # Get the block hash of
                                                    #   block n.
        yield blk_n, blk_hash, svc.getblock(blk_hash, True)


# Data Generators
# ===============
def find_address_transactions(address, max=None, min=0):
    """Iterate through blocks searching for transactions in from and out to
    `address`.
    Yields (in or out, parsed_transaction)
    """
    bal = 0.0
    max = max or svc.getblockcount()
    for blk_n, _, blk in iter_blks(max, min=min):
        for _, tx in iter_tx(blk):
            for n, address, amount in find_tx_out_address(tx, address):
                bal = increment(bal, amount)
                yield 'out', address, amount, bal
            for n, address, amount in find_tx_in_address(tx, address):
                bal = increment(bal, -amount)
                yield 'in', address, amount, bal


def all_transactions(max=None, min=0):
    """Iterate through blocks yielding all transactions.
    yields (in or out, address, amount)
    """
    max = max or svc.getblockcount()
    for blk_n, _, blk in iter_blks(max, min=min):
        for _, tx in iter_tx(blk):
            for _, address, amount in parse_tx_in(tx):
                yield 'in', address, amount, blk_n
            for _, address, amount in parse_tx_out(tx):
                yield 'out', address, amount, blk_n


# Data Views
# ==========
def print_tx(tx):
    print "INS:"
    for n, address, amount in parse_tx_in(tx):
        pprint((n, address, amount))
    print "OUTS:"
    for n, address, amount in parse_tx_out(tx):
        pprint((n, address, amount))


def print_address_transactions(address, max=None, min=0):
    for tx_type, address, amount, bal in find_address_transactions(address, max, min=min):
        print tx_type, address, amount, bal


# =======


def update_book(book, address, amount):
    if address not in book:
        book[address] = 0.0
    book[address] = increment(book[address], amount)
    assert book[address] >= 0.0, "Incorrect parsing. Should not be negative addresses. %s." % address
    if book[address] == 0.0: # Still zero, delete it.
        del book[address]


def cull_book(book):
    print("Culling the book.")
    for address, amount in book.items():
        if amount == 0:
            del book[address]


def rich_book(book, max=None, min=0):
    i=0
    for direction, address, amount, blk_n in all_transactions(max=max, min=min):
        if direction is 'in':
            update_book(book, address, -amount)
        elif direction is 'out':
            update_book(book, address, amount)
        #Persisting too often. Perhaps a transaction counter instead of on block.
        i+=1
        if i % 1000 == 0:
            persist(book, meta={'last_blk': blk_n,
                                'timestamp': time.time()}, datakey="book")
            print("Persisted block %s of %s" % (blk_n, max))


def main(svc=svc):
    try:
        state = load()
    except IOError:
        print("No saved state. Starting at block 0.")
        last_blk = 0
        book = {}
    else:
        meta = state['meta']
        last_blk = meta['last_blk']
        book = state['book']
        print("Loaded state from %s." % (persist_file))
        print("Last block parsed: %s at %s" % (last_blk, meta['timestamp']))
        cull_book(book)

    blk_count = svc.getblockcount()
    rich_book(book, blk_count, min=last_blk)

    sorted_book = reversed(sorted(book.iteritems(), key=operator.itemgetter(1)))

    json_build = []
    for address, amount in sorted_book:
        json_build.append([address, amount])

    json.dump(json_build, open('sorted_book.json', 'w'), indent=2)



if __name__ == '__main__':
    main()


# Have no clue why the parser isn't parsing correctly for all addresses.
# That means time for tests.

import unittest

class TestParser(unittest.TestCase):
    def setUp(self):
        # Expecting the Cinnicoin blockchain.
        self.max = 50100
        self.min = 50000
        self.svc = AuthServiceProxy("http://12345:[email protected]:2222")

    def test_iter_blks(self):
        for blk_n, blk_hash, blk in iter_blks(self.max, min=self.min,
                                              svc=self.svc):
            self.assertIsInstance(blk_n, int)
            self.assertIsInstance(blk_hash, basestring)
            self.assertIsInstance(blk, dict)
            self.assertEqual(len(blk_hash), 64)
            self.assertGreater(blk['confirmations'], 1)

    def test_iter_tx(self):
        for _, _, blk in iter_blks(self.max, min=self.min,
                                              svc=self.svc):
            for tx_hash, tx in iter_tx(blk):
                self.assertIsInstance(tx_hash, basestring)
                self.assertIsInstance(tx, dict)
                self.assertEqual(len(tx_hash), 64)
                self.assertIn('vin', tx)
                self.assertIn('vout', tx)

    def test_zip_tx(self):
        for _, _, blk in iter_blks(self.max, min=self.min, svc=self.svc):
            for _, tx in iter_tx(blk):
                ins, outs = zip_tx(tx)
                for n, address, amount in ins:
                    self.assertIsInstance(n, int)
                    self.assertIsInstance(address, basestring)
                    self.assertIsInstance(amount, float)
                    self.assertEqual(len(address), 34)
                    self.assertGreaterEqual(amount, 0.0)
                for n, address, amount in outs:
                    self.assertIsInstance(n, int)
                    self.assertIsInstance(address, basestring)
                    self.assertIsInstance(amount, float)
                    self.assertEqual(len(address), 34) # this is the issue.
                    self.assertGreaterEqual(amount, 0.0)