Nano Ledger/LMDB Account parsing (computes balance of account at every send and receive block, identifies send and receive address + amounts, stores to file)

traverser.py

######NOTE#######
#This script was written in python 2.7... no guarantees it works on python 3 without modification.
import lmdb
from pyblake2 import blake2b
import matplotlib.pyplot as plt

#CHANGE THIS TO YOUR DATABASE LOCATION (usually /home/user/RaiBlocks/)
#ALSO!! Python doesn't see the database unless it has extension .mdb
#however your nano-node/wallet won't see it unless it stays named as .ldb
#I suggest creating a symlink as follows: ln -s /PATH/TO/YOUR/DATABASE/data.ldb /PATH/TO/YOUR/DATABASE/data.mdb

lmdb_env = lmdb.open("/home/mike/RaiBlocks/",max_dbs=128)
lmdb_begin = lmdb_env.begin()
send_db = lmdb_env.open_db(key='send',txn=lmdb_begin,create=False) 
receive_db = lmdb_env.open_db(key='receive',txn=lmdb_begin,create=False)
open_db = lmdb_env.open_db(key='open',txn=lmdb_begin,create=False)
change_db = lmdb_env.open_db(key='change',txn=lmdb_begin,create=False)
send_curs = lmdb_begin.cursor(send_db)
receive_curs = lmdb_begin.cursor(receive_db)
open_curs = lmdb_begin.cursor(open_db)
change_curs = lmdb_begin.cursor(change_db)

GENESIS_AMT = (2**128) - 1
GENESIS_SRC_HASH = '\xe8\x92\x08\xdd\x03\x8f\xbb&\x99\x87h\x96!\xd5"\x92\xae\x9c5\x94\x1at\x84un\xcc\xed\x92\xa6P\x93\xba'

account_lookup = "13456789abcdefghijkmnopqrstuwxyz"

###### BLOCK HASH TO BE TRAVERSED BACKWARDS FROM ########
starting_hash = 0xEDE6B93CC8D96C6A7E94C578D624F45B81BBA74C0D72CC771A6033D51A49CA27 #bg1
#starting_hash = 0xB6D2EDBA44FBC0BAE1B1EE2ABAADA68B55EFBC5A944C6A6BBBDB3525A05EE037 #bg2
#starting_hash = 0x26C12FAE1F5BA06B8314CA406F68348382BA61C496BBC47E1312B79CA48B8FC6 #binance
#starting_hash = 0x6716F053723AD96C93C6288071ABA7BE2CAC58EC5BA55B6E0535BFDAD7491C20 #mercatox


def raw2addr(r):
	#hash
	hb = blake2b(digest_size=5)
	hb.update(r)
	check = hb.digest()
	#convert to bytes
	tmp = ''
	for z in check:
		tmp = z+tmp
	b_check = int(tmp.encode('hex'),16)
	b = [ord(c) for c in r]
	result = long(0)
	shift = 0	
	for i in b:
		result <<= shift
		result |= i
		shift = 8

	number_l = result
	number_l <<= 40
	number_l |= b_check
	z = ''
	for i in range(0,60):
		z = z + account_lookup[number_l & 0x1F]
		number_l >>= 5
		
	z = z + '_brx'
	return z[::-1]


def int2hash(h):
	h=hex(h)
	#convert to key_string
	k=2
	key_str = ''
	for i in range(2,len(h)/2 +1):
		key_str = key_str + chr(int('0x'+h[k:k+2],16))
		k=k+2
	return key_str


def block_get(h1):
	block_prev = send_curs.get(h1)
	if block_prev != None:
		#Found a send block
		return ['send',block_prev]
	else:
		block_prev = receive_curs.get(h1)
		if block_prev != None:
			#found rx
			return ['receive',block_prev]
		else:
			block_prev = open_curs.get(h1)
			if block_prev != None:
				return ['open',block_prev]
			else:
				block_prev = change_curs.get(h1)
				if block_prev != None:
					return ['change',block_prev]
				else:
					print "CANT FIND HASH FAILURE!"
					print h1.encode('hex')
					return ['','']
			


#Function used recursively to traverse the ledger until we find 2 subsequent send blocks or the genesis acct
#From there we work backwards to compute all the deltas
def traverse_until_genesis(h1):
	#STEP 1,traverse down this account's chain until we find A) another send block or B) an open block
	#For any open or receive blocks we store their hash's as we need to traverse their ledger to get their amounts
	hash_list = []
	while(1):
		block_prev = block_get(h1)
		if block_prev[0] == 'send':
			#found send, record balance
			balance_old = long(block_prev[1][64:80].encode('hex'),16)
			break
		elif block_prev[0] == 'receive':
			#Found receive, add its matching send block to list and keep traversing
			hash_list.append(block_prev[1][32:64])
			h1 = block_prev[1][0:32]
			continue
		elif block_prev[0] == 'open':
			#found open, set balance to zero, and treat as a receive block
			if block_prev[1][0:32] == GENESIS_SRC_HASH:
				balance_old = GENESIS_AMT #For genesis account, there is no matching "send" , only keep balance
			else:			
				balance_old = 0
				hash_list.append(block_prev[1][0:32]) #this grabs source hash which is the matching send hash
			break
		elif block_prev[0] == 'change':
			#change blocks have no effect for us, keep moving downward
			h1 = block_prev[1][0:32]
			continue
		else:
			print "FAILED TO FIND A HASH!!!"
			break
	
	#STEP 2, take each list of hashes for the matching send blocks, record their balance field
	#And then call traverse_until_genesis to compute the delta on each
	tmp_total_sends = 0
	for i in hash_list:
		tmp_block = send_curs.get(i)
		tmp_total_sends += traverse_until_genesis(tmp_block[0:32]) - long(tmp_block[64:80].encode('hex'),16)

	#Finally, return the total account balance (initial balance of the older send we found + the amount of receive's he got inbetween)
	return balance_old + tmp_total_sends

#This function is used to traverse a given account's block chain until we find
#either a receive block or an open block. The purpose of this is to find this
#account's address. 
#Input should be the hash of a send block from chain B to chain A (that we found
#from a chain A receive block). We are traversing chain B's ledger to find his addr
#address.
def traverse_until_receive_or_open(h1):
	prev = h1
	#start the loop
	while(1):
		block_prev = block_get(prev)
		if block_prev[0] == 'send':
			#found send, keep going
			prev = block_prev[1][0:32]
			continue
		elif block_prev[0] == 'receive':
			#Found receive,find it's match, extract its dest addr, and return
			match = send_curs.get(block_prev[1][32:64])
			return match[32:64]
		elif block_prev[0] == 'open':
			#found open, return the address 
			return block_prev[1][64:96]
		elif block_prev[0] == 'change':
			#change blocks have no effect for us, keep moving downward
			prev = block_prev[1][0:32]
			continue
		else:
			print "FAILED TO FIND A HASH!!!"
			break
		

data = []
event_number = 0
h = int2hash(starting_hash)

#Iterating down an accounts ledger grabbing all of the balance's noted in his send blocks
#And computing all of the amount of receive transactions that he has
while True:
	raw = block_get(h)

	if raw[0] == 'send':
		h = raw[1][0:32]
		#grab the destination address
		destaddr = raw2addr(raw[1][32:64])
		#Note that balance is the sender's acct balance after removing sent funds
		balance = long(raw[1][64:80].encode('hex'),16)
		#save the current account balance and destination for the send		
		data.insert(0,['send',balance,destaddr])
		event_number += 1
	elif raw[0] == 'receive':
		h=raw[1][0:32]
		match =raw[1][32:64]
		#Identify who sent this transaction to us
		senders_addr = raw2addr(traverse_until_receive_or_open(match))
		#Get the matching send block from node j
		raw = send_curs.get(match)
		#save the balance at node j
		balance_1 = long(raw[64:80].encode('hex'),16)
		#prev block for node j			
		match_prev = raw[0:32]
		balance_2 = traverse_until_genesis(match_prev)

		data.insert(0,['receive',balance_2 - balance_1, senders_addr])
	elif raw[0] == 'open':
		#found an opening
		match =raw[1][0:32]
		#Identify who sent this transaction to us
		senders_addr = raw2addr(traverse_until_receive_or_open(match))
		#Get the matching send block from node j
		raw = send_curs.get(match)
		#save the balance at node j
		balance_1 = long(raw[64:80].encode('hex'),16)
		#prev block for node j			
		match_prev = raw[0:32]
		balance_2 = traverse_until_genesis(match_prev)
		data.insert(0,['receive',balance_2 - balance_1,senders_addr])
		break
	elif raw[0] == 'change':
		#change blocks have no effect for us, keep moving downward
		h = raw[1][0:32]
		continue
	else:
		print "FAILED TO FIND A HASH!!!"
		break
		



#Now we loop through the list and compute the send amounts, we do this by also
#keeping a running total of the balance
deposits_withdrawals = []
account_balance = [0]

k=1
for i in data:
	if i[0] == 'receive':
		account_balance.append(i[1]+account_balance[k-1])
		deposits_withdrawals.append(i)
	else:
		#amuont from send transaction is just the balance after removing send amt
		account_balance.append(i[1])
		#send out of the account, compute the delta
		delta = account_balance[k-1] - i[1]
		deposits_withdrawals.append(['send',delta,i[2]])
	
	k+=1
 

#save to file 
#NOTE, order of transactions is from oldest to newest!!
thefile = open("BG1_account_deposits_withdrawals.txt","w")
for item in deposits_withdrawals:
	thefile.write("%s\n" % item)

thefile = open("BG1_account_balances.txt","w")
for item in account_balance:
	thefile.write("%s\n" % item)

#plot account balances
#plot in regular xrb units
account_balance_XRB  = [x / (10**30) for x in account_balance]
plt.plot(account_balance_XRB)
plt.show()	

Note: in order to identify the address of someone who sent the account a deposit, I added the traverse_until_receive_or_open function to traverse that account's ledger until we can find information to identify their address. The downside is that this appears to slow things quite a bit, so if you're not concerned in the address of people sending to the account, you can edit this out.

On second thought, maybe its not much slower. I was able to do BG1 and BG2 full traverse in <2min.