Skip to content

Instantly share code, notes, and snippets.

@str4d

str4d/extract_p2sh_opcodes.py

Last active December 27, 2020 21:00
Show Gist options
  • Save str4d/9d80f1b60e6787310897044502cb025b to your computer and use it in GitHub Desktop.
Save str4d/9d80f1b60e6787310897044502cb025b to your computer and use it in GitHub Desktop.
Download ZIP
Zcash P2SH metadata collection from zcashd RPCs
Raw
extract_p2sh_opcodes.py
import csv
from enum import IntEnum
FILENAME = 'zcash-p2sh-data.csv'
class OpCode(IntEnum):
# push value
OP_0 = 0x00
OP_FALSE = OP_0
OP_PUSHDATA1 = 0x4c
OP_PUSHDATA2 = 0x4d
OP_PUSHDATA4 = 0x4e
OP_1NEGATE = 0x4f
OP_RESERVED = 0x50
OP_1 = 0x51
OP_TRUE=OP_1
OP_2 = 0x52
OP_3 = 0x53
OP_4 = 0x54
OP_5 = 0x55
OP_6 = 0x56
OP_7 = 0x57
OP_8 = 0x58
OP_9 = 0x59
OP_10 = 0x5a
OP_11 = 0x5b
OP_12 = 0x5c
OP_13 = 0x5d
OP_14 = 0x5e
OP_15 = 0x5f
OP_16 = 0x60
# control
OP_NOP = 0x61
OP_VER = 0x62
OP_IF = 0x63
OP_NOTIF = 0x64
OP_VERIF = 0x65
OP_VERNOTIF = 0x66
OP_ELSE = 0x67
OP_ENDIF = 0x68
OP_VERIFY = 0x69
OP_RETURN = 0x6a
# stack ops
OP_TOALTSTACK = 0x6b
OP_FROMALTSTACK = 0x6c
OP_2DROP = 0x6d
OP_2DUP = 0x6e
OP_3DUP = 0x6f
OP_2OVER = 0x70
OP_2ROT = 0x71
OP_2SWAP = 0x72
OP_IFDUP = 0x73
OP_DEPTH = 0x74
OP_DROP = 0x75
OP_DUP = 0x76
OP_NIP = 0x77
OP_OVER = 0x78
OP_PICK = 0x79
OP_ROLL = 0x7a
OP_ROT = 0x7b
OP_SWAP = 0x7c
OP_TUCK = 0x7d
# splice ops
OP_CAT = 0x7e
OP_SUBSTR = 0x7f
OP_LEFT = 0x80
OP_RIGHT = 0x81
OP_SIZE = 0x82
# bit logic
OP_INVERT = 0x83
OP_AND = 0x84
OP_OR = 0x85
OP_XOR = 0x86
OP_EQUAL = 0x87
OP_EQUALVERIFY = 0x88
OP_RESERVED1 = 0x89
OP_RESERVED2 = 0x8a
# numeric
OP_1ADD = 0x8b
OP_1SUB = 0x8c
OP_2MUL = 0x8d
OP_2DIV = 0x8e
OP_NEGATE = 0x8f
OP_ABS = 0x90
OP_NOT = 0x91
OP_0NOTEQUAL = 0x92
OP_ADD = 0x93
OP_SUB = 0x94
OP_MUL = 0x95
OP_DIV = 0x96
OP_MOD = 0x97
OP_LSHIFT = 0x98
OP_RSHIFT = 0x99
OP_BOOLAND = 0x9a
OP_BOOLOR = 0x9b
OP_NUMEQUAL = 0x9c
OP_NUMEQUALVERIFY = 0x9d
OP_NUMNOTEQUAL = 0x9e
OP_LESSTHAN = 0x9f
OP_GREATERTHAN = 0xa0
OP_LESSTHANOREQUAL = 0xa1
OP_GREATERTHANOREQUAL = 0xa2
OP_MIN = 0xa3
OP_MAX = 0xa4
OP_WITHIN = 0xa5
# crypto
OP_RIPEMD160 = 0xa6
OP_SHA1 = 0xa7
OP_SHA256 = 0xa8
OP_HASH160 = 0xa9
OP_HASH256 = 0xaa
OP_CODESEPARATOR = 0xab
OP_CHECKSIG = 0xac
OP_CHECKSIGVERIFY = 0xad
OP_CHECKMULTISIG = 0xae
OP_CHECKMULTISIGVERIFY = 0xaf
# expansion
OP_NOP1 = 0xb0
OP_CHECKLOCKTIMEVERIFY = 0xb1
OP_NOP3 = 0xb2
OP_NOP4 = 0xb3
OP_NOP5 = 0xb4
OP_NOP6 = 0xb5
OP_NOP7 = 0xb6
OP_NOP8 = 0xb7
OP_NOP9 = 0xb8
OP_NOP10 = 0xb9
# template matching params
OP_SMALLDATA = 0xf9
OP_SMALLINTEGER = 0xfa
OP_PUBKEYS = 0xfb
OP_PUBKEYHASH = 0xfd
OP_PUBKEY = 0xfe
OP_INVALIDOPCODE = 0xff
LENGTHS = [
OpCode.OP_0,
OpCode.OP_1,
OpCode.OP_2,
OpCode.OP_3,
OpCode.OP_4,
OpCode.OP_5,
OpCode.OP_6,
OpCode.OP_7,
OpCode.OP_8,
OpCode.OP_9,
]
def to_opcodes(script):
script = bytes.fromhex(script)
parsed = []
i = 0
while i < len(script):
opcode = script[i]
i += 1
if opcode > OpCode.OP_0 and opcode <= OpCode.OP_PUSHDATA4:
if opcode < OpCode.OP_PUSHDATA1:
size = opcode
elif opcode == OpCode.OP_PUSHDATA1:
size = script[i]
i += 1
elif opcode == OpCode.OP_PUSHDATA2:
size = int.from_bytes(script[i:i+2], 'little')
i += 2
elif opcode == OpCode.OP_PUSHDATA4:
size = int.from_bytes(script[i:i+4], 'little')
i += 4
if i + size > len(script):
raise ValueError
data = script[i:i+size]
i += size
parsed.append(data.hex())
else:
parsed.append(OpCode(opcode))
return parsed
def label_script(script):
# Detect t-of-n multisig scripts
if script[-1] == OpCode.OP_CHECKMULTISIG:
if script[0] in LENGTHS and script[-2] in LENGTHS:
t = LENGTHS.index(script[0])
n = LENGTHS.index(script[-2])
pubkeys = script[1:-2]
if len(pubkeys) == n and [type(x) for x in pubkeys] == [type('')] * n:
if pubkeys == ['<data:33>'] * n:
kind = 'compressed'
elif pubkeys == ['<data:65>'] * n:
kind = 'uncompressed'
else:
kind = 'mixed'
return '{}-of-{} multisig with {} pubkeys'.format(t, n, kind)
# Templates we have identified before
if script == [
OpCode.OP_2, '<data:33>', '<data:33>', '<data:33>', OpCode.OP_3, OpCode.OP_CHECKMULTISIGVERIFY,
'<data:4>', OpCode.OP_DROP,
OpCode.OP_DEPTH, OpCode.OP_0, OpCode.OP_EQUAL]:
return '2-of-3 multisig with compressed pubkeys, a 4-byte ignored data value, and an empty stack check'
if script == [
OpCode.OP_1, '<data:33>', '<data:33>', '<data:33>', OpCode.OP_3, OpCode.OP_CHECKMULTISIGVERIFY,
OpCode.OP_2, '<data:33>', '<data:33>', OpCode.OP_2, OpCode.OP_CHECKMULTISIG]:
return '1-of-3 and 2-of-2 combined multisig with compressed pubkeys'
if script == [
'<data:32>', OpCode.OP_DROP,
OpCode.OP_DUP,
OpCode.OP_HASH160, '<data:20>', OpCode.OP_EQUALVERIFY,
OpCode.OP_CHECKSIG]:
return 'P2PKH inside P2SH with a 32-byte ignored data value'
if script == [
OpCode.OP_0, OpCode.OP_DROP,
OpCode.OP_DUP,
OpCode.OP_HASH160, '<data:20>', OpCode.OP_EQUALVERIFY,
OpCode.OP_CHECKSIG]:
return 'P2PKH inside P2SH with a zero-value placeholder ignored data value'
if script == ['<data:33>', OpCode.OP_CHECKSIG]:
return 'Pay-to-(compressed-)pubkey inside P2SH'
if script == [
'<data:33>', OpCode.OP_CHECKSIGVERIFY,
OpCode.OP_0, OpCode.OP_DROP,
OpCode.OP_DEPTH, OpCode.OP_0, OpCode.OP_EQUAL]:
return 'Pay-to-(compressed-)pubkey inside P2SH with an empty stack check'
if script == [
OpCode.OP_IF,
'<data:4>', OpCode.OP_CHECKLOCKTIMEVERIFY, OpCode.OP_DROP,
'<data:33>', OpCode.OP_CHECKSIG,
OpCode.OP_ELSE,
OpCode.OP_HASH160, '<data:20>', OpCode.OP_EQUALVERIFY,
'<data:33>', OpCode.OP_CHECKSIG,
OpCode.OP_ENDIF]:
return 'Hash160 HTLC'
if script == [
OpCode.OP_IF,
'<data:4>', OpCode.OP_CHECKLOCKTIMEVERIFY, OpCode.OP_DROP,
'<data:33>', OpCode.OP_CHECKSIG,
OpCode.OP_ELSE,
OpCode.OP_SIZE, '20', OpCode.OP_EQUALVERIFY,
OpCode.OP_HASH160, '<data:20>', OpCode.OP_EQUALVERIFY,
'<data:33>', OpCode.OP_CHECKSIG,
OpCode.OP_ENDIF]:
return 'Hash160 HTLC with size check'
if script == [
OpCode.OP_IF,
OpCode.OP_HASH160, '<data:20>', OpCode.OP_EQUALVERIFY,
OpCode.OP_DUP,
OpCode.OP_HASH160, '<data:20>',
OpCode.OP_ELSE,
'<data:4>', OpCode.OP_CHECKLOCKTIMEVERIFY, OpCode.OP_DROP,
OpCode.OP_DUP,
OpCode.OP_HASH160, '<data:20>',
OpCode.OP_ENDIF,
OpCode.OP_EQUALVERIFY,
OpCode.OP_CHECKSIG]:
return 'Hash160 HTLC'
if script == [
OpCode.OP_IF,
'<data:4>', OpCode.OP_CHECKLOCKTIMEVERIFY, OpCode.OP_DROP,
'<data:33>', OpCode.OP_CHECKSIG,
OpCode.OP_ELSE,
OpCode.OP_SHA256, '<data:32>', OpCode.OP_EQUALVERIFY,
'<data:33>', OpCode.OP_CHECKSIG,
OpCode.OP_ENDIF]:
return 'SHA-256 HTLC'
if script == [
OpCode.OP_IF,
OpCode.OP_SHA256, '<data:32>', OpCode.OP_EQUALVERIFY,
OpCode.OP_DUP,
OpCode.OP_HASH160, '<data:20>',
OpCode.OP_ELSE,
'<data:2>', OpCode.OP_CHECKLOCKTIMEVERIFY, OpCode.OP_DROP, # 2-byte CLTV
OpCode.OP_DUP,
OpCode.OP_HASH160, '<data:20>',
OpCode.OP_ENDIF,
OpCode.OP_EQUALVERIFY,
OpCode.OP_CHECKSIG]:
return 'SHA-256 HTLC (2-byte CLTV)'
if script == [
OpCode.OP_IF,
OpCode.OP_SHA256, '<data:32>', OpCode.OP_EQUALVERIFY,
OpCode.OP_DUP,
OpCode.OP_HASH160, '<data:20>',
OpCode.OP_ELSE,
'<data:3>', OpCode.OP_CHECKLOCKTIMEVERIFY, OpCode.OP_DROP, # 3-byte CLTV
OpCode.OP_DUP,
OpCode.OP_HASH160, '<data:20>',
OpCode.OP_ENDIF,
OpCode.OP_EQUALVERIFY,
OpCode.OP_CHECKSIG]:
return 'SHA-256 HTLC (3-byte CLTV)'
if script == [
OpCode.OP_IF,
OpCode.OP_SIZE, '20', OpCode.OP_EQUALVERIFY,
OpCode.OP_SHA256, '<data:32>', OpCode.OP_EQUALVERIFY,
OpCode.OP_DUP,
OpCode.OP_HASH160, '<data:20>',
OpCode.OP_ELSE,
'<data:4>', OpCode.OP_CHECKLOCKTIMEVERIFY, OpCode.OP_DROP, # 4-byte CLTV
OpCode.OP_DUP,
OpCode.OP_HASH160, '<data:20>',
OpCode.OP_ENDIF,
OpCode.OP_EQUALVERIFY,
OpCode.OP_CHECKSIG]:
return 'SHA-256 HTLC with size check'
if script == [OpCode.OP_SHA256, '<data:32>', OpCode.OP_EQUAL]:
return 'SHA-256 hashlock'
if script == [
OpCode.OP_IF,
OpCode.OP_SHA256, '<data:32>', OpCode.OP_EQUALVERIFY,
OpCode.OP_DUP,
OpCode.OP_HASH160, '<data:20>',
OpCode.OP_ELSE,
OpCode.OP_1, OpCode.OP_DROP,
OpCode.OP_HASH160, '<data:20>',
OpCode.OP_ENDIF,
OpCode.OP_EQUALVERIFY,
OpCode.OP_CHECKSIG]:
return 'One party has SHA-256 hashlock, other party can spend unconditionally'
if script == [
OpCode.OP_IF,
'<data:4>', OpCode.OP_CHECKLOCKTIMEVERIFY, OpCode.OP_DROP,
OpCode.OP_SIZE, '20', OpCode.OP_EQUALVERIFY,
OpCode.OP_HASH160, '<data:20>', OpCode.OP_EQUALVERIFY,
'<data:33>', OpCode.OP_CHECKSIG,
OpCode.OP_ELSE,
OpCode.OP_SIZE, '20', OpCode.OP_EQUALVERIFY,
OpCode.OP_HASH160, '<data:20>', OpCode.OP_EQUALVERIFY,
'<data:33>', OpCode.OP_CHECKSIG,
OpCode.OP_ENDIF]:
return 'Two-sided Hash160 HTLC with size checks'
# Unknown script template
return None
def main():
script_types = {}
script_labels = {}
with open(FILENAME, 'r') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
script_sig = to_opcodes(row['firstScriptSig'])
if not script_sig:
continue
p2sh_script = to_opcodes(script_sig[-1])
def templateify(x):
if type(x) == type(''):
if len(x) > 2:
return '<data:%d>' % (len(x)/2)
else:
return x
else:
return x
def stringify(x):
if type(x) == type(''):
return x
else:
return x.name
p2sh_template = [templateify(x) for x in p2sh_script]
p2sh_string = ' '.join([stringify(x) for x in p2sh_template])
if script_types.get(p2sh_string):
script_types[p2sh_string] += 1
else:
script_types[p2sh_string] = 1
p2sh_label = label_script(p2sh_template)
if p2sh_label is None:
p2sh_label = p2sh_string
script_labels[p2sh_string] = p2sh_label
script_types = sorted(script_types.items(), key=lambda item: item[1], reverse=True)
for script, occurrences in script_types:
print('%d: %s' % (occurrences, script))
print()
print('With labels:')
print()
for script, occurrences in script_types:
print('%d: %s' % (occurrences, script_labels[script]))
if __name__ == '__main__':
main()
Raw
fetch_p2sh_data.py
from bloom_filter import BloomFilter
import csv
from slickrpc import Proxy
try:
import progressbar
except:
progressbar = None
print('Install the progressbar2 module to show a progress bar')
DATA_FILENAME = 'zcash-p2sh-data.csv'
STATS_FILENAME = 'zcash-p2sh-stats.csv'
FR_ADDRESSES = [
't3Vz22vK5z2LcKEdg16Yv4FFneEL1zg9ojd',
't3cL9AucCajm3HXDhb5jBnJK2vapVoXsop3',
't3fqvkzrrNaMcamkQMwAyHRjfDdM2xQvDTR',
't3TgZ9ZT2CTSK44AnUPi6qeNaHa2eC7pUyF',
't3SpkcPQPfuRYHsP5vz3Pv86PgKo5m9KVmx',
't3Xt4oQMRPagwbpQqkgAViQgtST4VoSWR6S',
't3ayBkZ4w6kKXynwoHZFUSSgXRKtogTXNgb',
't3adJBQuaa21u7NxbR8YMzp3km3TbSZ4MGB',
't3K4aLYagSSBySdrfAGGeUd5H9z5Qvz88t2',
't3RYnsc5nhEvKiva3ZPhfRSk7eyh1CrA6Rk',
't3Ut4KUq2ZSMTPNE67pBU5LqYCi2q36KpXQ',
't3ZnCNAvgu6CSyHm1vWtrx3aiN98dSAGpnD',
't3fB9cB3eSYim64BS9xfwAHQUKLgQQroBDG',
't3cwZfKNNj2vXMAHBQeewm6pXhKFdhk18kD',
't3YcoujXfspWy7rbNUsGKxFEWZqNstGpeG4',
't3bLvCLigc6rbNrUTS5NwkgyVrZcZumTRa4',
't3VvHWa7r3oy67YtU4LZKGCWa2J6eGHvShi',
't3eF9X6X2dSo7MCvTjfZEzwWrVzquxRLNeY',
't3esCNwwmcyc8i9qQfyTbYhTqmYXZ9AwK3X',
't3M4jN7hYE2e27yLsuQPPjuVek81WV3VbBj',
't3gGWxdC67CYNoBbPjNvrrWLAWxPqZLxrVY',
't3LTWeoxeWPbmdkUD3NWBquk4WkazhFBmvU',
't3P5KKX97gXYFSaSjJPiruQEX84yF5z3Tjq',
't3f3T3nCWsEpzmD35VK62JgQfFig74dV8C9',
't3Rqonuzz7afkF7156ZA4vi4iimRSEn41hj',
't3fJZ5jYsyxDtvNrWBeoMbvJaQCj4JJgbgX',
't3Pnbg7XjP7FGPBUuz75H65aczphHgkpoJW',
't3WeKQDxCijL5X7rwFem1MTL9ZwVJkUFhpF',
't3Y9FNi26J7UtAUC4moaETLbMo8KS1Be6ME',
't3aNRLLsL2y8xcjPheZZwFy3Pcv7CsTwBec',
't3gQDEavk5VzAAHK8TrQu2BWDLxEiF1unBm',
't3Rbykhx1TUFrgXrmBYrAJe2STxRKFL7G9r',
't3aaW4aTdP7a8d1VTE1Bod2yhbeggHgMajR',
't3YEiAa6uEjXwFL2v5ztU1fn3yKgzMQqNyo',
't3g1yUUwt2PbmDvMDevTCPWUcbDatL2iQGP',
't3dPWnep6YqGPuY1CecgbeZrY9iUwH8Yd4z',
't3QRZXHDPh2hwU46iQs2776kRuuWfwFp4dV',
't3enhACRxi1ZD7e8ePomVGKn7wp7N9fFJ3r',
't3PkLgT71TnF112nSwBToXsD77yNbx2gJJY',
't3LQtHUDoe7ZhhvddRv4vnaoNAhCr2f4oFN',
't3fNcdBUbycvbCtsD2n9q3LuxG7jVPvFB8L',
't3dKojUU2EMjs28nHV84TvkVEUDu1M1FaEx',
't3aKH6NiWN1ofGd8c19rZiqgYpkJ3n679ME',
't3MEXDF9Wsi63KwpPuQdD6by32Mw2bNTbEa',
't3WDhPfik343yNmPTqtkZAoQZeqA83K7Y3f',
't3PSn5TbMMAEw7Eu36DYctFezRzpX1hzf3M',
't3R3Y5vnBLrEn8L6wFjPjBLnxSUQsKnmFpv',
't3Pcm737EsVkGTbhsu2NekKtJeG92mvYyoN',
]
DEV_FUND_ECC_ADDRESSES = [
't3LmX1cxWPPPqL4TZHx42HU3U5ghbFjRiif',
't3Toxk1vJQ6UjWQ42tUJz2rV2feUWkpbTDs',
't3ZBdBe4iokmsjdhMuwkxEdqMCFN16YxKe6',
't3ZuaJziLM8xZ32rjDUzVjVtyYdDSz8GLWB',
't3bAtYWa4bi8VrtvqySxnbr5uqcG9czQGTZ',
't3dktADfb5Rmxncpe1HS5BRS5Gcj7MZWYBi',
't3hgskquvKKoCtvxw86yN7q8bzwRxNgUZmc',
't3R1VrLzwcxAZzkX4mX3KGbWpNsgtYtMntj',
't3ff6fhemqPMVujD3AQurxRxTdvS1pPSaa2',
't3cEUQFG3KYnFG6qYhPxSNgGi3HDjUPwC3J',
't3WR9F5U4QvUFqqx9zFmwT6xFqduqRRXnaa',
't3PYc1LWngrdUrJJbHkYPCKvJuvJjcm85Ch',
't3bgkjiUeatWNkhxY3cWyLbTxKksAfk561R',
't3Z5rrR8zahxUpZ8itmCKhMSfxiKjUp5Dk5',
't3PU1j7YW3fJ67jUbkGhSRto8qK2qXCUiW3',
't3S3yaT7EwNLaFZCamfsxxKwamQW2aRGEkh',
't3eutXKJ9tEaPSxZpmowhzKhPfJvmtwTEZK',
't3gbTb7brxLdVVghSPSd3ycGxzHbUpukeDm',
't3UCKW2LrHFqPMQFEbZn6FpjqnhAAbfpMYR',
't3NyHsrnYbqaySoQqEQRyTWkjvM2PLkU7Uu',
't3QEFL6acxuZwiXtW3YvV6njDVGjJ1qeaRo',
't3PdBRr2S1XTDzrV8bnZkXF3SJcrzHWe1wj',
't3ZWyRPpWRo23pKxTLtWsnfEKeq9T4XPxKM',
't3he6QytKCTydhpztykFsSsb9PmBT5JBZLi',
't3VWxWDsLb2TURNEP6tA1ZSeQzUmPKFNxRY',
't3NmWLvZkbciNAipauzsFRMxoZGqmtJksbz',
't3cKr4YxVPvPBG1mCvzaoTTdBNokohsRJ8n',
't3T3smGZn6BoSFXWWXa1RaoQdcyaFjMfuYK',
't3gkDUe9Gm4GGpjMk86TiJZqhztBVMiUSSA',
't3eretuBeBXFHe5jAqeSpUS1cpxVh51fAeb',
't3dN8g9zi2UGJdixGe9txeSxeofLS9t3yFQ',
't3S799pq9sYBFwccRecoTJ3SvQXRHPrHqvx',
't3fhYnv1S5dXwau7GED3c1XErzt4n4vDxmf',
't3cmE3vsBc5xfDJKXXZdpydCPSdZqt6AcNi',
't3h5fPdjJVHaH4HwynYDM5BB3J7uQaoUwKi',
't3Ma35c68BgRX8sdLDJ6WR1PCrKiWHG4Da9',
't3LokMKPL1J8rkJZvVpfuH7dLu6oUWqZKQK',
't3WFFGbEbhJWnASZxVLw2iTJBZfJGGX73mM',
't3L8GLEsUn4QHNaRYcX3EGyXmQ8kjpT1zTa',
't3PgfByBhaBSkH8uq4nYJ9ZBX4NhGCJBVYm',
't3WecsqKDhWXD4JAgBVcnaCC2itzyNZhJrv',
't3ZG9cSfopnsMQupKW5v9sTotjcP5P6RTbn',
't3hC1Ywb5zDwUYYV8LwhvF5rZ6m49jxXSG5',
't3VgMqDL15ZcyQDeqBsBW3W6rzfftrWP2yB',
't3LC94Y6BwLoDtBoK2NuewaEbnko1zvR9rm',
't3cWCUZJR3GtALaTcatrrpNJ3MGbMFVLRwQ',
't3YYF4rPLVxDcF9hHFsXyc5Yq1TFfbojCY6',
't3XHAGxRP2FNfhAjxGjxbrQPYtQQjc3RCQD',
]
DEV_FUND_ZF_ADDRESS = 't3dvVE3SQEi7kqNzwrfNePxZ1d4hUyztBA1'
DEV_FUND_MG_ADDRESS = 't3XyYW8yBFRuMnfvm5KLGFbEVz25kckZXym'
def fetch_data():
# Block height 1 (genesis block coinbase details can't be accessed)
cur_block_hash = '0007bc227e1c57a4a70e237cad00e7b7ce565155ab49166bc57397a26d339283'
rpc_connection = Proxy(conf_file='/home/str4d/.zcash/zcash.conf')
utxos = {}
p2sh_addrs = {}
other_types = {}
def init_other_type(t, max_elements):
other_types[t] = {}
other_types[t]['filter'] = BloomFilter(max_elements, error_rate = 0.1)
other_types[t]['scripts'] = 0
other_types[t]['outputs'] = 0
init_other_type('pubkey', 100000)
init_other_type('pubkeyhash', 10000000)
init_other_type('multisig', 10000)
init_other_type('nulldata', 50000)
init_other_type('nonstandard', 10000)
cur_height = rpc_connection.getblockcount()
pbar = progressbar.ProgressBar(max_value=cur_height) if progressbar else None
with open(DATA_FILENAME, 'a') as datafile:
with open(STATS_FILENAME, 'a') as statsfile:
data_fieldnames = [
'address',
'firstSeenHeight',
'firstSeenTxId',
'firstSeenPos',
'firstScriptSig',
'firstSpentHeight',
'firstSpentTxId',
'firstSpentPos',
]
data_writer = csv.DictWriter(datafile, data_fieldnames, delimiter=',',
quotechar='|', quoting=csv.QUOTE_MINIMAL)
data_writer.writeheader()
stats_fieldnames = [
'height',
'time',
'p2shAddrs',
'spentP2shAddrs',
'unknownP2shUtxos',
'pubkey',
'pubkeyOutputs',
'pubkeyHash',
'pubkeyHashOutputs',
'multisig',
'multisigOutputs',
'nullData',
'nullDataOutputs',
'nonStandard',
'nonStandardOutputs',
]
stats_writer = csv.DictWriter(statsfile, stats_fieldnames, delimiter=',',
quotechar='|', quoting=csv.QUOTE_MINIMAL)
stats_writer.writeheader()
while cur_block_hash:
block_data = rpc_connection.getblock(cur_block_hash)
height = block_data['height']
tx_data = [rpc_connection.getrawtransaction(txid, 1) for txid in block_data['tx']]
for tx in tx_data:
txid = tx['txid']
# Scan for the use of existing P2SH addresses
for i in range(len(tx['vin'])):
txin = tx['vin'][i]
if txin.get('coinbase') is not None:
continue
utxo = (txin['txid'], txin['vout'])
p2sh = utxos.get(utxo)
if p2sh is not None:
# We've found the script for a P2SH address
scriptSig = txin['scriptSig']['hex']
data_writer.writerow({
'address': p2sh,
'firstSeenHeight': p2sh_addrs[p2sh]['firstSeenHeight'],
'firstSeenTxId': p2sh_addrs[p2sh]['firstSeenTxId'],
'firstSeenPos': p2sh_addrs[p2sh]['firstSeenPos'],
'firstScriptSig': scriptSig,
'firstSpentHeight': height,
'firstSpentTxId': txid,
'firstSpentPos': i,
})
if p2sh_addrs[p2sh].get('utxos') is None:
print('Error!')
print(utxo)
print(p2sh)
print(p2sh_addrs)
# No need to track any more UTXOs for this address
for txo in p2sh_addrs[p2sh]['utxos']:
del utxos[txo]
p2sh_addrs[p2sh]['utxos'] = None
# Scan for new P2SH addresses
for i in range(len(tx['vout'])):
txout = tx['vout'][i]
out_type = txout['scriptPubKey']['type']
if out_type == 'scripthash':
addresses = txout['scriptPubKey']['addresses']
if len(addresses) > 1:
print('Unexpected additional addresses in tx %s:' % txid)
print(addresses)
p2sh = addresses[0]
utxo = (txid, txout['n'])
if p2sh in (
FR_ADDRESSES +
DEV_FUND_ECC_ADDRESSES +
[DEV_FUND_ZF_ADDRESS, DEV_FUND_MG_ADDRESS]
):
# Ignore
pass
elif p2sh_addrs.get(p2sh) is None:
# We haven't seen this
p2sh_addrs[p2sh] = {
'firstSeenHeight': height,
'firstSeenTxId': txid,
'firstSeenPos': i,
'utxos': [utxo],
}
utxos[utxo] = p2sh
elif p2sh_addrs[p2sh]['utxos'] is not None:
# We are still looking for the script
p2sh_addrs[p2sh]['utxos'].append(utxo)
utxos[utxo] = p2sh
else:
# Add a prefix to handle zero-length scriptPubKeys
filter_key = 'scriptPubKey' + txout['scriptPubKey']['hex']
if filter_key not in other_types[out_type]['filter']:
other_types[out_type]['scripts'] = other_types[out_type]['scripts'] + 1
other_types[out_type]['filter'].add(filter_key)
other_types[out_type]['outputs'] = other_types[out_type]['outputs'] + 1
if height % 1000 == 0:
stats_writer.writerow({
'height': height,
'time': block_data['time'],
'p2shAddrs': len(p2sh_addrs),
'spentP2shAddrs': len([x for x in p2sh_addrs if p2sh_addrs[x]['utxos'] is None]),
'unknownP2shUtxos': len(utxos),
'pubkey': other_types['pubkey']['scripts'],
'pubkeyOutputs': other_types['pubkey']['outputs'],
'pubkeyHash': other_types['pubkeyhash']['scripts'],
'pubkeyHashOutputs': other_types['pubkeyhash']['outputs'],
'multisig': other_types['multisig']['scripts'],
'multisigOutputs': other_types['multisig']['outputs'],
'nullData': other_types['nulldata']['scripts'],
'nullDataOutputs': other_types['nulldata']['outputs'],
'nonStandard': other_types['nonstandard']['scripts'],
'nonStandardOutputs': other_types['nonstandard']['outputs'],
})
if pbar and height <= cur_height:
pbar.update(height)
cur_block_hash = block_data.get('nextblockhash', None)
# Write out P2SH addresses we haven't seen yet
for (p2sh, data) in p2sh_addrs.items():
if data['utxos'] is not None:
data_writer.writerow({
'address': p2sh,
'firstSeenHeight': data['firstSeenHeight'],
'firstSeenTxId': data['firstSeenTxId'],
'firstSeenPos': data['firstSeenPos'],
})
if __name__ == '__main__':
fetch_data()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment