Skip to content

Instantly share code, notes, and snippets.

@noleto
Created March 7, 2025 22:11
Show Gist options
  • Save noleto/dec3e678850b72cd4f647b7b9cb01700 to your computer and use it in GitHub Desktop.
Save noleto/dec3e678850b72cd4f647b7b9cb01700 to your computer and use it in GitHub Desktop.
Reading Bitcoin blocks in pseudo parallel using py-bitcoinkernel
from pathlib import Path
from bitcoin.core import CBlock
import pbk
import concurrent.futures
import threading
# ChainstateManager is currently not threadsafe so we need to wrap it
class ThreadSafeChainstateManager:
def __init__(self, chainman):
self.chainman = chainman
self.lock = threading.Lock()
def read_block_from_disk(self, block_index):
with self.lock:
return self.chainman.read_block_from_disk(block_index)
def dump_block(block_index):
block_data = safe_chainman.read_block_from_disk(block_index).data
cblock = CBlock.deserialize(block_data)
# Quick and dirty: Create a CSV record for the block
block_hash = cblock.GetHash().hex()
merkle_root = cblock.hashMerkleRoot.hex()
timestamp = cblock.nTime
difficulty = cblock.difficulty
tx_count = len(cblock.vtx)
# Format as CSV row
csv_row = f"{block_index.height},{block_hash},{merkle_root},{timestamp},{difficulty},{tx_count}\n"
# Append to CSV file (create if doesn't exist)
with open(f"blocks/block_{threading.current_thread().name}.csv", "a") as f:
f.write(csv_row)
if __name__ == "__main__":
# Enable logging in case libbitcoinkernel raises any erros/warnings
log = pbk.LoggingConnection()
# Set the datadir to the path of your bitcoin core installation (default is ~/.bitcoin)
datadir = Path("/home/<datadir>")
chainman = pbk.load_chainman(datadir, pbk.ChainType.MAINNET)
safe_chainman = ThreadSafeChainstateManager(chainman)
# Get all block indices first
block_indices = list(pbk.block_index_generator(chainman, start=0, end=1000))
# Process blocks in "parallel" using ThreadPoolExecutor (not really parallel, but it's the best we can do with the current libbitcoinkernel)
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
results = list(executor.map(dump_block, block_indices))
print("All blocks dumped")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment