Reading Bitcoin blocks in pseudo parallel using py-bitcoinkernel

parallel_block_dump.py

from pathlib import Path
from bitcoin.core import CBlock
import pbk
import concurrent.futures
import threading

# ChainstateManager is currently not threadsafe so we need to wrap it
class ThreadSafeChainstateManager:
    def __init__(self, chainman):
        self.chainman = chainman
        self.lock = threading.Lock()
    
    def read_block_from_disk(self, block_index):
        with self.lock:
            return self.chainman.read_block_from_disk(block_index)

def dump_block(block_index):
    block_data = safe_chainman.read_block_from_disk(block_index).data
    cblock = CBlock.deserialize(block_data)
    
    # Quick and dirty: Create a CSV record for the block
    block_hash = cblock.GetHash().hex()
    merkle_root = cblock.hashMerkleRoot.hex()
    timestamp = cblock.nTime
    difficulty = cblock.difficulty
    tx_count = len(cblock.vtx)
    
    # Format as CSV row
    csv_row = f"{block_index.height},{block_hash},{merkle_root},{timestamp},{difficulty},{tx_count}\n"
    
    # Append to CSV file (create if doesn't exist)
    with open(f"blocks/block_{threading.current_thread().name}.csv", "a") as f:
        f.write(csv_row)
            
if __name__ == "__main__":
    # Enable logging in case libbitcoinkernel raises any erros/warnings
    log = pbk.LoggingConnection()
    
    # Set the datadir to the path of your bitcoin core installation (default is ~/.bitcoin)
    datadir = Path("/home/<datadir>")
    chainman = pbk.load_chainman(datadir, pbk.ChainType.MAINNET)
    safe_chainman = ThreadSafeChainstateManager(chainman)
    

    # Get all block indices first
    block_indices = list(pbk.block_index_generator(chainman, start=0, end=1000))
    
    # Process blocks in "parallel" using ThreadPoolExecutor (not really parallel, but it's the best we can do with the current libbitcoinkernel)
    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
        results = list(executor.map(dump_block, block_indices))
    
    print("All blocks dumped")