Created
March 7, 2025 22:11
-
-
Save noleto/dec3e678850b72cd4f647b7b9cb01700 to your computer and use it in GitHub Desktop.
Reading Bitcoin blocks in pseudo parallel using py-bitcoinkernel
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from pathlib import Path | |
from bitcoin.core import CBlock | |
import pbk | |
import concurrent.futures | |
import threading | |
# ChainstateManager is currently not threadsafe so we need to wrap it | |
class ThreadSafeChainstateManager: | |
def __init__(self, chainman): | |
self.chainman = chainman | |
self.lock = threading.Lock() | |
def read_block_from_disk(self, block_index): | |
with self.lock: | |
return self.chainman.read_block_from_disk(block_index) | |
def dump_block(block_index): | |
block_data = safe_chainman.read_block_from_disk(block_index).data | |
cblock = CBlock.deserialize(block_data) | |
# Quick and dirty: Create a CSV record for the block | |
block_hash = cblock.GetHash().hex() | |
merkle_root = cblock.hashMerkleRoot.hex() | |
timestamp = cblock.nTime | |
difficulty = cblock.difficulty | |
tx_count = len(cblock.vtx) | |
# Format as CSV row | |
csv_row = f"{block_index.height},{block_hash},{merkle_root},{timestamp},{difficulty},{tx_count}\n" | |
# Append to CSV file (create if doesn't exist) | |
with open(f"blocks/block_{threading.current_thread().name}.csv", "a") as f: | |
f.write(csv_row) | |
if __name__ == "__main__": | |
# Enable logging in case libbitcoinkernel raises any erros/warnings | |
log = pbk.LoggingConnection() | |
# Set the datadir to the path of your bitcoin core installation (default is ~/.bitcoin) | |
datadir = Path("/home/<datadir>") | |
chainman = pbk.load_chainman(datadir, pbk.ChainType.MAINNET) | |
safe_chainman = ThreadSafeChainstateManager(chainman) | |
# Get all block indices first | |
block_indices = list(pbk.block_index_generator(chainman, start=0, end=1000)) | |
# Process blocks in "parallel" using ThreadPoolExecutor (not really parallel, but it's the best we can do with the current libbitcoinkernel) | |
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: | |
results = list(executor.map(dump_block, block_indices)) | |
print("All blocks dumped") | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment