Extracting raw big map values from lmdb

lmdb_big_map.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lmdb\n",
    "import itertools\n",
    "from pytezos import pytezos\n",
    "from pytezos.encoding import base58_decode, base58_encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = lmdb.open('/home/mickey/.tezos-node/context')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "txn = env.begin()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cursor = txn.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cursor.last()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(itertools.islice(cursor.iternext_nodup(), 50))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for item in itertools.islice(cursor.iternext_nodup(), 1000000):\n",
    "    if not item.startswith(b'contents'):\n",
    "        print(item)\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(b'B\\x00\\xfaB\\x85\\x0e\\xd7`\\xe5\\xc3\\x86\\xf9\\x84$\\x0e#\\xe0\\xd9\\x81\\x1e\\xc0 \\xb0\\xd9_\\x92\\xbb\\x06\\xe6\\xc6h\\x16')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "txn.get(b'commit/' + base58_decode(b'BMMJdpT2d8L6iRK8KGgDkwbyqBTfLdVJ6527sQVnXpXV3npXNSr'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "key = b'commit/' + base58_decode(b'CoUqY3A2iqYA1dfZtfsFepcCirHz6uFSpnbFw8VgvJeP3svJxwKf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = txn.get(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pytezos.shell.blocks[339819].header()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "int.from_bytes(data[4:8], 'big')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[8:40]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[40:44]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[44:48]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[48:52]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "int.from_bytes(data[52:56], 'big')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[56:88]  # predecessor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[88:92]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "int.from_bytes(data[92:96], 'big')  # timestamp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[96:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "int.from_bytes(data[100:104], 'big')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[104:109]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[109:113]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "int.from_bytes(data[113:117], 'big')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[117:157]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "node = txn.get(b'node/' + data[8:40])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "txn.get(b'commit/' + data[56:88])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "node"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "int.from_bytes(b'\\x00\\x00\\x03\\x00', 'big')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "node[21:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "int.from_bytes(node[25:29], 'big')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "node[29:61]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "txn.get(b'node/' + node[29:61])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "txn.get(b'node/' + b'\\x9e\\xb5\\xc6\\x8dn:_r\\xe7M\"\\xaf\\xc0\\xfc\\xb6\\x17\\xe6i\\xaa!\\x19\\x0c\\xab\\xae[\\xcc#\\xed\\xa2,I\\xf2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_node(key_hash):\n",
    "    # header 9 bytes (7 unknown, 2 flags)\n",
    "    # separator 4 bytes\n",
    "    # length of key\n",
    "    # key\n",
    "    # separator 4 bytes\n",
    "    # length of value\n",
    "    # value\n",
    "    # flag 1 byte\n",
    "    # separator 4 bytes\n",
    "    node = txn.get(b'node/' + key_hash)\n",
    "    \n",
    "    items = {}\n",
    "    ptr = 9\n",
    "    \n",
    "    while ptr < len(node):    \n",
    "        ptr += 4\n",
    "\n",
    "        key_len = int.from_bytes(node[ptr:ptr+4], 'big')\n",
    "        ptr += 4\n",
    "\n",
    "        key = node[ptr:ptr+key_len].decode()\n",
    "        ptr += key_len\n",
    "\n",
    "        ptr += 4\n",
    "\n",
    "        val_len = int.from_bytes(node[ptr:ptr+4], 'big')\n",
    "        ptr += 4\n",
    "\n",
    "        val = node[ptr:ptr+val_len]\n",
    "        ptr += val_len\n",
    "        \n",
    "        items[key] = val\n",
    "        \n",
    "        if ptr < len(node):\n",
    "            ptr += 1\n",
    "        \n",
    "    return items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nod = txn.get(b'node/' + data[8:40])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nod"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n1 = parse_node(nod)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n2 = parse_node(n1['data'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n3 = parse_node(n2['contracts'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n4 = parse_node(n3['index'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n5 = parse_node(n4['originated'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base58_decode(b'KT1P3j1VonQytW3b2SzCnGVpjdf3oWajM79E')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parse_node(n5['9e'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pprint import pformat\n",
    "\n",
    "\n",
    "def read_node(txn, key_hash):\n",
    "    ptr = 9  # skip header\n",
    "    node = txn.get(b'node/' + key_hash)\n",
    "    items = {}\n",
    "    \n",
    "    def read_data():\n",
    "        nonlocal ptr\n",
    "        ptr += 4\n",
    "        length = int.from_bytes(node[ptr:ptr + 4], 'big')\n",
    "        ptr += 4\n",
    "        data = node[ptr:ptr + length]\n",
    "        ptr += length\n",
    "        return data\n",
    "            \n",
    "    while ptr < len(node):\n",
    "        key = read_data().decode()\n",
    "        val = read_data()\n",
    "        items[key] = val\n",
    "        if ptr < len(node):\n",
    "            ptr += 1  # some flag\n",
    "            \n",
    "    return items\n",
    "        \n",
    "\n",
    "class IrminNode:\n",
    "    \n",
    "    def __init__(self, txn, key_hash):\n",
    "        self.txn = txn\n",
    "        self.items = read_node(txn, key_hash)        \n",
    "        \n",
    "    def __repr__(self):\n",
    "        return pformat(self.items)\n",
    "    \n",
    "    def __getitem__(self, item):\n",
    "        if item.startswith('KT'):\n",
    "            key_hash = base58_decode(b'KT1P3j1VonQytW3b2SzCnGVpjdf3oWajM79E').hex()\n",
    "            child = self\n",
    "            for i in range(5):\n",
    "                kh = key_hash[2 * i:2 * i + 2]\n",
    "                child = child[kh]\n",
    "            return child[key_hash[10:]]\n",
    "        else:\n",
    "            return IrminNode(txn, self.items[item])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "root = IrminNode(txn, data[8:40])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "root"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cc = root['data']['contracts']['index']['originated']['KT1P3j1VonQytW3b2SzCnGVpjdf3oWajM79E']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cc['big_map']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bmi = cc['big_map']['2f']['17']['7b']['89']['57']['c4bb8bfb1d757b465cc76024b6ae6c002dea5af493e4b3c8406e69']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bmi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "txn.get(b'contents/' + bmi.items['data'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "txn.get(b'contents/' + bmi.items['len'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}