Skip to content

Instantly share code, notes, and snippets.

@0xB10C

0xB10C/bloom_nodes_kits_data.ipynb

Created May 1, 2023 10:49
Show Gist options
  • Save 0xB10C/4bd642db1b9710ae31e626c3d6143ff9 to your computer and use it in GitHub Desktop.
Save 0xB10C/4bd642db1b9710ae31e626c3d6143ff9 to your computer and use it in GitHub Desktop.
Download ZIP
Percentage of NODE_BLOOM nodes based on https://www.dsn.kastel.kit.edu/bitcoin/ data
Display the source blob
Display the rendered blob
Raw
bloom_nodes_kits_data.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "located-deviation",
"metadata": {
"id": "located-deviation"
},
"outputs": [],
"source": [
"import urllib.request, json \n",
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "continental-sensitivity",
"metadata": {
"id": "continental-sensitivity"
},
"outputs": [],
"source": [
"NODE_BLOOM = 4"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "married-verse",
"metadata": {
"id": "married-verse"
},
"outputs": [],
"source": [
"# https://www.dsn.kastel.kit.edu/bitcoin/data.html\n",
"# https://www.dsn.kastel.kit.edu/bitcoin/snapshots/\n",
"dataurl = \"https://www.dsn.kastel.kit.edu/bitcoin/snapshots/20230430_124551_dossier.json\""
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "legislative-bathroom",
"metadata": {
"id": "legislative-bathroom"
},
"outputs": [],
"source": [
"with urllib.request.urlopen(dataurl) as url:\n",
" data = json.loads(url.read().decode())"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "twelve-complex",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 206
},
"id": "twelve-complex",
"outputId": "824a215a-8d59-4d12-cbbc-60dc1f1b32bd"
},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
" lastConnect ip \\\n",
"9585cb2e28bf638b 2022-04-06T20:13:43 {'version': 4} \n",
"5d677e6453c509c5 2022-04-06T20:14:00 {'version': 4} \n",
"d547cbcd86d67952 2022-04-06T20:14:00 {'version': 6, 'tunnel': 'native'} \n",
"dfcd31830e84bffb 2022-04-06T20:14:01 {'version': 4} \n",
"810513fc04d08fb6 2022-04-06T20:14:08 {'version': 4} \n",
"\n",
" versionstr services versionid \\\n",
"9585cb2e28bf638b /Satoshi:0.21.0/ 1033 70016 \n",
"5d677e6453c509c5 /Satoshi:0.13.0/ 5 70014 \n",
"d547cbcd86d67952 /Satoshi:22.0.0/ 1037 70016 \n",
"dfcd31830e84bffb /Satoshi:0.18.1/ 1037 70015 \n",
"810513fc04d08fb6 /Satoshi:0.19.0.1/ 1033 70015 \n",
"\n",
" whois \\\n",
"9585cb2e28bf638b {'asn': '15169', 'asn_country_code': 'US', 'na... \n",
"5d677e6453c509c5 {'asn': '48614', 'asn_country_code': 'CZ', 'na... \n",
"d547cbcd86d67952 {'asn': '24940', 'asn_country_code': 'DE', 'na... \n",
"dfcd31830e84bffb {'asn': '16276', 'asn_country_code': 'FR', 'na... \n",
"810513fc04d08fb6 {'asn': '24940', 'asn_country_code': 'DE', 'na... \n",
"\n",
" geo \\\n",
"9585cb2e28bf638b {'lat': 39.9044, 'long': -85.9929} \n",
"5d677e6453c509c5 {'lat': 55.7386, 'long': 37.6068} \n",
"d547cbcd86d67952 {'lat': 51.0, 'long': 9.0} \n",
"dfcd31830e84bffb {'lat': 48.9018, 'long': 2.4893} \n",
"810513fc04d08fb6 {'lat': 60.1708, 'long': 24.9375} \n",
"\n",
" latency \\\n",
"9585cb2e28bf638b {'icmp': {'median': 57, 'count': 60, 'mean': 5... \n",
"5d677e6453c509c5 NaN \n",
"d547cbcd86d67952 {'btc': {'median': 6, 'count': 30, 'mean': 73,... \n",
"dfcd31830e84bffb {'icmp': {'median': 6, 'count': 60, 'mean': 5,... \n",
"810513fc04d08fb6 {'icmp': {'median': 11, 'count': 60, 'mean': 1... \n",
"\n",
" inv \n",
"9585cb2e28bf638b {'invperhour': 11665, 'txperhour': 9502, 'bloc... \n",
"5d677e6453c509c5 NaN \n",
"d547cbcd86d67952 {'invperhour': 25313, 'txperhour': 22335, 'blo... \n",
"dfcd31830e84bffb {'invperhour': 10684, 'txperhour': 8767, 'bloc... \n",
"810513fc04d08fb6 {'invperhour': 11655, 'txperhour': 9494, 'bloc... "
],
"text/html": [
"\n",
" <div id=\"df-2a9968ec-c0c5-4c66-b96c-e6c367478db2\">\n",
" <div class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>lastConnect</th>\n",
" <th>ip</th>\n",
" <th>versionstr</th>\n",
" <th>services</th>\n",
" <th>versionid</th>\n",
" <th>whois</th>\n",
" <th>geo</th>\n",
" <th>latency</th>\n",
" <th>inv</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>9585cb2e28bf638b</th>\n",
" <td>2022-04-06T20:13:43</td>\n",
" <td>{'version': 4}</td>\n",
" <td>/Satoshi:0.21.0/</td>\n",
" <td>1033</td>\n",
" <td>70016</td>\n",
" <td>{'asn': '15169', 'asn_country_code': 'US', 'na...</td>\n",
" <td>{'lat': 39.9044, 'long': -85.9929}</td>\n",
" <td>{'icmp': {'median': 57, 'count': 60, 'mean': 5...</td>\n",
" <td>{'invperhour': 11665, 'txperhour': 9502, 'bloc...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5d677e6453c509c5</th>\n",
" <td>2022-04-06T20:14:00</td>\n",
" <td>{'version': 4}</td>\n",
" <td>/Satoshi:0.13.0/</td>\n",
" <td>5</td>\n",
" <td>70014</td>\n",
" <td>{'asn': '48614', 'asn_country_code': 'CZ', 'na...</td>\n",
" <td>{'lat': 55.7386, 'long': 37.6068}</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>d547cbcd86d67952</th>\n",
" <td>2022-04-06T20:14:00</td>\n",
" <td>{'version': 6, 'tunnel': 'native'}</td>\n",
" <td>/Satoshi:22.0.0/</td>\n",
" <td>1037</td>\n",
" <td>70016</td>\n",
" <td>{'asn': '24940', 'asn_country_code': 'DE', 'na...</td>\n",
" <td>{'lat': 51.0, 'long': 9.0}</td>\n",
" <td>{'btc': {'median': 6, 'count': 30, 'mean': 73,...</td>\n",
" <td>{'invperhour': 25313, 'txperhour': 22335, 'blo...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>dfcd31830e84bffb</th>\n",
" <td>2022-04-06T20:14:01</td>\n",
" <td>{'version': 4}</td>\n",
" <td>/Satoshi:0.18.1/</td>\n",
" <td>1037</td>\n",
" <td>70015</td>\n",
" <td>{'asn': '16276', 'asn_country_code': 'FR', 'na...</td>\n",
" <td>{'lat': 48.9018, 'long': 2.4893}</td>\n",
" <td>{'icmp': {'median': 6, 'count': 60, 'mean': 5,...</td>\n",
" <td>{'invperhour': 10684, 'txperhour': 8767, 'bloc...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>810513fc04d08fb6</th>\n",
" <td>2022-04-06T20:14:08</td>\n",
" <td>{'version': 4}</td>\n",
" <td>/Satoshi:0.19.0.1/</td>\n",
" <td>1033</td>\n",
" <td>70015</td>\n",
" <td>{'asn': '24940', 'asn_country_code': 'DE', 'na...</td>\n",
" <td>{'lat': 60.1708, 'long': 24.9375}</td>\n",
" <td>{'icmp': {'median': 11, 'count': 60, 'mean': 1...</td>\n",
" <td>{'invperhour': 11655, 'txperhour': 9494, 'bloc...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2a9968ec-c0c5-4c66-b96c-e6c367478db2')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
" \n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
" width=\"24px\">\n",
" <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
" <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
" </svg>\n",
" </button>\n",
" \n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" flex-wrap:wrap;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-2a9968ec-c0c5-4c66-b96c-e6c367478db2 button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-2a9968ec-c0c5-4c66-b96c-e6c367478db2');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
" </div>\n",
" "
]
},
"metadata": {},
"execution_count": 4
}
],
"source": [
"df = pd.DataFrame(data).transpose()\n",
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "signal-official",
"metadata": {
"id": "signal-official"
},
"outputs": [],
"source": [
"df[\"NODE_BLOOM\"] = df.services.fillna(0).apply(lambda x: bool(int(x) & NODE_BLOOM))"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "short-needle",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "short-needle",
"outputId": "d6f24b48-34d1-4594-90c6-2a0c5a0535c9"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Publicly reachable nodes with NODE_BLOOM: 1733 (22%)\n"
]
}
],
"source": [
"print(\n",
" f\"Publicly reachable nodes with NODE_BLOOM: {len(df[df.NODE_BLOOM])} ({round(len(df[df.NODE_BLOOM]) / len(df)*100)}%)\"\n",
")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 for machine learning",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.9"
},
"colab": {
"provenance": []
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment