docker_invocation

docker run --entrypoint /bin/bash -it gcr.io/o1labs-192920/mina-daemon:2.0.0umt -c 'curl https://gist.githubusercontent.com/emberian/cefc1a198b0f81b5f1a442dfd43a42f1/raw/fetch-and-verify.sh | bash'

fetch-and-verify.sh

curl https://storage.googleapis.com/o1labs-gitops-infrastructure/o1labs-umt-pre-fork-run-1/o1labs-umt-pre-fork-run-1-state-dump-3NLnD1Yp4MS9LtMXikD1YyySZNVgCXA82b5eQVpmYZ5kyTo4Xsr7-0f4e15012e2de8fbc16ea9c003ea851714dfafbbb6467508128ee4595123451d.json.gz | gunzip > fork_config.json
curl -O https://gist.githubusercontent.com/emberian/cefc1a198b0f81b5f1a442dfd43a42f1/raw/mina-verify-packaged-fork-config-nowait
chmod +x mina-verify-packaged-fork-config-nowait
./mina-verify-packaged-fork-config-nowait fork_config.json /tmp/verification-nowait

mina-verify-packaged-fork-config-nowait

#!/usr/bin/env bash

set -eo pipefail

if [ $# -lt 2 ]; then
    echo "Usage: $0 <mainnet-fork-config.json> <working-dir>"
    cat <<EOF
This script is used to validate that an installed package is correct
according to an exported fork_config.json file.

Inputs:
- The exported mainnet full config.json fork config, with all accounts
- A working directory where ledgers/configs will be created
- Installed MINA_EXE (default: mina) and MINA_GENESIS_EXE (default: mina-create-genesis) programs
- PACKAGED_DAEMON_CONFIG (default: /var/lib/coda/config_*.json)
  the runtime config generated by the HF packaging
- CREATE_RUNTIME_CONFIG (default: mina-hf-create-runtime-config)
  a copy of ./scripts/hardfork/create_runtime_config.sh
- GENESIS_LEDGER_DIR (default: /var/lib/coda)
  where the tarfiles to verify are stored
- FORKING_FROM_CONFIG_JSON (default: /var/lib/coda/mainnet.json)
  the pre-fork genesis ledger
- SECONDS_PER_SLOT (default: 180)

Ensures:
- The accounts listed in config.json are the ones in the PACKAGED_DAEMON_CONFIG
- The genesis ledger directory tarfile contents match reference copies created
  fresh from config.json. This takes a long time (>20min) due to rehashing.

Outputs:
- Exit code 0 if validated, 1 otherwise.
EOF
    
    exit 1
fi

source_build_fallback() {
    if [ ! -e "$1" ]; then
        if [ ! -e "$2" ]; then
            echo "Error: did not find useable $1 or $2" >&2
            exit 1
        else
            echo "$2"
        fi
    else
      echo "$1"
    fi
}

MINA_EXE=${MINA_EXE:-$(source_build_fallback "$(command -v mina)" ./_build/default/src/app/cli/src/mina.exe)}
MINA_GENESIS_EXE=${MINA_GENESIS_EXE:-$(source_build_fallback "$(command -v mina-create-genesis)" ./_build/default/src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe)}
CREATE_RUNTIME_CONFIG=${CREATE_RUNTIME_CONFIG:-$(source_build_fallback "$(command -v mina-hf-create-runtime-config)" ./scripts/hardfork/create_runtime_config.sh)}

installed_config=$(echo /var/lib/coda/config_*.json)
PACKAGED_DAEMON_CONFIG=${PACKAGED_DAEMON_CONFIG:-$installed_config}
if [ ! -e "$installed_config" ]; then
    echo "Error: set PACKAGED_DAEMON_CONFIG to the path to the JSON file to verify"
    exit 1
fi


GENESIS_LEDGER_DIR=${GENESIS_LEDGER_DIR:-"/var/lib/coda"}

SECONDS_PER_SLOT=${SECONDS_PER_SLOT:=180}

export FORKING_FROM_CONFIG_JSON=${FORKING_FROM_CONFIG_JSON:-$(source_build_fallback /var/lib/coda/mainnet.json genesis_ledgers/mainnet.json)}

export MINA_LIBP2P_PASS=''

workdir=$2
mkdir -p "$workdir"
mkdir -p "$workdir/ledgers"
mkdir -p "$workdir/ledgers-backup"
mkdir -p "$workdir/keys"
chmod 700 "$workdir/keys"

if [ ! -e "$workdir/keys/p2p" ]; then
    "$MINA_EXE" libp2p generate-keypair --privkey-path "$workdir/keys/p2p"
fi

echo "generating genesis ledgers ... (this may take a while)" >&2

cp "$1" "$workdir/config.json"
sed -i -e 's/"set_verification_key": "signature"/"set_verification_key": {"auth": "signature", "txn_version": "1"}/' "$workdir/config.json"
"$MINA_GENESIS_EXE" --config-file "$workdir/config.json"  --genesis-dir "$workdir/ledgers"  --hash-output-file "$workdir/hashes.json"

FORK_CONFIG_JSON="$1" \
LEDGER_HASHES_JSON="$workdir/hashes.json" \
GENESIS_TIMESTAMP=$(jq -r '.genesis.genesis_state_timestamp' "$PACKAGED_DAEMON_CONFIG") \
"$CREATE_RUNTIME_CONFIG" > "$workdir/config-substituted.json"

echo "exporting ledgers from running node ... (this may take a while)" >&2

# export the packaged ledgers in a way where we know which one is which
function extract_ledgers() {
    config_file=$1
    ledger_dir=$2
    json_prefix=$3
    "$MINA_EXE" daemon --libp2p-keypair "$workdir/keys/p2p" --config-file "$config_file" --seed --genesis-ledger-dir "$ledger_dir" &
    
    while ! "$MINA_EXE" ledger export staged-ledger  | jq >"$json_prefix-staged.json"; do
        sleep 1m
        if ! grep -qFx "$(cat ~/.mina-config/.mina-lock)" <(jobs -rp); then
            echo "daemon died before exporting ledgers" >&2
            exit 1
        fi
    done
    
    "$MINA_EXE" ledger export staking-epoch-ledger | jq > "$json_prefix-staking.json"
    "$MINA_EXE" ledger export next-epoch-ledger | jq > "$json_prefix-next.json"
    
    "$MINA_EXE" client stop
}

#extract_ledgers "$PACKAGED_DAEMON_CONFIG" "$GENESIS_LEDGER_DIR" "$workdir/packaged"
#mv -t "$workdir/ledgers-backup" /var/lib/coda/*.tar.gz
#extract_ledgers "$workdir/config-substituted.json" "$workdir/ledgers" "$workdir/reference"
#mv -t /var/lib/coda "$workdir/ledgers-backup"/*

echo "Performing final comparisons..." >&2
error=0

result=$(jq --slurpfile a "$workdir/config-substituted.json" --slurpfile b "$PACKAGED_DAEMON_CONFIG" -n '
  ($a[0].epoch_data.staking.hash == $b[0].epoch_data.staking.hash and
   $a[0].epoch_data.next.hash == $b[0].epoch_data.next.hash and
   $a[0].ledger.hash == $b[0].ledger.hash)')

if [ "$result" != "true" ]; then
    echo "Packaged config hashes in $PACKAGED_DAEMON_CONFIG not expected compared to $workdir/config-substituted.json" >&2
    error=1
fi

#for file in "$workdir"/packaged-*.json; do
#    name=$(basename "$file")
#    name=${name%.json}
#    name=${name#packaged-}
#    
#    if ! cmp "$file" "$workdir/reference-$name.json"; then
#        echo "Error: $file does not match reference" >&2
#        error=1
#    fi
#done

ldb_cmd=$(command -v rocksdb-ldb || command -v rocksdb_ldb || command -v ldb)
for file in "$workdir"/ledgers/*.tar.gz; do
    tarname=$(basename "$file")
    tarname=${tarname%.tar.gz}
    tardir="$workdir/ledgers/$tarname"
    mkdir -p "$tardir"/{packaged,generated,web}
    tar -xzf "$file" -C "$tardir/generated"
    tar -xzf "$GENESIS_LEDGER_DIR/$tarname.tar.gz" -C "$tardir/packaged"
    curl "https://s3-us-west-2.amazonaws.com/snark-keys.o1test.net/$tarname.tar.gz" | tar -xz -C "$tardir/web"

    $ldb_cmd --hex --db="$tardir/packaged" scan > "$workdir/packaged.scan"
    $ldb_cmd --hex --db="$tardir/web" scan > "$workdir/web.scan"
    $ldb_cmd --hex --db="$tardir/generated" scan > "$workdir/generated.scan"

    if ! cmp "$workdir/generated.scan" "$workdir/packaged.scan" || ! cmp "$workdir/packaged.scan" "$workdir/web.scan"; then
        echo "Error: kvdb contents mismatch for $tarname" >&2
        error=1
    fi
done

if [ $error -ne 0 ]; then
    echo "Error: failed validation" >&2
    exit 1
else
    echo "Validation successful" >&2
    exit 0
fi

mina-verify-packaged-fork-config.patch

--- mina-verify-packaged-fork-config-nowait	2024-02-29 16:40:51.025999259 +0000
+++ mina-verify-packaged-fork-config	2024-02-29 16:41:54.670813302 +0000
@@ -112,10 +112,10 @@
     "$MINA_EXE" client stop
 }
 
-#extract_ledgers "$PACKAGED_DAEMON_CONFIG" "$GENESIS_LEDGER_DIR" "$workdir/packaged"
-#mv -t "$workdir/ledgers-backup" /var/lib/coda/*.tar.gz
-#extract_ledgers "$workdir/config-substituted.json" "$workdir/ledgers" "$workdir/reference"
-#mv -t /var/lib/coda "$workdir/ledgers-backup"/*
+extract_ledgers "$PACKAGED_DAEMON_CONFIG" "$GENESIS_LEDGER_DIR" "$workdir/packaged"
+mv -t "$workdir/ledgers-backup" /var/lib/coda/*.tar.gz
+extract_ledgers "$workdir/config-substituted.json" "$workdir/ledgers" "$workdir/reference"
+mv -t /var/lib/coda "$workdir/ledgers-backup"/*
 
 echo "Performing final comparisons..." >&2
 error=0
@@ -130,16 +130,16 @@
     error=1
 fi
 
-#for file in "$workdir"/packaged-*.json; do
-#    name=$(basename "$file")
-#    name=${name%.json}
-#    name=${name#packaged-}
-#    
-#    if ! cmp "$file" "$workdir/reference-$name.json"; then
-#        echo "Error: $file does not match reference" >&2
-#        error=1
-#    fi
-#done
+for file in "$workdir"/packaged-*.json; do
+    name=$(basename "$file")
+    name=${name%.json}
+    name=${name#packaged-}
+    
+    if ! cmp "$file" "$workdir/reference-$name.json"; then
+        echo "Error: $file does not match reference" >&2
+        error=1
+    fi
+done
 
 ldb_cmd=$(command -v rocksdb-ldb || command -v rocksdb_ldb || command -v ldb)
 for file in "$workdir"/ledgers/*.tar.gz; do

packaged_config_reference.json

{
  "genesis": {
    "genesis_state_timestamp": "2024-02-29T21:00:00Z"
  },
  "proof": {
    "fork": {
      "state_hash": "3NLnD1Yp4MS9LtMXikD1YyySZNVgCXA82b5eQVpmYZ5kyTo4Xsr7",
      "blockchain_length": 1276,
      "global_slot_since_genesis": 3100
    }
  },
  "ledger": {
    "add_genesis_winner": false,
    "hash": "jx5cxmCuZzR6Qmidwm9nVqoEePJTLfLgSSYntM4C8AZGx9DyTUm",
    "s3_data_hash": "75d7398c275dcb5e4ef15179dd355a236d399e58ac133133752cd7d97d2bcc81"
  },
  "epoch_data": {
    "staking": {
      "seed": "2vbZRewiphtua2p2cv6gTuvG3a4pa9HeCZHECNmZSzLrvswaxNjq",
      "hash": "jwvTmFhQ7hFZefVhW4WvcobsLLoE1QPa835irgrXxkJGBA9pRoq",
      "s3_data_hash": "e1bbeaefbc1c63340b75ad0d98127f2f9ae4e1fbddd1ac5ec1b4648ca8bf8762"
    },
    "next": {
      "seed": "2vbmjbc5xYycXjL7d2xGVtMmoPuN5FpWbQMr13zHoekyf23BcQP8",
      "hash": "jxP12zezrW1e68Rd8YmgEDno4qkRdvhEPvzYbUH3zGEe5jFFSkX",
      "s3_data_hash": "11dae58352dec0d278452f923ec770eb0e36b6210d34a8542348e998d4859ab4"
    }
  }
}