Bartek's YOLO Go benchmarking scripts.

bench.sh

#!/usr/bin/env bash

set -e

# Copyright (c) Bartłomiej Płotka @bwplotka
# Licensed under the Apache License 2.0.

# Yolo script allowing nice benchmark framework for iterative work on Go performance.
# Requirements:
#  * Prepare worktree if you want to benchmark in background: git worktree add ../thanos_b yolo
# Example usage:
#  * Commit changes you want to test against e.g c513dab8752dc69a46e29de7a035680947778593.
#  * Name this comparable with some human name like `1mblimit`:
#  * Run bash bench.sh 1mblimit c513dab8752dc69a46e29de7a035680947778593 make sure WORK_DIR points to your repo.
#  * Expect results being printed and persisted in ./results for further use (e.g comparing via cmp.sh

# Shared @ https://gist.github.com/bwplotka/3b853c31ed11e77c975b9df45d105d74

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# TODO: Hardcoded for current use ):
WORK_DIR="${DIR}/../../../prometheus_b"

# TODO: Hardcoded, change it, or parametrize.
PKG="./tsdb"
TEST="BenchmarkDBBlocksQuerierSelect"
STEST=${TEST//\//-}

# TODO: Parametrize all via flags.
RUN="${1}"
COMMIT="${2}"
BENCH_TIME="2m"
DATE=$(date '+%Y-%m-%d-T%H-%M-%S')

echo "Checking out ${COMMIT} in ${WORK_DIR}"
cd "${WORK_DIR}"
git reset --hard "${COMMIT}"

echo "Running ${TEST} (sanitized: ${STEST}) in ${WORK_DIR} for commit ${COMMIT} as ${RUN}"
TEST_DIR="${DIR}/results/${STEST}/${RUN}"
OUT_FILE="${TEST_DIR}/${DATE}.bench.out"
CPU_PROF_FILE="${TEST_DIR}/${DATE}.cpu.pprof"
MEM_PROF_FILE="${TEST_DIR}/${DATE}.mem.pprof"

mkdir -p ${TEST_DIR}

# Give context.
echo "${COMMIT} ${TEST} ${PKG} 2h ${BENCH_TIME} ${WORK_DIR} on core #3" | tee ${OUT_FILE}
git --no-pager log --oneline -n 10 | tee -a ${OUT_FILE}

echo "-------- BEGIN BENCHMARK --------" | tee -a ${OUT_FILE}

# Alternatively add -memprofile ${TEST_DIR}/${DATE}.memprofile.out but only one package at the time allowed.
# Pinned to Core #3
taskset -c 3 go test -bench=${TEST} -run=^$ -cpuprofile ${CPU_PROF_FILE} -memprofile ${MEM_PROF_FILE} -benchmem -timeout 2h -benchtime ${BENCH_TIME} ${PKG} | tee -a ${OUT_FILE}

bench_local.sh

#!/usr/bin/env bash

set -e

# Copyright (c) Bartłomiej Płotka @bwplotka
# Licensed under the Apache License 2.0.

# Yolo script allowing nice benchmark framework for iterative work on Go performance.
# Requirements:
#  * Install github.com/aclements/perflock (+daemon).
#  * Prepare worktree if you want to benchmark in background: git worktree add ../thanos_b yolo
# Example usage:
#  * Commit changes you want to test against e.g c513dab8752dc69a46e29de7a035680947778593.
#  * Name this comparable with some human name like `1mblimit`:
#  * Run bash bench.sh 1mblimit c513dab8752dc69a46e29de7a035680947778593 make sure WORK_DIR points to your repo.
#  * Expect results being printed and persisted in ./results for further use (e.g comparing via cmp.sh

# Shared @ https://gist.github.com/bwplotka/3b853c31ed11e77c975b9df45d105d74

DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PKG="./pkg/store"
TEST="BenchmarkTelemeterRealData_Series"
STEST=${TEST//\//-}

# TODO: Parametrize all via flags.
RUN="${1}"
COMMIT=$(git rev-parse --short HEAD)
BENCH_TIME="2m"
DATE=$(date '+%Y-%m-%d-T%H-%M-%S')

echo "Running ${TEST} (sanitized: ${STEST}) in ${WORK_DIR} for commit ${COMMIT} as ${RUN}"
TEST_DIR="${DIR}/results/${STEST}/${RUN}"
OUT_FILE="${TEST_DIR}/${DATE}.bench.out"
CPU_PROF_FILE="${TEST_DIR}/${DATE}.cpu.pprof"
MEM_PROF_FILE="${TEST_DIR}/${DATE}.mem.pprof"

mkdir -p ${TEST_DIR}

# Give context.
echo "${COMMIT} ${TEST} ${PKG} 2h ${BENCH_TIME} ${WORK_DIR} on core #3" | tee ${OUT_FILE}
git --no-pager log --oneline -n 10 | tee -a ${OUT_FILE}

echo "-------- BEGIN BENCHMARK --------" | tee -a ${OUT_FILE}

# Alternatively add -memprofile ${TEST_DIR}/${DATE}.memprofile.out but only one package at the time allowed.
# Pinned to Core #3
taskset -c 3 go test -bench=${TEST} -run=^$ -cpuprofile ${CPU_PROF_FILE} -memprofile ${MEM_PROF_FILE} -benchmem -timeout 2h -benchtime ${BENCH_TIME} ${PKG} | tee -a ${OUT_FILE}

cmp.sh

#!/usr/bin/env bash

set -e

# Copyright (c) Bartłomiej Płotka @bwplotka
# Licensed under the Apache License 2.0.

# Yolo script allowing nice benchmark framework for iterative work on Go performance.
# Requirements:
#  * go install golang.org/x/perf/cmd/benchstat
#  * Results produce by previous bench.sh runs.
# Example usage:
#  * Commit changes you want to test against e.g c513dab8752dc69a46e29de7a035680947778593.
#  * Name this comparable with some human name like `1mblimit`:
#  * Run bash bench.sh 1mblimit c513dab8752dc69a46e29de7a035680947778593 make sure WORK_DIR points to your repo.
#  * Expect results being printed and persisted in ./results for further use (e.g comparing via cmp.sh
#  * Run some other version with different name: bash bench.sh nolimit a693a667f9eea1fbd56c7bc2a2140fb3c598ef34
#  * Compare using bash cmp.sh nolimit 1mblimt

# Shared @ https://gist.github.com/bwplotka/3b853c31ed11e77c975b9df45d105d74

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

OLD="${1}"
NEW="${2}"

# TODO: Hardcoded, change it, or parametrize.
TESTS="BenchmarkDBBlocksQuerierSelect"
DATE=$(date '+%Y-%m-%d-T%H-%M-%S')

for TEST in ${TESTS}
do
  STEST=${TEST//\//-}
  TEST_DIR="${DIR}/diffs/${STEST}/"
  mkdir -p ${TEST_DIR}

  # TODO(bwplotka): Support more results than just two.
  OLD_OUT=$(ls ${DIR}/results/${STEST}/${OLD}/*.bench.out | tail -n 1)
  OLD_SPEC=$(cat ${OLD_OUT} | head -n 1)
  NEW_OUT=$(ls ${DIR}/results/${STEST}/${NEW}/*.bench.out | tail -n 1)
  NEW_SPEC=$(cat ${OLD_OUT} | head -n 1)

  echo "Comparing '${OLD_OUT}' (${OLD_SPEC}) with (new) '${NEW_OUT}' (${NEW_SPEC})"
  benchstat -delta-test=none ${OLD_OUT} ${NEW_OUT} | tee "${TEST_DIR}/${DATE}.${OLD}-vs-${NEW}.diff.out"
done