This code uses a running deployment to test throughput in a range of situations. Replicas are changed elsewhere with MLOps UI or API

mlops_throughput.sh

#!/bin/bash
# Define the base command and endpoint
URL="https://model.dev.h2o.ai/496095fa-d429-4da0-b7d1-8239fe644410/model/score"
BASE_CMD="ab -T \"application/json\" "
TEST_SIZE=500

# Initialize the results table
echo "| Test Size | Batch Size | Replicas | Concurrency | r/s |"
echo "|-----------|------------|----------|-------------|-----|"
# Define test scenarios
payloads=(["1"]="payload_1_row.txt" ["10"]="payload_10_row.txt")
concurrency=(1 4 8 16 32)
# Loop through scenarios and run tests
for n in "${!payloads[@]}"; do
  for c in "${concurrency[@]}"; do
    # Run ApacheBench and extract Requests per second
    result=$($BASE_CMD -n $TEST_SIZE/$n -c $c -p "${payloads[$n]}" $URL | grep "Requests per second" | awk '{print $4}')
    product=$( echo "$result * $n" | bc )
    echo "| $TEST_SIZE      | $n         | 1        | $c           | $product |"
  done
done

output.md

Test Size	Batch Size	Replicas	Concurrency	r/s
100	1	1	1	3.20
100	10	1	1	32.30
100	100	1	1	319.00

payload_10_row.txt

 {
   "fields": [
      "Account Length","Area Code","Int'l Plan","VMail Plan","VMail Message","Day Mins","Day Calls",
      "Day Charge","Eve Mins","Eve Calls","Eve Charge","Night Mins","Night Calls","Night Charge",
      "Intl Mins","Intl Calls","Intl Charge","CustServ Calls"
   ],
   "rows": [
      ["0","0","text","text","0","0","0","0","0","0","0","0","0","0","0","0","0","0"],
      ["0","0","text","text","0","0","0","0","0","0","0","0","0","0","0","0","0","0"],
      ["0","0","text","text","0","0","0","0","0","0","0","0","0","0","0","0","0","0"],
      ["0","0","text","text","0","0","0","0","0","0","0","0","0","0","0","0","0","0"],
      ["0","0","text","text","0","0","0","0","0","0","0","0","0","0","0","0","0","0"],
      ["0","0","text","text","0","0","0","0","0","0","0","0","0","0","0","0","0","0"],
      ["0","0","text","text","0","0","0","0","0","0","0","0","0","0","0","0","0","0"],
      ["0","0","text","text","0","0","0","0","0","0","0","0","0","0","0","0","0","0"],
      ["0","0","text","text","0","0","0","0","0","0","0","0","0","0","0","0","0","0"],
      ["0","0","text","text","0","0","0","0","0","0","0","0","0","0","0","0","0","0"],
   ]
}

payload_1_row.txt

 {
   "fields": [
      "Account Length","Area Code","Int'l Plan","VMail Plan","VMail Message","Day Mins","Day Calls",
      "Day Charge","Eve Mins","Eve Calls","Eve Charge","Night Mins","Night Calls","Night Charge",
      "Intl Mins","Intl Calls","Intl Charge","CustServ Calls"
   ],
   "rows": [
      ["0","0","text","text","0","0","0","0","0","0","0","0","0","0","0","0","0","0"]
   ]
}