Skip to content

Instantly share code, notes, and snippets.

Query operation_type count(*) avg(end_to_end_latency_in_ms) min(end_to_end_latency_in_ms) max(end_to_end_latency_in_ms) avg(server_latency_in_ms) min(server_latency_in_ms) max(server_latency_in_ms) E2E99thP E2E999thP E2E9999thP
q10 CopyBlob 50 17.76 7 95 17.76 7 95 95 95 95
q10 CopyBlobDestination 50 17.76 7 95 17.76 7 95 95 95 95
q10 CopyBlobSource 50 17.76 7 95 17.76 7 95 95 95 95
q10 DeleteBlob 52 4.980769231 3 38 4.980769231 3 38 38 38 38
q10 GetBlob 31242 199.13293 1 13570 76.33883874 1 667 1251 4287 9187
q10 GetBlobProperties 56321 2.278102306 0 147 2.278102306 0 147 147 147 147
q10 GetContainerProperties 2549 2.770498235 1 127 2.770498235 1 127 127 127 127
q10 GetPageRegions 123 2.06504065 1 4 1.975609756 1 4 4 4 4
q10 ListBlobs 2269 3.374614368 1 158 2.927721463 1 158 158 158 158
operation_type E2E99thP
CopyBlob 95
CopyBlobDestination 95
CopyBlobSource 95
DeleteBlob 38
GetBlob 1251
GetBlobProperties 147
GetContainerProperties 127
GetPageRegions 4
ListBlobs 158
operation_type count(*) avg(end_to_end_latency_in_ms) min(end_to_end_latency_in_ms) max(end_to_end_latency_in_ms) avg(server_latency_in_ms) min(server_latency_in_ms) max(server_latency_in_ms)
CopyBlob 60 44.18333333 6 296 44.18333333 6 296
CopyBlobDestination 60 44.18333333 6 296 44.18333333 6 296
CopyBlobSource 60 44.18333333 6 296 44.18333333 6 296
DeleteBlob 62 5.290322581 2 58 5.290322581 2 58
GetBlob 48491 100.4425564 2 5754 73.59421336 1 722
GetBlobProperties 68634 2.367631203 1 206 2.367631203 1 206
GetContainerProperties 2605 2.731669866 1 138 2.731669866 1 138
GetPageRegions 78 2.064102564 1 3 1.987179487 1 3
ListBlobs 2485 3.388732394 1 175 2.707847082 1 175
export NUM_THREADS=$1; export HADOOP_CLIENT_OPTS="-Djline.terminal=jline.UnsupportedTerminal";
for f in {12,13,15,19,21,26,31,32,34,39,3,40,42,43,45,46,48,52,54,55,56,58,60,63,66,68,71,73,79,82,83,84,85,87,89,90,91,96,98};
do
# wait till this query is submitted
while true;
do
# wait for a lot to become available
while [[ $(jobs -rp | wc -l) -ge $NUM_THREADS ]] ;
do
sleep 1;
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"newStorageAccountName": {
"type": "string",
"metadata": {
"description": "The name of the new storage account created to store the VMs disks"
}
},
#!/bin/bash
set -eux
# check if we have atleast 4 nodes
nodes=$(curl -L http://headnodehost:8088/ws/v1/cluster/nodes | grep -o '"nodeHostName":"[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*"' | wc -l)
if [[ $nodes -lt 4 ]]; then
echo "you need atleast 4 node hadoop cluster to run presto on HDI. Aborting."
exit 1
fi
#!/bin/bash
set -eux
nodes=$(curl -L http://headnodehost:8088/ws/v1/cluster/nodes | grep -o '"nodeHostName":"[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*"' | wc -l)
if [[ $nodes -lt 4 ]]; then
echo "you need atlead 4 node hadoop cluster to run presto on HDI. Aborting."
exit 1
fi
VERSION=0.174
if [[ $(hostname -s) = hn0-* ]]; then
nodes=$(curl -L http://headnodehost:8088/ws/v1/cluster/nodes | grep -o '"nodeHostName":"[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*"' | wc -l)
if [[ $nodes -lt 4 ]]; then
echo "you need atlead 4 node hadoop cluster to run presto on HDI. Aborting"
fi
fi
#!/bin/bash
set -eux
VERSION=0.174
mkdir -p /var/lib/presto
chmod -R 777 /var/lib/presto/
if [[ $(hostname -s) = hn0-* ]]; then
apt-get update
rm -rf /var/lib/presto