Skip to content

Instantly share code, notes, and snippets.

@alexamies
Last active July 11, 2018 16:22
Show Gist options
  • Save alexamies/0d7b393739114ceadfea85257b72fbcd to your computer and use it in GitHub Desktop.
Save alexamies/0d7b393739114ceadfea85257b72fbcd to your computer and use it in GitHub Desktop.
Monitoring of HTTP network services

Client Errors Monitoring project

This is a simple probe for measuring sources of HTTP connection latency, including DNS lookup, TLS handshake, connection establishment, and data transfer.

Enable the Stackdriver and Kubernetes Engine APIs in the Cloud Console. You will need to enable Stackdriver premium for use of custom metrics.

Export the name of your project and zone to the shell environment

export PROJECT_ID=[YOUR PROJECT ID]
gcloud config set project $PROJECT_ID
ZONE=us-west1-a
gcloud config set compute/zone $ZONE

Create a service account in the console and download the key json file into the directory that you run these commands from. Assign the service account role Project > Owner when you create it.

export GOOGLE_APPLICATION_CREDENTIALS=???.json

Export a shell variable set to the target URL you want your probe to send requests to:

export TARGET_URL=http://[YOUR URL]

Define the Stackdriver custom metric:

docker build -f Dockerfile-setup -t setup_sd .
docker run -it --env PROJECT_ID=$PROJECT_ID \
  --env GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS \
  setup_sd

Build and deploy the probe in a Docker container:

docker build -f Dockerfile-httpprobe -t httpprobe .
docker run -it  --env PROJECT_ID=$PROJECT_ID \
  --env GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS \
  --env TARGET_URL=$TARGET_URL \
  httpprobe

Uploading to the Google Container Rep

gcloud auth configure-docker
TAG=v1
docker tag httpprobe gcr.io/$PROJECT_ID/httpprobe:$TAG
docker push gcr.io/$PROJECT_ID/httpprobe:$TAG

Start up a GKE cluster:

gcloud components install kubectl
gcloud config set project $PROJECT_ID
#ZONE=europe-west4-a
ZONE=us-west1-a
gcloud config set compute/zone $ZONE
CLUSTER_NAME=httpprobe-$ZONE
gcloud container clusters create $CLUSTER_NAME --zone $ZONE --num-nodes 1
gcloud container clusters get-credentials $CLUSTER_NAME

Deploy the probe to the cluster

kubectl run httpprobe --image gcr.io/$PROJECT_ID/httpprobe:$TAG \
  --env="PROJECT_ID=$PROJECT_ID" \
  --env "GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS" \
  --env TARGET_URL=$TARGET_URL 

License

Apache 2.0. Copyright 2018 Google. All rights reserved.

FROM google/cloud-sdk:latest
ADD httpprobe.py /
ADD *.json /
RUN pip install --upgrade google-cloud-monitoring
CMD [ "python", "./httpprobe.py", "$PROJECT_ID", "$GOOGLE_APPLICATION_CREDENTIALS", "$TARGET_URL"]
FROM google/cloud-sdk:latest
ADD setup_sd.py /
ADD *.json /
RUN pip install --upgrade google-cloud-monitoring
CMD [ "python", "./setup_sd.py", "$PROJECT_ID", "$GOOGLE_APPLICATION_CREDENTIALS"]
# Program to measure and report times for DNS lookup, connection establishment,
# TLS handshake, ready to start transfer, and completion of response.
# See https://github.com/reorx/httpstat for a more robust curl wrapper
# See https://cloud.google.com/monitoring/docs/reference/libraries#client-libraries-usage-python
# for monitoring background
# Beforing running, install the Stackdriver monitoring libraries with
# pip install --upgrade google-cloud-monitoring==0.28.1
import json
import os
import subprocess
import time
from google.cloud import monitoring_v3
def httpstat(url):
write_format = """{\n\"time_namelookup\": %{time_namelookup},
\"response_code\": %{response_code},
\"time_appconnect\": %{time_appconnect},
\"time_connect\": %{time_connect},
\"time_starttransfer\": %{time_starttransfer},
\"time_total\": %{time_total}
}"""
cmd = ['curl', '-w', write_format, '-o' '/dev/null', url, '-s', '-S']
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
if p.returncode != 0:
if err:
print('Error calling command: {}'.format(err))
else:
print('Error calling command')
return {}
try:
result = json.loads(out)
return result
except ValueError as e:
print('Error parsing json: {}'.format(e))
print(out)
return {}
def poll():
client = monitoring_v3.MetricServiceClient()
project_id = os.getenv('PROJECT_ID')
url = os.getenv('TARGET_URL')
project_name = client.project_path(project_id)
while 0 < 1:
data = httpstat(url)
if len(data) > 0:
write_metrics(data, client, project_name)
else:
print('No data to report')
time.sleep(60)
def write_metrics(data, client, project_name):
now = time.time()
write_metric_double('custom.googleapis.com/namelookup',
data['time_namelookup'], now, client, project_name)
write_metric_double('custom.googleapis.com/time_connect',
data['time_connect'], now, client, project_name)
write_metric_double('custom.googleapis.com/tls_handshake',
data['time_appconnect'], now, client, project_name)
write_metric_double('custom.googleapis.com/starttransfer',
data['time_starttransfer'], now, client, project_name)
write_metric_double('custom.googleapis.com/time_total',
data['time_total'], now, client, project_name)
write_metric_int('custom.googleapis.com/responsecode',
data['response_code'], now, client, project_name)
msg = 'Successfully wrote time series: {0}, {1}, {2}, {3}, {4}, {5}'
print(msg.format(data['time_namelookup'], data['time_connect'],
data['time_appconnect'], data['time_starttransfer'],
data['time_total'], data['response_code']))
def write_metric_double(metric_type, value, now, client, project_name):
series = monitoring_v3.types.TimeSeries()
series.metric.type = metric_type
series.resource.type = 'global'
point = series.points.add()
point.value.double_value = value
point.interval.end_time.seconds = int(now)
point.interval.end_time.nanos = int(
(now - point.interval.end_time.seconds) * 10**9)
client.create_time_series(project_name, [series])
def write_metric_int(metric_type, value, now, client, project_name):
series = monitoring_v3.types.TimeSeries()
series.metric.type = metric_type
series.resource.type = 'global'
point = series.points.add()
point.value.int64_value = value
point.interval.end_time.seconds = int(now)
point.interval.end_time.nanos = int(
(now - point.interval.end_time.seconds) * 10**9)
client.create_time_series(project_name, [series])
def main():
poll()
if __name__ == '__main__':
main()
# Script for defining metric descriptors
import os
from google.cloud import monitoring_v3
print "Setting up custom metrics\n"
project = os.getenv('PROJECT_ID')
client = monitoring_v3.MetricServiceClient()
name = client.project_path(project)
name1 = 'projects/{}/metricDescriptors/custom.googleapis.com%2Fnamelookup'.format(project)
metric_descriptor1 = {
'name': name1,
'type': 'custom.googleapis.com/namelookup',
'description':'DNS lookup time for URL',
'metric_kind': 'GAUGE',
'value_type': 'DOUBLE'}
response = client.create_metric_descriptor(name, metric_descriptor1)
name2 = 'projects/{}/metricDescriptors/custom.googleapis.com%2Ftime_connect'.format(project)
metric_descriptor2 = {
'name': name2,
'type': 'custom.googleapis.com/time_connect',
'description':'Connection time',
'metric_kind': 'GAUGE',
'value_type': 'DOUBLE'}
response = client.create_metric_descriptor(name, metric_descriptor2)
name3 = 'projects/{}/metricDescriptors/custom.googleapis.com%2Ftls_handshake'.format(project)
metric_descriptor3 = {
'name': name3,
'type': 'custom.googleapis.com/tls_handshake',
'description':'Time for TLS handshake to complete',
'metric_kind': 'GAUGE',
'value_type': 'DOUBLE'}
response = client.create_metric_descriptor(name, metric_descriptor3)
name4 = 'projects/{}/metricDescriptors/custom.googleapis.com%2Fstarttransfer'.format(project)
metric_descriptor4 = {
'name': name4,
'type': 'custom.googleapis.com/starttransfer',
'description':'Time for first byte was just about to be transferred',
'metric_kind': 'GAUGE',
'value_type': 'DOUBLE'}
response = client.create_metric_descriptor(name, metric_descriptor4)
name5 = 'projects/{}/metricDescriptors/custom.googleapis.com%2Ftime_total'.format(project)
metric_descriptor5 = {
'name': name5,
'type': 'custom.googleapis.com/time_total',
'description':'Total ime for HTTP request',
'metric_kind': 'GAUGE',
'value_type': 'DOUBLE'}
response = client.create_metric_descriptor(name, metric_descriptor5)
name6 = 'projects/{}/metricDescriptors/custom.googleapis.com%2Fresponsecode'.format(project)
metric_descriptor6 = {
'name': name6,
'type': 'custom.googleapis.com/responsecode',
'description':'HTTP response code',
'metric_kind': 'GAUGE',
'value_type': 'INT64'}
response = client.create_metric_descriptor(name, metric_descriptor6)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment