Example of a HTTP probe with OpenCensus

README.md

Example of a HTTP probe with OpenCensus

Background

This code demonstrates development of a HTTP probe for monitoring of a web application. It take advantage of OpenCensus instrumentation with the Go HTTP client and other features in OpenCensus, including integration with Stackdriver.

Basic Setup

This example can be run either locally, on Google Kubernetes Engine, or be adapted to run elsewhere. Before running it enable the Stackdriver Monitoring and Trace APIs in order to permit collection of metrics. Also, create a service account. Download service account credentials json file and export the shell environment variable GOOGLE_APPLICATION_CREDENTIALS:

export GOOGLE_APPLICATION_CREDENTIALS=credentials.json

Edit the file config.yaml and set the target URL to the target that you intend to monitor.

Run locally

To run locally, first install Go and OpenCensus.

export ZONE=my-zone
go run http_probe.go

Run locally with Docker

Install Docker. Run the commands

docker build -t http_probe_container .
docker run -it --rm \
  -e "GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS" \
  -e "ZONE=$ZONE" \
  --name http_probe \
  http_probe_container 

Run with Kubernetes

Upload to the Google Container Repository

gcloud auth configure-docker
TAG=v1
PROJECT_ID=[your project id]
docker tag http_probe_container gcr.io/$PROJECT_ID/httpgoprobe:$TAG
docker push gcr.io/$PROJECT_ID/httpgoprobe:$TAG

Start up a GKE cluster in each zone:

export ZONE=us-central1-a
gcloud components install kubectl
gcloud config set project $PROJECT_ID
gcloud config set compute/zone $ZONE
CLUSTER_NAME=probes-$ZONE
gcloud container clusters create $CLUSTER_NAME \
  --zone $ZONE \
  --num-nodes 1 \
  --enable-cloud-logging
gcloud container clusters get-credentials $CLUSTER_NAME --zone $ZONE

Deploy a probe for each zone

HTTP_PROBE_WORKLOAD=httpprobe-$ZONE
kubectl run $HTTP_PROBE_WORKLOAD --image gcr.io/$PROJECT_ID/httpgoprobe:$TAG \
  --env "GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS" \
  --env "ZONE=$ZONE" \
  --limits="cpu=50m,memory=128Mi"

Related links

https://opencensus.io/gogrpc/index.html https://github.com/census-instrumentation/opencensus-go https://github.com/census-instrumentation/opencensus-go/tree/master/examples/http https://godoc.org/go.opencensus.io/plugin/ochttp

License

Apache 2.0 Copyright 2018 Google. All rights reserved.

config.yaml

targeturl: # URLs for the probes to send HTTP requests to
 - https://www.google.com
 - https://www.wikipedia.org

Dockerfile

FROM golang:1.10

WORKDIR /go/src/http_probe
COPY *.json .
COPY *.yaml .
COPY *.go .

RUN go get -d -v ./...
RUN go install -v ./...

CMD ["http_probe"]

http_probe.go

// Copyright 2018, Google
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
  "context"
  "io/ioutil"
  "log"
  "net/http"
  "os"
  "time"

  "go.opencensus.io/exporter/stackdriver"
  "go.opencensus.io/plugin/ochttp"
  "go.opencensus.io/stats/view"
  "go.opencensus.io/tag"

  "gopkg.in/yaml.v2"
)

var (
  zoneKey tag.Key
)

type Config struct {
    Targeturl []string
}

func main() {
  log.Printf("Starting HTTP probe")
  config := readConfig()
  ocSetup()
  probe(config)

}

// Initialization for OpenCensus
func ocSetup() {

  log.Printf("Configure HTTP probe")

  // Create and register exporters for monitoring metrics and trace
  prefix := "http-probe"
  se, err := stackdriver.NewExporter(stackdriver.Options{
    MetricPrefix: prefix,
  })
  if err != nil {
    log.Fatalf("Error creating Stackdriver exporter: %v", err)
  }
  view.RegisterExporter(se)
  view.SetReportingPeriod(10 * time.Second)

  // Custom tag
  zoneKey, err = tag.NewKey("zone")
  if err != nil {
    log.Fatal(err)
  }

  // Montoring stats for HTTP requests
  if err := view.Register(
    ochttp.ClientResponseCountByStatusCode,
    &view.View{
        Name:        "httpclient_latency_by_hostpath",
        TagKeys:     []tag.Key{ochttp.Host, ochttp.Path, zoneKey},
        Measure:     ochttp.ClientLatency,
        Aggregation: ochttp.DefaultLatencyDistribution,
    },
  ); err != nil {
    log.Fatal(err)
  }

}

// Run the HTTP probe
func probe(config Config) {
  log.Printf("Running HTTP probe")
  ctx := context.Background()
  zone := os.Getenv("ZONE")
  ctx, err := tag.New(ctx,
    tag.Insert(zoneKey, zone),
  )
  if err != nil {
    log.Fatal(err)
  }

  urls := config.Targeturl
  client := &http.Client{Transport: &ochttp.Transport{}}
  for true {
    for _, url := range urls {
      log.Printf("Sending request to: %s", url)
      req, _ := http.NewRequest("GET", url, nil)
      req = req.WithContext(ctx)
      resp, err := client.Do(req)
      if err != nil {
        log.Printf("No HTTP response: %v", err)
      } else {
        resp.Body.Close()
      }
    }
    time.Sleep(10 * time.Second)
  }
}

// Initialization for OpenCensus
func readConfig() Config {
  log.Printf("Reading HTTP probe config")
  var config Config
  contents, err := ioutil.ReadFile("config.yaml")
  if err != nil {
    log.Printf("Failed to open config file")
    panic(err)
  }
  err = yaml.Unmarshal(contents, &config)
  if err != nil {
    log.Printf("Failed to unmarshal config file")
    panic(err)
  }
  return config
}