lb_response_code_logging.py

#!/usr/bin/python
import json
import re
import datetime
import logging
import pprint
import requests
from time import sleep
from googleapiclient import discovery
from googleapiclient.errors import HttpError
from oauth2client.client import GoogleCredentials

METADATA_URL = 'http://metadata.google.internal/computeMetadata/v1/'
METADATA_HEADERS = {'Metadata-Flavor': 'Google'}
TEST_ENDPOINT = 'http://130.211.10.37/'
LABEL = 'response_code'
ACCUMULATION_WINDOW = 5 # minutes
# change to DEBUG in case of troubles 
LOG_LEVEL = logging.INFO  

logger = logging.getLogger("log2monitor")
logger.setLevel(LOG_LEVEL)
ch = logging.StreamHandler()
ch.setLevel(LOG_LEVEL)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
ch.setFormatter(formatter)
logger.addHandler(ch)

def create_service():
  # Get the application default credentials. When running locally, these are
  # available after running `gcloud auth`. When running on compute
  # engine, these are available from the environment.
  credentials = GoogleCredentials.get_application_default()

  # Construct the service object for interacting with the Google Cloud API
  return discovery.build('monitoring', 'v3', credentials=credentials)


def who_am_i():
  url = METADATA_URL + 'project/project-id'

  while True:
    r = requests.get(url, headers=METADATA_HEADERS)

    # During maintenance the service can return a 503, so these should
    # be retried.
    if r.status_code == 503:
      sleep(1)
      continue
    r.raise_for_status()

    return (r.text)

def createMetricDescriptor(service, project_id):
  name='projects/' + project_id
  body = {
  "name": "",
  "description": "HTTP response counts by respose code.",
  "displayName": "response count by code",
  'type': 'custom.googleapis.com/global_lb/' + LABEL + '_count',
  "metricKind": "CUMULATIVE",
  "valueType": "INT64",
  "labels": [
    {
      "key": LABEL,
      "valueType": "INT64",
      "description": "The HTTP response code of the service."
    },
  ],
  }
  write_request = service.projects().metricDescriptors().create(
      name='projects/' + project_id,
      body=body
  )
  # TODO: retry on transient error and graceful handling of API issues
  write_response = write_request.execute()

def log2monitor(service, project_id):
  r = requests.get(TEST_ENDPOINT)
  now = datetime.datetime.now()
  # aggrewindowEnd X minutes blocks, so the start time is rounding down to the last X minutes
  windowStart = now - datetime.timedelta(minutes=now.minute % ACCUMULATION_WINDOW)
  windowStart = windowStart.replace(second=0, microsecond=0)
  windowEnd = windowStart + datetime.timedelta(minutes=ACCUMULATION_WINDOW)
  windowStart = windowStart.strftime('%Y-%m-%dT%H:%M:%SZ')
  windowEnd = windowEnd.strftime('%Y-%m-%dT%H:%M:%SZ')
  # start and end time can not be the same so in an edge case of modolus zero and zero seconds
  # extend the window between now and the next second
  now = now.strftime('%Y-%m-%dT%H:%M:%SZ')
  metric = {
      'type': 'custom.googleapis.com/global_lb/' + LABEL + '_count',
      'labels': {
           LABEL: str(r.status_code)
      }
  }
  resource = {
      'type': 'global'
  }
  points = [{
      'interval': {
          'startTime': windowStart,
          'endTime': windowEnd
      },
      'value': {
          'int64Value': 60 * ACCUMULATION_WINDOW
      }
  }]
  logger.debug('Writing %d at %s' % (r.status_code, now))

  # Write a new data point.
  write_request = service.projects().timeSeries().create(
      name='projects/' + project_id,
      body={'timeSeries': [
          {
              'metric': metric,
              'resource': resource,
              'points': points
          }
      ]})
  try:
    # the request is not retried as it is running peridocally
    write_response = write_request.execute()
  except HttpError, err:
    if err.resp.status == 400:
       error = json.loads(err.content)['error']
       filter = r'^.*[\"\']' + LABEL + '[\"\']: Unrecognized metric label\.$'
       unknown_label = re.match(filter , error['message'])
       if unknown_label and error['status'] == 'INVALID_ARGUMENT':
         # this is the first time we push this metric so we need to describe it
         createMetricDescriptor(service, project_id)
         # second try afer decriptor creation
         write_response = write_request.execute()
       else:
         raise
    else:
      raise

  logger.debug('Response: ' + pprint.pformat(write_response))


def main():
  service = create_service()
  project_id = who_am_i()
  log2monitor(service, project_id)

if __name__ == '__main__':
  main()