Tornado-based asynchronous web service listener

jira-maestro.py

#!/usr/bin/env python
# coding=utf-8

# JIRA webhook listener "Maestro"
# uses python Tornado scalable web server to handle requests coming
# coming in via webhooks from Jira instance

import tornado.httpserver
import tornado.ioloop
import tornado.options
import tornado.web
import json
import os
import sys
import threading
from sh import python
from maestro.maestro import init_log, get_dict, check_param, handler

base_dir = '/opt/jira-maestro'
plugin = 'maestro'

os.chdir(base_dir)

# set logging
log_dir = base_dir+'/logs/'

if not os.path.exists(log_dir):
    os.makedirs(log_dir)

log_file = log_dir+'listener.log'
log = init_log(plugin, log_file)

from tornado.options import define, options
define("port", default=5500, help="run on the given port", type=int)

# WEBHOOK PROCESSING
class MaestroHandler(tornado.web.RequestHandler):
    @tornado.web.asynchronous
    def post(self):
        log.info('------- Incoming Request')
        data = json.loads(self.request.body)   
        # check if incoming json is comment change - Jira webhook JQL limitation fix 
        if any([get_dict(data, 'issue_type_name') == 'issue_commented',
                get_dict(data, 'issue_type_name') == 'issue_comment_deleted',
                get_dict(data, 'issue_type_name') == 'issue_comment_edited',
                get_dict(data, 'webhookEvent') == 'comment_created']):
            log.warn('incoming json is updates to jira comments, not actual ticket data.., skipping')
            sys.exit()
        
        # get general Jira fields
        issue_key   = get_dict(data, 'issue.key')
        issue_type  = get_dict(data, 'issue.fields.issuetype.name')
        status      = get_dict(data, 'issue.fields.status.name')

        check_param('issue_key', issue_key)
        check_param('issue_type', issue_type)
        check_param('status', status)

        log.info("issue key: %s" % issue_key)
        log.info("issue type: %s" % issue_type)
        log.info("status: %s" % status)

# ---------------- [  ROUTE REQUEST TO PLUGINS  ] -----------------------------
        @handler
        def route_to_init():
            for root, dirs, files in os.walk(base_dir+'/plugins', topdown=False):
                for name in files:
                    if name == 'init':
                        init = os.path.join(root, name)
                        log.info("processing %s" % init)
                        python(init, json.dumps(data))
# -----------------------------------------------------------------------------

if __name__ == "__main__":
    tornado.options.parse_command_line()
    app = tornado.web.Application(handlers=[(r"/webhook", MaestroHandler)], debug=False)
    http_server = tornado.httpserver.HTTPServer(app)
    http_server.bind(options.port)
    http_server.start(0)
    tornado.ioloop.IOLoop.instance().start()
    

maestro.py

#!/usr/bin/env python
# coding=utf-8

# MAESTRO CLASS
# Various helper functions for Maestro process

import os
import csv
import logging
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
import sys
import yaml
from functools import wraps
from hashlib import md5
from jira import JIRA
from jira.exceptions import JIRAError

base_dir = '/opt/jira-maestro'

# Logging Globals
LOG_FORMAT = "%(asctime)s [%(levelname)s] %(message)s"

#------------------------------------------------------------------------------------------------
# Setup Logging
def init_log(plugin, log_file):
    ''' configures rotating log file for the plugin '''
    from logging.handlers import RotatingFileHandler
    log_dir = log_file.strip(log_file.split('/')[-1])
    global log
    log = logging.getLogger(plugin)
    handler = RotatingFileHandler(log_file, maxBytes=5000000, backupCount=3)
    formatter = logging.Formatter(LOG_FORMAT)
    handler.setFormatter(formatter)
    log.addHandler(handler)
    log.setLevel(logging.DEBUG)
    return log
 
#------------------------------------------------------------------------------------------------
def get_cf(fields_file, fname):
    ''' get custom field ID from fields.csv '''

    f = open(fields_file)
    csvfile = csv.reader(f, delimiter=',')

    for row in csvfile:
        if fname in row:
            return row[1]

#------------------------------------------------------------------------------------------------
def get_dict(data, path=None, default=None):
    ''' get a value from a Dictionary key
          > get_dict(mydict, "key1.key2[0]")
        or with custom value on lookup failure, 
          > get_dict(mydict, "key1.key2", default="No value here")
        or pass a parameter
          > get_dict(mydict, "key1.key2.{}".format(myparam))
    '''
    value = None
    keys = path.split(".")

    # reset path
    path = None

    # build proper path
    for key in keys:
        # check if key is a list
        if key.endswith(']'):
            temp = key.split('[')
            key = ''.join(temp[0])
            index = int(temp[1].strip(']'))
            path = path+"['"+key+"']"+"["+str(index)+"]"
        else:
            if path is None:
                path = "['"+key+"']"
            else:
                path = path + "['"+key+"']"

    lookup = 'data'+path
    try:
        value = eval(lookup)
        if value is None:
            value = default
    except (KeyError, ValueError, IndexError, TypeError):
        value = default
    finally:
        return value

#------------------------------------------------------------------------------------------------
def get_config(*args):
    ''' get config value from plugin's config file '''
    plugin = args[0]

    if plugin == 'maestro':
        conf_file = base_dir+'/config'
    else:
        conf_file = base_dir+'/plugins/'+plugin+'/config'

    # check if config file is present
    if not os.path.exists(conf_file):
        log.error('No config file present: %s' % conf_file)
        sys.exit()

    # read YAML
    try:
        with open(conf_file, 'r') as f:
            conf = yaml.load(f)
            argList = list(args)  # convert tuple to list
            argList.pop(0)  # remove Plugin from list

            # create lookup path
            parsepath = "conf"

            for arg in argList:
                parsepath = parsepath + "['" + arg + "']"
            try:
                return eval(parsepath)
            except:
                log.error('Error reading value for %s' % parsepath)
                sys.exit()
    except:
        log.error('Error reading %s config file' % plugin)
        sys.exit(1)
#------------------------------------------------------------------------------------------------
def check_param(param_name, param_value):
    ''' check parameter for null value 
        usage: check_param('name', name)
    '''
    if not param_value or param_value == '':
        log.error('parameter value missing: %s, exiting..' % param_name)
        sys.exit(1)

#------------------------------------------------------------------------------------------------
# get all Field IDs from the plugin's Jira server
def get_jira_fields(fields_file, plugin):
    ''' get Custom Fields file from Jira '''
    # some housekeeping.. get all fields from JIRA
    if not os.path.exists(fields_file):

        # get Jira creds from conf file
        jira_server = get_config(plugin, 'jira', 'server')
        jira_user = get_config(plugin, 'jira', 'user')
        jira_pw = get_config(plugin, 'jira', 'pw')

        # get field json from Jira
        url = "https://"+jira_server+"/rest/api/2/field"
        req = requests.get(url, auth=(jira_user, jira_pw), verify=False)
        if req.status_code != 200:
            log.error('Error connecting to Jira.. check "%s" config file' % plugin)
            sys.exit()
        jsonfile = req.json()

        # create multi-dim dictionary of key/val pairs
        with open(fields_file, 'w') as csvfile:
            writer = csv.writer(csvfile, delimiter='\n')

            log.info('creating Jira custom fields file')
            for item in jsonfile:
                fname = item.get("name").encode('utf-8')
                fid = item.get("id").encode('utf-8')
                pair = fname+','+fid
                writer.writerow([pair])
        csvfile.close()
#------------------------------------------------------------------------------------------------
def curl(method, url, user=None, pw=None, data=None, header=None):
    ''' make a Curl request to URL, returns a JSON '''

    if not header: header = {'Content-Type':'application/json'}

    # check method
    if method == 'get':
        if user:
            req = requests.get(url, auth=(user, pw), verify=False)
        else:
            req = requests.get(url, verify=False)
    if method == 'post':
        req = requests.post(url, auth=(user, pw), data=data, headers=header, verify=False)
    req.raise_for_status() # raise error if HTTP return code is not 200, 201 or 202
    return req.json()

#------------------------------------------------------------------------------------------------
def wget(url, file):
    ''' wget a file from url, save to file '''
    req = requests.get(url)
    req.raise_for_status()
    try:
        with open(file, 'wb') as f:
            f.write(req.content)
    except Exception as e:
        log.error('cannot wget file..')
        log.exception(str(e))


#------------------------------------------------------------------------------------------------
def render_template(template, **kwargs):
    ''' renders a Jinja template into HTML '''
    # check if template exists
    if not os.path.exists(template):
        log.error('No template file present: %s' % template)
        sys.exit()

    import jinja2
    templateLoader = jinja2.FileSystemLoader(searchpath="/")
    templateEnv = jinja2.Environment(loader=templateLoader)
    templ = templateEnv.get_template(template)
    return templ.render(**kwargs)

#------------------------------------------------------------------------------------------------
def send_email(to, sender='COMPANY<[email protected]>', cc=None, bcc=None, subject=None, body=None):
    ''' sends email using a Jinja HTML template '''
    import smtplib
    # Import the email modules
    from email.mime.multipart import MIMEMultipart
    from email.mime.text import MIMEText
    from email.header import Header
    from email.utils import formataddr

    # convert TO into list if string
    if type(to) is not list:
        to = to.split()

    to_list = to + [cc] + [bcc]
    to_list = filter(None, to_list) # remove null emails

    msg = MIMEMultipart('alternative')
    msg['From']    = sender
    msg['Subject'] = subject
    msg['To']      = ','.join(to)
    msg['Cc']      = cc
    msg['Bcc']     = bcc
    msg.attach(MIMEText(body, 'html'))
    server = smtplib.SMTP("127.0.0.1") # or your smtp server
    try:
        server.sendmail(sender, to_list, msg.as_string())
    except Exception as e:
        log.error('Error sending email')
        log.exception(str(e))
    finally:
        server.quit()

#------------------------------------------------------------------------------------------------
# Error Handler decorator
def handler(function):
    ''' error handling wrapper for set of actions '''
    try:
        log.info('running: %s' % function.__name__)
        return function()
    except Exception as err:
        log.error("Problem running function: %s" % function.__name__)
        log.exception(str(err))
        sys.exit(1)
#------------------------------------------------------------------------------------------------
def md5checksum(filePath):
    ''' get md5 checksum of a file '''
    with open(filePath, 'rb') as fh:
        m = md5()
        while True:
            data = fh.read(8192)
            if not data:
                break
            m.update(data)
        return m.hexdigest()
#------------------------------------------------------------------------------------------------
def jira_connect(server, user, pw):
    ''' connects to jira server and returns jira object '''
    try:
        log.info("Connecting to JIRA: %s" % server)
        jira_options = {'server': 'https://'+server, 'verify': False}
        jira = JIRA(options=jira_options, basic_auth=(user, pw))
        return jira
    except Exception, e:
        log.error("Failed to connect to JIRA: %s" % e)
        sys.exit(1)

#------------------------------------------------------------------------------------------------
def verify_email(email_addr):
    ''' checks email address for proper syntax '''
    import re
    email_addr = ' '.join(email_addr.split())
    log.info('verifying email: %s' % email_addr)
    match = re.match('^[_a-z0-9-]+(\.[_a-z0-9-]+)*@[a-z0-9-]+(\.[a-z0-9-]+)*(\.[a-z]{2,4})$', email_addr)
    if match == None:
        log.error("Invalid email address: %s" % email_addr)
        raise ValueError('Bad email syntax')