Submission to Milhouse via commandline tool.

mhouse_runner.py

"""
Usage:

python mhouse_runner.py submit my_awesome.csv --debug -m "http://milhouse:9999" internal_test.csv -a '.m2__m_loading_stats' '.m2__gAccuracy' '.m2__gAccuracyByReadLengthBoxplot' -n 'mk_test_01' --description "services-submitted-job"

This is included in the milhouse code (milhouse/src/milhouse/scripts/mhouse_runner.py
"""
import os
from pprint import pformat
import sys
import argparse
import logging
import json
import functools
import traceback
import urlparse

import requests
from requests.exceptions import ConnectionError

log = logging.getLogger()

_VERSION = "1.1"
_DEFAULT_MILHOUSE_URL = 'http://milhouse:8000'
_SUBMIT_URL = '/project/submit_this_project/'
_VALIDATE_URL = '/project/validate_csv/'
_DEFAULT_USER = "MilhouseUser"


def __validate_resource(func, p):
    if func(p):
        return os.path.abspath(p)
    else:
        raise IOError("Unable to find {f}".format(f=p))

_validate_file = functools.partial(__validate_resource, os.path.isfile)


def setup_log(alog, level=logging.INFO, file_name=None, log_filter=None,
              str_formatter='[%(levelname)s] %(asctime)-15s [%(name)s %(funcName)s %(lineno)d] %(message)s'):
    """Core Util to setup log handler

    :param alog: a log instance
    :param level: (int) Level of logging debug
    :param file_name: (str, None) if None, stdout is used, str write to file
    :param log_filter: (LogFilter, None)
    :param str_formatter: (str) log formatting str
    """
    alog.setLevel(logging.DEBUG)
    if file_name is None:
        handler = logging.StreamHandler(sys.stdout)
    else:
        handler = logging.FileHandler(file_name)
    formatter = logging.Formatter(str_formatter)
    handler.setFormatter(formatter)
    handler.setLevel(level)
    if log_filter:
        handler.addFilter(log_filter)
    alog.addHandler(handler)


def _add_opt_debug(p):
    p.add_argument('--debug',
                   action='store_true',
                   help="Print debug log to stdout")
    return p


def _add_opt_url(p):
    p.add_argument('-m',
                   '--milhouse_url',
                   type=str,
                   default=_DEFAULT_MILHOUSE_URL,
                   help="Milhouse URL")
    return p


def _add_opt_csv(p):
    p.add_argument('csv_path', help="Path to CSV file", type=_validate_file)
    return p


def _add_opt_analysis_ids(p):
    p.add_argument('-a', '--analysis_ids',
                   default=['.m2__gAccuracy', '.m2__m_loading_stats'],
                   type=str,
                   nargs="+",
                   help="Milhouse analysis procedure ids (e.g., '.m2__gAccuracy'). Can be supplied multiple times.")
    return p


def _add_opt_name(p):
    p.add_argument('-n', '--name', required=True, type=str,
                   help="Milhouse Project name")
    return p


def _add_opt_description(p):
    p.add_argument('--description', type=str, default=None,
                   help="Project description")
    return p


def _add_opt_user(p):
    p.add_argument('--user',
                   type=str,
                   default=_DEFAULT_USER,
                   help="user name submit job as")
    return p


def _submit_request(url, payload):
    """
    returns a value of the response
    """
    #log.debug("Submitting request to {u} with {p}".format(u=url, p=payload))
    headers = {'content-type': 'application/json'}

    try:
        r = requests.post(url, data=payload, headers=headers)
        log.debug("status code {s}".format(s=r.status_code))
    except ConnectionError:
        msg = "Unable to connect to {u}".format(u=url)
        sys.stderr.write(msg + "\n")
        raise ValueError(msg)

    try:
        d = json.loads(r.text)
        log.debug(pformat(d))
        msg = d['description']
    except Exception as e:
        msg = "Failed request to ({r}) with status code {s}. ".format(r=url, s=r.status_code)
        msg += str(e)
        log.error(msg, exc_info=True)
        log.error(r.text)

    return msg


def _submit_csv(url, csv_path, analysis_list, project_name, description=None, user=_DEFAULT_USER, tags=()):
    """


    :rtype: Bool
    """
    if description is None:
        description = " description for {f}".format(f=project_name)

    # read in the file and pass as a string
    with open(csv_path, 'r') as f:
        s = f.read()

    # is tags a list?
    if not tags:
        tags = "services-submitted"
    d = dict(user=user,
             inCSV=s,
             projectDescription=description,
             projectName=project_name,
             projectTags=tags)

    if analysis_list:
        d['analysisList'] = ','.join(analysis_list)

    log.debug("Analysis procedures {e}".format(e=analysis_list))
    payload = json.dumps(d)

    # d if status
    msg = _submit_request(url, payload)
    print msg
    return True


def _validate_csv(url, csv_path):

    with open(csv_path, 'r') as f:
        s = f.read()

    d = dict(inCSV=s)
    payload = json.dumps(d)
    msg = _submit_request(url, payload=payload)

    print msg

    return msg


def _to_submit_url(base_url):
    return urlparse.urljoin(base_url, _SUBMIT_URL)


def _to_validate_url(base_url):
    return urlparse.urljoin(base_url, _VALIDATE_URL)


def __args_validate_csv(args):
    #print "running validate {a}".format(a=args)
    url = _to_validate_url(args.milhouse_url)
    msg = _validate_csv(url, args.csv_path)
    return 0


def __args_submit_csv(args):
    #print "running submit csv {a}".format(a=args)
    user = _DEFAULT_USER if args.user is None else args.user
    url = _to_submit_url(args.milhouse_url)
    return _submit_csv(url, args.csv_path, args.analysis_ids, args.name, description=args.description, user=user)


def get_parser():
    desc = "Tool to submit jobs to milhouse analysis server."

    p = argparse.ArgumentParser(version="1.0",
                                description=desc,
                                formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    sp = p.add_subparsers(help='Commands to access Milhouse.')

    vp = sp.add_parser('validate')

    _add_opt_debug(vp)
    _add_opt_url(vp)
    _add_opt_csv(vp)
    vp.set_defaults(func=__args_validate_csv)

    submit_parser = sp.add_parser('submit')

    _add_opt_debug(submit_parser)
    _add_opt_url(submit_parser)
    _add_opt_csv(submit_parser)
    _add_opt_name(submit_parser)
    _add_opt_analysis_ids(submit_parser)
    _add_opt_description(submit_parser)
    _add_opt_user(submit_parser)
    submit_parser.set_defaults(func=__args_submit_csv)

    return p


def main():
    """

    u = 'http://0.0.0.0:8000/project/submit_this_project/'
    """

    p = get_parser()
    args = p.parse_args()

    if args.debug:
        setup_log(log, level=logging.DEBUG)
    else:
        log.addHandler(logging.NullHandler())

    log.info("Starting {f} v{v}".format(f=os.path.basename(__file__), v=_VERSION))
    try:
        state = args.func(args)
        rcode = 0 if state else 1
    except Exception:
        rcode = 1
        traceback.print_exc()
        log.info("Failed {f}".format(f=os.path.basename(__file__)), exc_info=True)

    log.info("Exiting {f} with exit code {e}".format(f=os.path.basename(__file__), e=rcode))
    return rcode


if __name__ == '__main__':
    sys.exit(main())