Project management

auto.py

#!/usr/bin/env python
# Copyright: see README and LICENSE under the project root directory.
# Author: @Leedehai
#
# File: auto.py
# ---------------------------
# A master management script, like 'fx' of Google's Fuchsia.
# Feeling overwhelmed? Follow README.md's instructions on how
# to build and test the project. Make use of option '--help'.
#
# Compatible with Python2.7+/Python3.5+, on Linux/macOS.
#
# NOTE When dropping this script to other projects, copy
#      //utils/auto/ as well.
# NOTE Adding a new handler FOO:
#      0) string FOO should be no londer than 10 characters (for
#         prettiness, preferrably exactly 5).
#      1) in //utils/auto/handle_FOO.py, implement the handler:
#         - make use of //utils/auto/common_{constants|utils}.py
#         - parse command arguments in run(), FOO's API whose:
#           - parameter is a list of string (see 6 below)
#           - return is what exit code auto.py should carry
#         - run() prints FOO's help message if '-h' or '--help'
#           is given in its parameter.
#      2) in //utils/auto/handlers.json, add FOO, its one-letter
#         abbreviation, and its short description (no more than
#         25 characters), sorted according to the order in which
#         you want them to appear in 'auto.py --help'.
#         * OptionDispatcher will dynamically pick up handlers.
#      3) if FOO writes logs, in //utils/auto/handle_admin.py
#         add the log paths to the function that takes care of
#         'auto.py admin --find-logs' which lists log paths
#
# For usage:
#   ./auto.py --help
#   ./auto.py <subcommand> --help
#     - same as above: ./auto.py help <subcommand>
# For all help messages: ./auto.py --help-all
#
# Examples:
#   ./auto.py clean --all
#   ./auto.py stage
#   ./auto.py build all
#   # not all *test exist, depending on the project
#   ./auto.py utest --run (unit tests)
#   ./auto.py ftest --run (functional tests)
#   ./auto.py ptest --run (performance tests)
#   ./auto.py ztest --run (fuzz tests)
#   # convenient to find logs for build, utest, ..
#   ./auto.py admin --find-logs (list log paths)

import os, sys
import argparse
import inspect
import json
import subprocess
from collections import OrderedDict
from importlib import import_module as dynamic_import
sys.dont_write_bytecode = True

# Each handler is in its own file, because all of them combined
# is 1500+ lines and thus hard to maintain in one giant file.
DIR_AUTO_IMPL = os.path.join(os.path.dirname(__file__), "utils", "auto")
if not os.path.isdir(DIR_AUTO_IMPL):
    sys.exit("[Error] implementation not found: %s" % DIR_AUTO_IMPL)
sys.path.insert(0, DIR_AUTO_IMPL)
import easter_egg
import passthroughs

HANDLERS_MANIFEST = os.path.join(DIR_AUTO_IMPL, "handlers.json")
if not os.path.isfile(HANDLERS_MANIFEST):
    sys.exit("[Error] manifest not found: %s" % HANDLERS_MANIFEST)
with open(HANDLERS_MANIFEST, 'r') as f:
    handler_data_list = json.load(f)
registered_handlers = OrderedDict() # key: str - name, value: struct
for e in handler_data_list:
    registered_handlers[e["name"]] = {
        "abbr": e["abbr"],
        "desc": e["desc"],
        "impl": dynamic_import("handle_%s" % e["name"])
    }

def format_short_help(handlers):
    max_item_each_line = 5
    lines, line_idx, count = {}, 0, 0
    for e in [ name for name in handlers if not name.endswith("test") ]:
        lines.setdefault(line_idx, []).append(e)
        count += 1
        if count == max_item_each_line:
            line_idx += 1
            count = 0
    line_idx += 1
    count = 0
    for e in [ name for name in handlers if name.endswith("test") ]:
        lines.setdefault(line_idx, []).append(e)
        count += 1
        if count == max_item_each_line:
            line_idx += 1
            count = 0
    return '\n'.join([ ("  %s" % (4 * ' ').join(line)) for line in lines.values() ]) + '\n'

def format_long_help(handlers):
    lines = []
    for name in handlers:
        h = handlers[name]
        lines.append("%s, %-11s %s" % (h["abbr"], name, h["desc"]))
    return '\n'.join(["    %s" % line for line in lines ]) + '\n'

DESCRIPTION = "Master management script"

USAGE = "auto.py [-h] [-H] <subcommand> [args]"

SHORT_HELP = """
Available subcommands are:
%s
For help: 'auto.py -h', 'auto.py <subcommand> -h'""" % format_short_help(registered_handlers)

LONG_HELP = """
Available subcommands:

tasks:
%s
passthroughs:
    gn             passthrough to gn
    ninja          passthrough to ninja

A few explanations:

* '//' is a shorthand which denotes the project root.
* a 'build flavor' is the label of a build, so as to
  distinguish among builds with different build args.
* each build's artifacts live in a directory in-tree
  with path in format //out/[build_flavor].[os_name].
* staging a build: set up the build's directory tree
  and generate build files from *.gn files with args.
* to re-stage a build, simply re-run 'auto.py stage'
  with suitable build args that serve your intention.
* each build only needs to be staged once: the build
  files will be re-generated before execution if any
  modification was made to the associated *.gn files.
* during build time, the build files are executed to
  generate artifacts at designated places inside the
  build directory; concurrency is enabled by default.

More help docs: 'auto.py <subcommand> --help'
  fyi. 'auto.py help <subcommand>' is the same.
Print all at once: 'auto.py --help-all'""" % format_long_help(registered_handlers)

def print_all_help(this_script):
    is_atty = sys.stdout.isatty() and sys.stderr.isatty()
    def print_title(s):
        print(("\x1b[33;1m%s\x1b[0m" % s) if is_atty else s)
    def execute(program, args_string):
        return subprocess.call([ program ] + args_string.split())
    print_title("PRINT ALL HELP DOCUMENTATION")
    print("\n%s" % (DESCRIPTION))
    print_title("\n# auto.py --help\n")
    print("%s\n%s" % (USAGE, LONG_HELP))
    # below: need to use subprocess instead of passing [ "--help" ]
    # to handle_*.run() because the process will exit after printing
    # a help message
    for name in registered_handlers:
        print_title("\n# auto.py %s --help\n" % name)
        execute(this_script, "%s --help" % name)
    print_title("\n# auto.py {gn, ninja} --help: long, execute them yourself")
    print("auto.py gn --help")
    print("auto.py ninja --help")

# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# OptionDispatcher
# - dispatch() dispatches according to sys.argv[1]
# - apart from dispatch(), each dynamically added method FOO is
#   a subcommand which calls the corresponding handler's run()
#   and returns run()'s return code
# - dispatch() calls FOO(), and returns with FOO()'s return code
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

class OptionDispatcher:
    @classmethod
    def dispatch(cls):
        parser = argparse.ArgumentParser(description=DESCRIPTION,
                                         usage=USAGE, epilog=LONG_HELP,
                                         formatter_class=argparse.RawTextHelpFormatter)
        parser.add_argument("subcommand", nargs='?',
                            help="subcommand to run")
        parser.add_argument("-H", "--help-all", action="store_true",
                            help="show all help messages and exit")
        # parse_args defaults to sys.argv[1:] for args, but you need to
        # exclude the rest of the args too, or validation will fail
        args = parser.parse_args(sys.argv[1:2])
        if args.help_all:
            return print_all_help(__file__)
        if (args.subcommand == "dispatch"
            or args.subcommand.startswith("_")
            or not hasattr(cls, args.subcommand)):
            sys.stderr.write("[Error] unrecognized subcommand '%s'.\n" % args.subcommand)
            sys.stderr.write(USAGE + "\n" + SHORT_HELP + "\n")
            return 1
        # use dispatch pattern to invoke method with same name
        return getattr(cls, args.subcommand)()

    @staticmethod
    def help():
        if len(sys.argv) < 3:
            sys.stderr.write(USAGE + "\n" + SHORT_HELP + "\n")
            return
        which_subcommand = sys.argv[2]
        if which_subcommand == "help":
            print("You found an easter-egg!")
            return easter_egg.run()
        return subprocess.call([ __file__, which_subcommand, "--help" ])

# Add handlers to OptionDispatcher dynamically
def make_run_caller(impl):
    return lambda : impl.run(sys.argv[2:])
for h_name in registered_handlers:
    h = registered_handlers[h_name]
    run_caller = make_run_caller(h["impl"])
    setattr(OptionDispatcher, h["abbr"], staticmethod(run_caller))
    setattr(OptionDispatcher, h_name,    staticmethod(run_caller))

if __name__ == "__main__":
    try:
        sys.exit(OptionDispatcher.dispatch())
    except KeyboardInterrupt:
        sys.exit(1)