best_anything.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Feb 19 13:34:45 2018

@author: tomin
"""

import os
import re
import time
import sys

from concurrent.futures import ProcessPoolExecutor
from subprocess import Popen, PIPE

regex = None


def run_subprocess(cmd):
    sp = Popen(cmd, stdout=PIPE, stderr=PIPE,
               shell=True, universal_newlines=True)
    comm = sp.communicate()
    exit_code = sp.returncode
    if exit_code != 0:
        print("There was an error running the subprocess.\n"
              "cmd: %s\n"
              "exit code: %d\n"
              "stdout: %s\n"
              "stderr: %s" % (cmd, exit_code, comm[0], comm[1]))
    return comm

def func(a):
    global regex
    with open(a,'r', errors='ignore') as data:
        content = data.read()
        res = re.findall(regex, content)
#        print(res)
        return len(res)

def get_tags(tag_name):
    cmd = "git tag -l %s" % tag_name
    comm = run_subprocess(cmd)
    return comm[0].strip("\n").split("\n")


def checkout_tag(tag_name):
    cmd = "git checkout %s" % tag_name
    comm = run_subprocess(cmd)

def files_list(path):
    fls = []
    for path, _, files in os.walk(path):
        for name in files:
            fls.append(os.path.join(path, name))
    return fls

def get_total_matches(files):
    ppe = ProcessPoolExecutor(4)

    result = list(ppe.map(func,files ))
#    print(sum(result))
    return sum(result)


if __name__ == '__main__':
    import argparse  # Only needed for main()
    parser = argparse.ArgumentParser()
    parser.add_argument("-j", action="store", dest="jobs", default=1, type=int,
                        metavar="N", help="number of jobs to run at once")
    parser.add_argument("tag_name", metavar="<Tag Name>",
                       help="tag name to search for (can contain wildcards)")
    parser.add_argument("strings_list", metavar="<Strings List>",
                       help="path to file with strings from blobs, produced by strings command")
    args = parser.parse_args()

#    tags = get_tags('LA\.BR\.1\.3\.6\.c2-*8976*')
    tags = get_tags(args.tag_name)
    if not tags:
        print("No tags to check. bailing.")
        sys.exit(1)

    print("number of tags to process:", len(tags) )
    start_time = time.time()
#    strings_file = '/home/tomin/devel/nx569j-cam-strings.txt'
    strings_file = args.strings_list
    print(strings_file)
    if not strings_file:
        print("No strings file. bailing.")
        sys.exit(1)
    source_dir_path = '/home/tomin/devel/camera-caf/QCamera2'
    all_strings = open(strings_file)
    strings_list = list()
    for line in all_strings:
        if (len(line) > 15):
            if (not line.startswith('_Z')):
                t = line.split(" ")
                if len(t) > 1:
                    strings_list.append(line.rstrip())

    regex = re.compile("(?=(" + "|".join(map(re.escape, strings_list)) + "))")


    best_matches_count = 0
    best_tag = ''
    for cur_tag in tags:
        print("processing tag", cur_tag )
        checkout_tag(cur_tag)
        files = files_list(source_dir_path)
        matches_count = get_total_matches(files)
        if (best_matches_count < matches_count):
            best_matches_count = matches_count
            best_tag = cur_tag
        print("tag", cur_tag)
        print("test %(matches)d/%(total)d" %{ "matches": matches_count, "total":len(strings_list)})


    print("Done")
    elapsed_time = time.time()-start_time
    print("elapsed time", elapsed_time)
    print("best_matches count", best_matches_count)
    print("best tag ", best_tag)