Get PM1 and PP1 tasks from GIMPS

pm1.py

import argparse
from pathlib import Path

FACTOR_PATH = "progress_factor.csv"
MORE_FACTOR_PATH = "more_factors.csv"

def expect_b1(exp, mem):
    # https://www.mersenneforum.org/showthread.php?t=27477
    import math
    base = 2.2**math.log2(20000000/exp)*1000000
    base *= math.sqrt(16/mem)
    digits = math.log10(base)

    # round up
    shift = int(digits) - 1
    return int(math.ceil(base / 10**shift)) * 10**shift

def load_factor_effort(path: Path):
    progress = []
    with path.open() as fin:
        for line in fin.readlines()[1:]:
            if not line.strip():
                continue

            exp, tf, b1, b2, ll, prp = line.split(',')
            progress.append((int(exp), int(tf), int(b1), int(b2), ""))
    progress.sort(key=lambda item: (item[2], item[3]))
    return progress

def load_worktodo(path):
    progress = []
    with path.open() as fin:
        for line in fin.readlines():
            if not line.strip():
                continue

            # e.g. Pminus1=1,2,40009,-1,6500000000,6500000000,67,"5761297,8296597725135896301239024760071"
            line, factors, _ = line.split('"')
            pm1, _, exp, _, b1, b2, tf, _ = line.split(',')
            progress.append((int(exp), int(tf), int(b1) / 1.3, int(b2) / 1.3, factors))
    progress.sort(key=lambda item: (item[2], item[3]))
    return progress
    
def generate_tasks(factor_path, args):
    available = []
    efforts = load_worktodo(factor_path) if args.exist else load_factor_effort(factor_path)

    for exp, tf, b1, b2, factors in efforts:
        b1_target = expect_b1(exp, args.memory)
        if args.exist and b1 == b2:
            # in the pm1 list generated by mersenne.ca, the B2's are generated equal to B1, so assume inf here if B1 = B2.
            # to use the actual b2, find the value and substitute it into the list
            b2 = float('inf')
        if b1_target > b1 * 5 or b1_target * 1000 > b2: # only find the ones with enough B1 gap or B2 diff
            available.append([exp, tf, b1_target, factors])

    def format(item):
        exp, tf, b1, factors = item
        line = "Pminus1=N/A,1,2,{exp},-1,{b1}".format(exp=exp, b1=b1)
        if args.first:
            line += ",{}".format(b1)
        else:
            line += ",0,{}".format(tf)
        if factors:
            line += ',"{}"'.format(factors)
        return line + "\n"

    with open("worktodo.add", "w") as fout:
        arg_text = []
        if args.exist:
            arg_text.append('-e')
        if args.split > 1:
            arg_text.extend(['-s', str(args.split)])
        if args.first:
            arg_text.append('-f')
        arg_text.extend(['-m', str(args.memory)])
        fout.write("; args: %s\n" % ' '.join(arg_text))
        
        if args.split > 1:
            for k in range(args.split):
                fout.write("[Worker #%d]\n" % (k + 1))
                for (i, item) in enumerate(available):
                    if i % args.split != k:
                        continue
                    fout.write(format(item))
        else:
            for item in available:
                fout.write(format(item))
    print("Write {} tasks".format(len(available)))

USAGE = """
Please follow these steps:
1. First get the factor progress report from the following link into %s
   https://www.mersenne.org/report_factoring_effort/?exp_lo={lo}&exp_hi={hi}&txt=1&assigned=-1
2. Then execute this program, which will generate a worktodo.add file with all available P-1 tasks.

Example range: -p 1700000+100000
""" % FACTOR_PATH

def parse_args():
    parser = argparse.ArgumentParser(usage=USAGE)
    parser.add_argument("-e", "--exist", action="store_true", help="Indicate that the data are for factored exponents, and they should be fetched from mersenne.ca into more_factors.csv")
    parser.add_argument("-p", "--print", type=str, help="Print the links and exit. The input formart is `lo+range`, e.g. `10000000+10000`.")
    parser.add_argument("-m", "--memory", type=float, default=16.0, help="Adjust the B1 based on available memory")
    parser.add_argument("-s", "--split", type=int, default=1, help="If bigger than 1, then the work will be split evenly into workers")
    parser.add_argument("-f", "--first", action="store_true", help="If the flag is set, the worktodo will limit B1=B2 for single-threaded stage 1, multi-threaded stage 2 setup.")
    return parser.parse_args()

def main():
    args = parse_args()

    if args.print:
        lo, span = args.print.split('+')
        lo, hi = int(lo), int(lo) + int(span)
        print(USAGE.format(lo=lo, hi=hi))
        exit(1)

    factor_path = Path(MORE_FACTOR_PATH if args.exist else FACTOR_PATH)
    if not factor_path.exists():
        print("Factor progress report not found!")
        print(USAGE)
        exit(-1)

    generate_tasks(factor_path, args)
    
main()

pp1.py

import argparse
from pathlib import Path

FACTOR_PATH = "progress_factor.csv"
PP1_PATH = "progress_pp1.txt"

def load_factor_effort(path: Path):
    progress = []
    with path.open() as fin:
        for line in fin.readlines()[1:]:
            if not line.strip():
                continue

            exp, tf = line.split(',')
            progress.append((int(exp), int(tf)))
    return progress
    
def load_pp1_progress(path: Path):
    progress = []
    with path.open() as fin:
        for line in fin.readlines()[1:]:
            if not line.strip():
                continue

            exp, b1, b2, start, date, time = line.split('\t')
            if start == "2/7":
                n = 1
            elif start == "6/5":
                n = 2
            else:
                n = 3
            progress.append((int(exp), int(b1), int(b2), n))
    return progress
    
def generate_tasks(factor_path, pp1_path, args):
    tf_progress = load_factor_effort(factor_path)
    pp1_progress = load_pp1_progress(pp1_path)
    
    available = dict(tf_progress)
    for exp, _, _, n in pp1_progress:
        if n == args.nth_run and exp in available:
            available.pop(exp)

    available = sorted(list(available.items()))
    with open("worktodo.add", "w") as fout:
        for exp, tf in available:
            fout.write("Pplus1=N/A,1,2,{exp},-1,{b1},{b2},{n},{tf}\n".format(
                exp=exp, b1=args.b1, b2=args.b2, n=args.nth_run, tf=tf
            ))
    print("Write {} tasks".format(len(available)))

USAGE = """
Please follow these steps:
1. First get the factor progress report from the following link into %s
   https://www.mersenne.org/report_factoring_effort/?exp_lo={lo}&exp_hi={hi}&txt=1&tfonly=1&assigned=-1&pm1=1
2. Then get the p+1 progress report from the following link into %s
   https://www.mersenne.org/report_pplus1/?exp_lo={lo}&exp_hi={hi}&txt=1
4. Lastly execute this program, which will generate a worktodo.add file with all available P+1 tasks.

Example range: lo=10000000, hi=10010000
""" % (FACTOR_PATH, PP1_PATH)

def parse_args():
    parser = argparse.ArgumentParser(usage=USAGE)
    parser.add_argument("-p", "--print", type=str, help="Print the links and exit. The input formart is `lo+range`, e.g. `10000000+10000`.")
    parser.add_argument("-n", "--nth-run", type=int, default=1,
                        help="The argument nth_run. 1 starts from 2/7, 2 starts from 6/5, 3 starts randomly")
    parser.add_argument("--b1", type=int, default=  2000000)
    parser.add_argument("--b2", type=int, default=150000000)

    return parser.parse_args()

def main():
    args = parse_args()

    if args.print:
        lo, span = args.print.split('+')
        lo, hi = int(lo), int(lo) + int(span)
        print(USAGE.format(lo=lo, hi=hi))
        exit(1)

    factor_path = Path(FACTOR_PATH)
    if not factor_path.exists():
        print("Factor progress report not found!")
        print(USAGE)
        exit(-1)

    pp1_path = Path(PP1_PATH)
    if not pp1_path.exists():
        print("P+1 progress report not found!")
        print(USAGE)
        exit(-1)

    generate_tasks(factor_path, pp1_path, args)
    
main()