Example script for submitting and downloading results through CGC pipeline

submit-cgc.py

#! /usr/bin/env python3

if __name__ == '__main__':
    import os
    import sys
    import time
    import argparse as ap
    import sevenbridges as sbg
    from sevenbridges.errors import SbgError

    parser = ap.ArgumentParser(
        prog='submit-cgc-jobs.py',
        conflict_handler='resolve',
        description=('Submit jobs to CGC/Download results for NG project.'))

    parser.add_argument('accessions', metavar="ACCESSIONS_TXT", type=str,
                        help='Text file containing accessions to process.')
    parser.add_argument('outdir', metavar="OUTPUT_DIR", type=str,
                        help='Directory to output files to.')
    parser.add_argument('--limit', metavar="INT", type=int, default=0,
                        help=('Limit the number of jobs to submit. (Default: '
                              'No limit)'))

    if len(sys.argv) == 1:
        parser.print_usage()
        sys.exit(1)

    args = parser.parse_args()

    if not os.path.exists(args.outdir):
        os.makedirs(args.outdir)

    # Submit job to CGC via script
    api = sbg.Api(config=sbg.Config(profile='cgc'))
    PROJECT = 'project/batchspadessra'
    APP = 'project/batchspadessra/process-run'
    GENOME_SIZE = 2153922  # N. gonorrhoeae strain 1090

    total = 0
    with open(args.accessions, 'r') as fh:
        for line in fh:
            run_accession = line.rstrip()

            # See if Run has already been processed
            files = api.files.query(
                project=PROJECT,
                names=['{0}-spades.tar.gz'.format(run_accession),
                       '{0}-illumina-cleanup.json'.format(run_accession)]
            )

            if files.total:
                # There are files to download
                for file in files:
                    local_file = '{0}/{1}'.format(args.outdir, file.name)
                    if os.path.exists(local_file):
                        print("SKIP: {0} exists".format(file.name))
                    else:
                        try:
                            print("DOWNLOAD: Downloading {0} to {1}/".format(
                                file.name, args.outdir
                            ))
                            file.download(local_file)
                        except SbgError:
                            print('\tERROR: Download failed')
            else:
                # Submit jobs to CGC
                inputs = {
                    "RUN_ACCESSION": run_accession,
                    "GENOME_SIZE_1": GENOME_SIZE,
                }

                try:
                    task = api.tasks.create(
                        name=run_accession, project=PROJECT, app=APP,
                        inputs=inputs, run=True, interruptible=True
                    )
                    print("CREATE JOB: Submitted {0} for processing.".format(
                        run_accession
                    ))
                except SbgError:
                    print('ERROR: Failed to submit {0}'.format(run_accession))
            total += 1
            if args.limit:
                if total >= args.limit:
                    break
            time.sleep(0.33)