Skip to content

Instantly share code, notes, and snippets.

@chapmanb

chapmanb/explore_batching_vcs.py

Created December 13, 2017 03:15
Show Gist options
  • Save chapmanb/093a77e6258d80b24f21393611624a40 to your computer and use it in GitHub Desktop.
Save chapmanb/093a77e6258d80b24f21393611624a40 to your computer and use it in GitHub Desktop.
Download ZIP
CWL intermediate staging with same filenames
Raw
explore_batching_vcs.py
import json
import subprocess
job = json.loads(subprocess.check_output(["dx", "describe", "job-F8fgxBQ0XFJF2G9k5FqYg08Z", "--json"]))
input_files = [x["primaryFile"]["$dnanexus_link"] for x in job["input"]["config__algorithm__validate"]]
output_files = [x[0]["config__algorithm__validate"]["primaryFile"]["$dnanexus_link"] for x in job["output"]["batch_rec"]]
for f in input_files:
f_d = json.loads(subprocess.check_output(["dx", "describe", f, "--json"]))
print("input", f, f_d["folder"], f_d["name"], f_d["size"])
for f in output_files:
f_d = json.loads(subprocess.check_output(["dx", "describe", f, "--json"]))
print("output", f, f_d["folder"], f_d["name"], f_d["size"])
Raw
input_output_stats.txt
('input', u'file-F8b75y00f5v7v6Z56Kkv7XVP', u'/reference_genomes/hg38/validation/giab-NA12878', u'truth_small_variants.vcf.gz', 113539619)
('input', u'file-F8b75yj0f5vG869V6X06Y20Y', u'/reference_genomes/hg38/validation/giab-NA24385', u'truth_small_variants.vcf.gz', 139437797)
('input', u'file-F8b75yj0f5vG869V6X06Y20Y', u'/reference_genomes/hg38/validation/giab-NA24385', u'truth_small_variants.vcf.gz', 139437797)
('input', u'file-F8b75y00f5v7v6Z56Kkv7XVP', u'/reference_genomes/hg38/validation/giab-NA12878', u'truth_small_variants.vcf.gz', 113539619)
('output', u'file-F8fyYXj0XFJ85zQK2xG2GzGQ', u'/dx-cwl-run/main-giab-joint-dragen-output/intermediate-results', u'truth_small_variants.vcf.gz', 113539619)
('output', u'file-F8fygv00XFJ4BJ4v2xYK5jkg', u'/dx-cwl-run/main-giab-joint-dragen-output/intermediate-results', u'truth_small_variants.vcf.gz', 113539619)
('output', u'file-F8fyvgQ0XFJ85zQK2xG2J8JB', u'/dx-cwl-run/main-giab-joint-dragen-output/intermediate-results', u'truth_small_variants.vcf.gz', 113539619)
('output', u'file-F8fz1780XFJ2QyYY0xqjjJyg', u'/dx-cwl-run/main-giab-joint-dragen-output/intermediate-results', u'truth_small_variants.vcf.gz', 113539619)
('output', u'file-F8fz5fQ0XFJ4BJ4v2xYK6jx5', u'/dx-cwl-run/main-giab-joint-dragen-output/intermediate-results', u'truth_small_variants.vcf.gz', 113539619)
('output', u'file-F8fzBj80XFJJ8bZ42y38KxXY', u'/dx-cwl-run/main-giab-joint-dragen-output/intermediate-results', u'truth_small_variants.vcf.gz', 113539619)
('output', u'file-F8fzP180XFJ7jkKj0y0x501P', u'/dx-cwl-run/main-giab-joint-dragen-output/intermediate-results', u'truth_small_variants.vcf.gz', 113539619)
('output', u'file-F8fzYV80XFJ7jkKj0y0x518g', u'/dx-cwl-run/main-giab-joint-dragen-output/intermediate-results', u'truth_small_variants.vcf.gz', 113539619)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment