A DeepVariant image for Singularity, reliant on a localimage pull of layers first

Singularity.spec

Bootstrap: localimage
From: deepvariant-0.6.1.simg

# Install Google Cloud SDK for the gcloud tool, then:
# We grab the official Docker image, push it to a locally running container repo, then get
# Singularity to pull it back.
# gcloud docker -- pull gcr.io/deepvariant-docker/deepvariant:0.6.1
# docker tag gcr.io/deepvariant-docker/deepvariant:0.6.1 localhost:5000/deepvariant:0.6.1
# docker run -d -p 5000:5000 --restart=always --name registry registry:2
# docker push localhost:5000/deepvariant:0.6.1
# SINGULARITY_NOHTTPS=1 singularity pull docker://localhost:5000/deepvariant:0.6.1
#
# Then use that container image to make this customized one:
# sudo singularity build deepvariant-custom.simg Singularity.spec

%environment
    PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
    DV_GPU_BUILD=0
    export PATH DV_GPU_BUILD

%apprun download_testdata
    mkdir input
    wget https://storage.googleapis.com/deepvariant/quickstart-testdata/NA12878_S1.chr20.10_10p1mb.bam -P input
    wget https://storage.googleapis.com/deepvariant/quickstart-testdata/NA12878_S1.chr20.10_10p1mb.bam.bai -P input
    wget https://storage.googleapis.com/deepvariant/quickstart-testdata/test_nist.b37_chr20_100kbp_at_10mb.bed -P input
    wget https://storage.googleapis.com/deepvariant/quickstart-testdata/test_nist.b37_chr20_100kbp_at_10mb.vcf.gz -P input
    wget https://storage.googleapis.com/deepvariant/quickstart-testdata/test_nist.b37_chr20_100kbp_at_10mb.vcf.gz.tbi -P input
    wget https://storage.googleapis.com/deepvariant/quickstart-testdata/ucsc.hg19.chr20.unittest.fasta -P input
    wget https://storage.googleapis.com/deepvariant/quickstart-testdata/ucsc.hg19.chr20.unittest.fasta.fai -P input
    wget https://storage.googleapis.com/deepvariant/quickstart-testdata/ucsc.hg19.chr20.unittest.fasta.gz -P input
    wget https://storage.googleapis.com/deepvariant/quickstart-testdata/ucsc.hg19.chr20.unittest.fasta.gz.fai -P input
    wget https://storage.googleapis.com/deepvariant/quickstart-testdata/ucsc.hg19.chr20.unittest.fasta.gz.gzi -P input

%apprun download_models
    mkdir models
    wget https://storage.googleapis.com/deepvariant/models/DeepVariant/0.5.0/DeepVariant-inception_v3-0.5.0%2Bcl-182548131.data-wgs_standard/model.ckpt.data-00000-of-00001 -P models
    wget https://storage.googleapis.com/deepvariant/models/DeepVariant/0.5.0/DeepVariant-inception_v3-0.5.0%2Bcl-182548131.data-wgs_standard/model.ckpt.index -P models
    wget https://storage.googleapis.com/deepvariant/models/DeepVariant/0.5.0/DeepVariant-inception_v3-0.5.0%2Bcl-182548131.data-wgs_standard/model.ckpt.meta -P models

%apprun make_examples
    exec /opt/deepvariant/bin/make_examples \
    --mode calling \
    --ref /dv2/input/ucsc.hg19.chr20.unittest.fasta.gz \
    --reads /dv2/input/NA12878_S1.chr20.10_10p1mb.bam \
    --examples output.examples.tfrecord \
    --regions "chr20:10,000,000-10,010,000"

%apprun call_variants
    exec /opt/deepvariant/bin/call_variants \
    --outfile call_variants_output.tfrecord \
    --examples output.examples.tfrecord \
    --checkpoint /dv2/models/model.ckpt

%apprun postprocess_variants
 exec /opt/deepvariant/bin/postprocess_variants \
    --ref /dv2/input/ucsc.hg19.chr20.unittest.fasta.gz \
    --infile call_variants_output.tfrecord \
    --outfile output.vcf


%runscript
if [ $# -eq 0 ]; then
    echo '''Example Usage:

    # download data to input and models
    singularity run --app download_testdata deepvariant
    singularity run --app download_models deepvariant

    # make the examples, mapping inputs
    singularity run --bind input:/dv2/input/ --app make_examples deepvariant

    # call variants, mapping models
    singularity run --bind models:/dv2/models/ --app call_variants deepvariant

    # postprocess variants
    singularity run --bind input:/dv2/input/ --app postprocess_variants deepvariant

    # https://github.com/google/deepvariant/blob/master/docs/deepvariant-docker.md
'''
else
    exec "$@"
fi
    

%post
    apt-get -y update && apt-get install -y wget && rm -rf /var/lib/apt/lists/*

    # https://github.com/google/deepvariant/blob/r0.5/deepvariant/docker/Dockerfile
    BASH_HEADER='#!/bin/bash' && \
    printf "%s\n%s\n" \
        "${BASH_HEADER}" \
        'python /opt/deepvariant/bin/make_examples.zip "$@"' > \
        /opt/deepvariant/bin/make_examples && \
    printf "%s\n%s\n" \
        "${BASH_HEADER}" \
        'python /opt/deepvariant/bin/call_variants.zip "$@"' > \
        /opt/deepvariant/bin/call_variants && \
    printf "%s\n%s\n" \
        "${BASH_HEADER}" \
        'python /opt/deepvariant/bin/postprocess_variants.zip "$@"' > \
        /opt/deepvariant/bin/postprocess_variants && \
    printf "%s\n%s\n" \
        "${BASH_HEADER}" \
        'python /opt/deepvariant/bin/model_train.zip "$@"' > \
        /opt/deepvariant/bin/model_train && \
    printf "%s\n%s\n" \
        "${BASH_HEADER}" \
        'python /opt/deepvariant/bin/model_eval.zip "$@"' > \
        /opt/deepvariant/bin/model_eval && \
    chmod +x /opt/deepvariant/bin/make_examples \
             /opt/deepvariant/bin/call_variants \
             /opt/deepvariant/bin/postprocess_variants \
             /opt/deepvariant/bin/model_train \
             /opt/deepvariant/bin/model_eval

Hi Andrew,

I tried to replicate your work with deepvariant 0.7.2. When I run the singularity run --bind input:/dv2/input/ --app make_examples deepvariant-custom.sif. I get this error message.

ERROR: ld.so: object 'libgtk3-nocsd.so.0' from LD_PRELOAD cannot be preloaded (cannot open shared object file): ignored.
ERROR: ld.so: object 'libgtk3-nocsd.so.0' from LD_PRELOAD cannot be preloaded (cannot open shared object file): ignored.
ERROR: ld.so: object 'libgtk3-nocsd.so.0' from LD_PRELOAD cannot be preloaded (cannot open shared object file): ignored.
ERROR: ld.so: object 'libgtk3-nocsd.so.0' from LD_PRELOAD cannot be preloaded (cannot open shared object file): ignored.
ERROR: ld.so: object 'libgtk3-nocsd.so.0' from LD_PRELOAD cannot be preloaded (cannot open shared object file): ignored.
Traceback (most recent call last):
File "/tmp/Bazel.runfiles_SoiERu/runfiles/com_google_deepvariant/deepvariant/make_examples.py", line 37, in
import numpy as np
File "/home/kerdawy/.local/lib/python2.7/site-packages/numpy/init.py", line 142, in
from . import add_newdocs
File "/home/kerdawy/.local/lib/python2.7/site-packages/numpy/add_newdocs.py", line 13, in
from numpy.lib import add_newdoc
File "/home/kerdawy/.local/lib/python2.7/site-packages/numpy/lib/init.py", line 8, in
from .type_check import *
File "/home/kerdawy/.local/lib/python2.7/site-packages/numpy/lib/type_check.py", line 11, in
import numpy.core.numeric as _nx
File "/home/kerdawy/.local/lib/python2.7/site-packages/numpy/core/init.py", line 26, in
raise ImportError(msg)
ImportError:
Importing the multiarray numpy extension module failed. Most
likely you are trying to import a failed build of numpy.
If you're working with a numpy git repo, try git clean -xdf (removes all
files not under version control). Otherwise reinstall numpy.

Original error was: libblas.so.3: cannot open shared object file: No such file or directory

Does that mean I have to go through all the python packages that deepvariant needs and install it in %post?

I would really appreciate your help as I have a bery limited knowledge in building containers

Thanks!