ckandoth · November 3, 2025 08:28
diff --git a/az-dgn.md b/az-dgn.md
diff --git a/process_fastqs.sh b/process_fastqs.sh
 #!/bin/bash

 set -uo pipefail

 # Clean up sentinel files from prior run if any
 rm -f /tmp/.failure /tmp/.success

 error() {
    echo "Error: $1" >&2
    touch /tmp/.failure # Signals failure to the orchestrator
    exit 1
 }

 if (( $# != 3 )); then
    cat >&2 << EOM_USAGE
 Usage: ./process_fastqs.sh [FASTQ_BLOB_DIR] [REF_BLOB_DIR] [OUTPUT_BLOB_DIR]

 Purpose: Process a single sample whose FASTQs are stored in a blob storage folder, and upload results back to blob storage

 Command-line arguments:
  FASTQ_BLOB_DIR - e.g. "fqs/giab/HG002" where "fqs" is the container and "giab/HG002" is the subfolder containing FASTQs to process
  REF_BLOB_DIR - e.g. "ref/hg38" where "ref" is the container and "hg38" is the subfolder containing Dragen reference data as a tar file
  OUTPUT_BLOB_DIR - e.g. "dgn/giab/HG002" where "dgn" is the container, "HG002" is the sample name, and "giab/HG002" is the subfolder for outputs

 Environment variables:
  AZURE_STORAGE_ACCOUNT - the name of the ADLS Gen2 storage account we will use

 EOM_USAGE
 touch /tmp/.failure
 exit 1
 fi

 if [[ -z "${AZURE_STORAGE_ACCOUNT:-}" ]]; then
    error "AZURE_STORAGE_ACCOUNT environment variable is not set"
 fi

 FASTQ_BLOB_DIR=$1
 REF_BLOB_DIR=$2
 OUTPUT_BLOB_DIR=$3

 echo "Processing ${FASTQ_BLOB_DIR} on VM $(hostname)"

 # Piece together the full blob storage URLs we will need
 STORAGE_ACCT_ENDPOINT=https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net
 FASTQ_BLOB_URL="${STORAGE_ACCT_ENDPOINT}/${FASTQ_BLOB_DIR}"
 REF_BLOB_URL="${STORAGE_ACCT_ENDPOINT}/${REF_BLOB_DIR}"
 OUTPUT_BLOB_URL="${STORAGE_ACCT_ENDPOINT}/${OUTPUT_BLOB_DIR}"

 # Format the local disk with xfs for better handling of large files and to clean up data from a prior run of this script
 PARTITION=$(findmnt -no SOURCE /mnt)
 sudo umount /mnt
 sudo mkfs.xfs -qf $PARTITION
 sudo mount $PARTITION /mnt
 sudo chown -R $USER:$GROUPS /mnt || error "Failed to change ownership of /mnt"

 # Create some directories we will need if they don't already exist
 mkdir -p /mnt/{fqs,dgn} || error "Failed to create directories in /mnt"
 mkdir -p /tmp/{ref,dgn} || error "Failed to create directories in /tmp"

 # Use Azure CLI to login with the managed identity, which stores creds under ~/.azure that AzCopy can use
 echo -n "Logging into managed identity in Azure subscription " && az login --identity --query [].name --output tsv
 export AZCOPY_AUTO_LOGIN_TYPE=AZCLI

 # If needed, upgrade AzCopy to the latest version to support auto-login with Azure CLI creds
 AZCOPY_VERSION=$(azcopy --version | awk '{print $3}')
 OLDER_VERSION=$(echo -e $AZCOPY_VERSION"\n10.31.0" | sort -V | head -n1)
 if [[ "$OLDER_VERSION" != "10.31.0" ]]; then
    echo "Upgrading AzCopy to support auto-login with Azure CLI creds..."
    curl -sSL -O https://packages.microsoft.com/config/alma/8/packages-microsoft-prod.rpm
    sudo rpm -i packages-microsoft-prod.rpm
    rm packages-microsoft-prod.rpm
    sudo yum install --quiet --assumeyes azcopy
 fi

 # Download reference data into /tmp if it wasn't already downloaded by a previous run of this script
 REF_DIR="/tmp/${REF_BLOB_DIR}"
 if [[ ! -d "${REF_DIR}" ]]; then
    echo "Downloading reference data into ${REF_DIR}..."
    azcopy cp "${REF_BLOB_URL}/*" ${REF_DIR} --recursive --output-level=quiet || error "Failed to download reference data"
 else
    echo "Reusing existing reference data under ${REF_DIR}..."
 fi

 # Download FASTQs and parse headers to make a FASTQ list for use with Dragen
 FQS_DIR="/mnt/${FASTQ_BLOB_DIR}"
 SAMPLE=$(basename $OUTPUT_BLOB_DIR)
 echo "Downloading FASTQs and creating a FASTQ list for Dragen under ${FQS_DIR}..."
 azcopy cp "${FASTQ_BLOB_URL}/*" ${FQS_DIR} --output-level=quiet || error "Failed to download FASTQs"
 FASTQ_LIST="${FQS_DIR}/fastq_list.csv"
 echo "RGID,RGSM,RGLB,Lane,Read1File,Read2File" > ${FASTQ_LIST}
 for fq1 in ${FQS_DIR}/*_R1*.fastq.gz; do
    HEADER=$(gzip -dc "$fq1" | head -n1)
    FLOWCELL=$(echo "$HEADER" | cut -f3 -d:)
    LANE=$(echo "$HEADER" | cut -f4 -d:)
    RGID="${FLOWCELL}.${LANE}.${SAMPLE}"
    fq2=$(echo "$fq1" | sed 's/_R1/_R2/')
    if [[ -f "${fq2}" ]]; then
        echo "$RGID,$SAMPLE,UnknownLibrary,$LANE,$fq1,$fq2" >> ${FASTQ_LIST}
    else
        error "Could not find R2 FASTQ for $fq1"
    fi
 done

 # Locate the population SNP VCF for use by the ASCN caller to measure B-allele counts
 ASCN_SNP_VCF=$(find ${REF_DIR} -name "*.ascn.snps.vcf.gz")

 echo "Running Dragen on sample ${SAMPLE} using ${FASTQ_LIST}..."
 OUTPUT_DIR="/mnt/${OUTPUT_BLOB_DIR}"
 mkdir -p $OUTPUT_DIR
 dragen --intermediate-results-dir /tmp/dgn --ref-dir "${REF_DIR}" --enable-map-align true --enable-map-align-output true --output-format CRAM --cram-version 3.1 --enable-duplicate-marking true --generate-sa-tags true --enable-sort true --qc-coverage-ignore-overlaps true --qc-detect-contamination true --enable-variant-caller true --vc-emit-ref-confidence GVCF --vc-compact-gvcf true --vc-enable-vcf-output true --vc-combine-phased-variants-distance 6 --enable-targeted true --targeted-merge-vc true --enable-sv true --enable-cnv true --cnv-population-b-allele-vcf "${ASCN_SNP_VCF}" --cnv-enable-cyto-output true --cnv-enable-mosaic-calling true --cnv-interval-width 10000 --cnv-enable-self-normalization true --cnv-enable-gcbias-correction true --cnv-counts-method start --cnv-enable-segdups-extension true --cnv-enable-tracks false --enable-hla true --enable-star-allele true --enable-pgx true --repeat-genotype-enable true --enable-mrjd true --mrjd-enable-high-sensitivity-mode true --fastq-list "${FASTQ_LIST}" --fastq-list-sample-id "${SAMPLE}" --output-directory "${OUTPUT_DIR}" --output-file-prefix "${SAMPLE}" || error "Dragen run failed."

 echo "Dragen run successful. Uploading outputs to ${OUTPUT_BLOB_URL}..."
 azcopy cp "${OUTPUT_DIR}/*" "${OUTPUT_BLOB_URL}" --output-level=quiet || error "Failed to upload outputs"

 # Cleanup and signal completion to the orchestrator
 rm -rf /tmp/dgn || error "Failed to delete /tmp/dgn"
 touch /tmp/.success
 echo "Finished with sample ${SAMPLE}."
	#!/bin/bash

	set -uo pipefail

	# Clean up sentinel files from prior run if any
	rm -f /tmp/.failure /tmp/.success

	error() {
	echo "Error: $1" >&2
	touch /tmp/.failure # Signals failure to the orchestrator
	exit 1
	}

	if (( $# != 3 )); then
	cat >&2 << EOM_USAGE
	Usage: ./process_fastqs.sh [FASTQ_BLOB_DIR] [REF_BLOB_DIR] [OUTPUT_BLOB_DIR]

	Purpose: Process a single sample whose FASTQs are stored in a blob storage folder, and upload results back to blob storage

	Command-line arguments:
	FASTQ_BLOB_DIR - e.g. "fqs/giab/HG002" where "fqs" is the container and "giab/HG002" is the subfolder containing FASTQs to process
	REF_BLOB_DIR - e.g. "ref/hg38" where "ref" is the container and "hg38" is the subfolder containing Dragen reference data as a tar file
	OUTPUT_BLOB_DIR - e.g. "dgn/giab/HG002" where "dgn" is the container, "HG002" is the sample name, and "giab/HG002" is the subfolder for outputs

	Environment variables:
	AZURE_STORAGE_ACCOUNT - the name of the ADLS Gen2 storage account we will use

	EOM_USAGE
	touch /tmp/.failure
	exit 1
	fi

	if [[ -z "${AZURE_STORAGE_ACCOUNT:-}" ]]; then
	error "AZURE_STORAGE_ACCOUNT environment variable is not set"
	fi

	FASTQ_BLOB_DIR=$1
	REF_BLOB_DIR=$2
	OUTPUT_BLOB_DIR=$3

	echo "Processing ${FASTQ_BLOB_DIR} on VM $(hostname)"

	# Piece together the full blob storage URLs we will need
	STORAGE_ACCT_ENDPOINT=https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net
	FASTQ_BLOB_URL="${STORAGE_ACCT_ENDPOINT}/${FASTQ_BLOB_DIR}"
	REF_BLOB_URL="${STORAGE_ACCT_ENDPOINT}/${REF_BLOB_DIR}"
	OUTPUT_BLOB_URL="${STORAGE_ACCT_ENDPOINT}/${OUTPUT_BLOB_DIR}"

	# Format the local disk with xfs for better handling of large files and to clean up data from a prior run of this script
	PARTITION=$(findmnt -no SOURCE /mnt)
	sudo umount /mnt
	sudo mkfs.xfs -qf $PARTITION
	sudo mount $PARTITION /mnt
	sudo chown -R $USER:$GROUPS /mnt \|\| error "Failed to change ownership of /mnt"

	# Create some directories we will need if they don't already exist
	mkdir -p /mnt/{fqs,dgn} \|\| error "Failed to create directories in /mnt"
	mkdir -p /tmp/{ref,dgn} \|\| error "Failed to create directories in /tmp"

	# Use Azure CLI to login with the managed identity, which stores creds under ~/.azure that AzCopy can use
	echo -n "Logging into managed identity in Azure subscription " && az login --identity --query [].name --output tsv
	export AZCOPY_AUTO_LOGIN_TYPE=AZCLI

	# If needed, upgrade AzCopy to the latest version to support auto-login with Azure CLI creds
	AZCOPY_VERSION=$(azcopy --version \| awk '{print $3}')
	OLDER_VERSION=$(echo -e $AZCOPY_VERSION"\n10.31.0" \| sort -V \| head -n1)
	if [[ "$OLDER_VERSION" != "10.31.0" ]]; then
	echo "Upgrading AzCopy to support auto-login with Azure CLI creds..."
	curl -sSL -O https://packages.microsoft.com/config/alma/8/packages-microsoft-prod.rpm
	sudo rpm -i packages-microsoft-prod.rpm
	rm packages-microsoft-prod.rpm
	sudo yum install --quiet --assumeyes azcopy
	fi

	# Download reference data into /tmp if it wasn't already downloaded by a previous run of this script
	REF_DIR="/tmp/${REF_BLOB_DIR}"
	if [[ ! -d "${REF_DIR}" ]]; then
	echo "Downloading reference data into ${REF_DIR}..."
	azcopy cp "${REF_BLOB_URL}/*" ${REF_DIR} --recursive --output-level=quiet \|\| error "Failed to download reference data"
	else
	echo "Reusing existing reference data under ${REF_DIR}..."
	fi

	# Download FASTQs and parse headers to make a FASTQ list for use with Dragen
	FQS_DIR="/mnt/${FASTQ_BLOB_DIR}"
	SAMPLE=$(basename $OUTPUT_BLOB_DIR)
	echo "Downloading FASTQs and creating a FASTQ list for Dragen under ${FQS_DIR}..."
	azcopy cp "${FASTQ_BLOB_URL}/*" ${FQS_DIR} --output-level=quiet \|\| error "Failed to download FASTQs"
	FASTQ_LIST="${FQS_DIR}/fastq_list.csv"
	echo "RGID,RGSM,RGLB,Lane,Read1File,Read2File" > ${FASTQ_LIST}
	for fq1 in ${FQS_DIR}/_R1.fastq.gz; do
	HEADER=$(gzip -dc "$fq1" \| head -n1)
	FLOWCELL=$(echo "$HEADER" \| cut -f3 -d:)
	LANE=$(echo "$HEADER" \| cut -f4 -d:)
	RGID="${FLOWCELL}.${LANE}.${SAMPLE}"
	fq2=$(echo "$fq1" \| sed 's/_R1/_R2/')
	if [[ -f "${fq2}" ]]; then
	echo "$RGID,$SAMPLE,UnknownLibrary,$LANE,$fq1,$fq2" >> ${FASTQ_LIST}
	else
	error "Could not find R2 FASTQ for $fq1"
	fi
	done

	# Locate the population SNP VCF for use by the ASCN caller to measure B-allele counts
	ASCN_SNP_VCF=$(find ${REF_DIR} -name "*.ascn.snps.vcf.gz")

	echo "Running Dragen on sample ${SAMPLE} using ${FASTQ_LIST}..."
	OUTPUT_DIR="/mnt/${OUTPUT_BLOB_DIR}"
	mkdir -p $OUTPUT_DIR
	dragen --intermediate-results-dir /tmp/dgn --ref-dir "${REF_DIR}" --enable-map-align true --enable-map-align-output true --output-format CRAM --cram-version 3.1 --enable-duplicate-marking true --generate-sa-tags true --enable-sort true --qc-coverage-ignore-overlaps true --qc-detect-contamination true --enable-variant-caller true --vc-emit-ref-confidence GVCF --vc-compact-gvcf true --vc-enable-vcf-output true --vc-combine-phased-variants-distance 6 --enable-targeted true --targeted-merge-vc true --enable-sv true --enable-cnv true --cnv-population-b-allele-vcf "${ASCN_SNP_VCF}" --cnv-enable-cyto-output true --cnv-enable-mosaic-calling true --cnv-interval-width 10000 --cnv-enable-self-normalization true --cnv-enable-gcbias-correction true --cnv-counts-method start --cnv-enable-segdups-extension true --cnv-enable-tracks false --enable-hla true --enable-star-allele true --enable-pgx true --repeat-genotype-enable true --enable-mrjd true --mrjd-enable-high-sensitivity-mode true --fastq-list "${FASTQ_LIST}" --fastq-list-sample-id "${SAMPLE}" --output-directory "${OUTPUT_DIR}" --output-file-prefix "${SAMPLE}" \|\| error "Dragen run failed."

	echo "Dragen run successful. Uploading outputs to ${OUTPUT_BLOB_URL}..."
	azcopy cp "${OUTPUT_DIR}/*" "${OUTPUT_BLOB_URL}" --output-level=quiet \|\| error "Failed to upload outputs"

	# Cleanup and signal completion to the orchestrator
	rm -rf /tmp/dgn \|\| error "Failed to delete /tmp/dgn"
	touch /tmp/.success
	echo "Finished with sample ${SAMPLE}."