Census build workflow

argo-census-build.yml

apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
  generateName: census-build-quick
spec:
  serviceAccountName: vcp-dev-census-sa
  entrypoint: build
  VolumeClaimGC:
    strategy: OnWorkflowCompletion      # delete volume when workflow is done
  volumeClaimTemplates:                 # define volume, same syntax as k8s Pod spec
  - metadata:
      name: workdir                     # name of volume claim
    spec:
      accessModes: [ "ReadWriteOnce" ]
      storageClassName: "ebs-io2"
      resources:
        requests:
          storage: "8Ti"       
  arguments:
    parameters:
    - name: build-tag
    - name: build-args
      value: ""
    - name: branch
      value: main
    - name: target-uri
      value: s3://ivirshup-tmp/census-builds/
  templates:
  - name: build
    volumes:
    - name: host-devices
      hostPath:
        path: /dev
        type: Directory
    metadata:
      annotations:
        karpenter.sh/do-not-disrupt: "true"
    nodeSelector:
      node.kubernetes.io/instance-type: "r6id.32xlarge"
    script:
      image: ubuntu:24.04
      securityContext:
        privileged: true
      resources:
        requests:
      command: [bash]
      volumeMounts:                     # same syntax as k8s Pod spec
      - name: workdir
        mountPath: /mnt/vol
      - name: host-devices
        mountPath: /dev
      source: |
        set -ex

        # Install stuff

        export DEBIAN_FRONTEND=noninteractive
        export TZ=UTC

        apt update
        apt install -y build-essential software-properties-common git curl zip
        add-apt-repository ppa:deadsnakes/ppa
        apt update
        apt install -y python3.11-dev python3.11-full

        # Make sure pip is available and up to date
        python3.11 -m ensurepip
        python3.11 -m pip install --upgrade pip --break-system-packages

        # Install AWS CLI
        curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
        unzip awscliv2.zip
        ./aws/install

        # Install census builder
        git clone https://github.com/chanzuckerberg/cellxgene-census.git
        cd cellxgene-census
        git checkout {{inputs.parameters.branch}}
        python3.11 -m pip install -e tools/cellxgene_census_builder/ --break-system-packages

        # Mounts instance storage as swap, note that the value is hardcoded for the current instance type "r6id.32xlarge"
        tools/scripts/aws/swapon_instance_storage.sh 3

        # Preparing path for build
        CENSUS_BUILDER_WORKING_DIR=/mnt/vol/census-builds/
        mkdir -p $CENSUS_BUILDER_WORKING_DIR

        # Do build
        python3.11 -m cellxgene_census_builder.build_soma \
          -v \
          --build-tag {{inputs.parameters.build-tag}} \
          $CENSUS_BUILDER_WORKING_DIR build \
          {{inputs.parameters.build-args}}
        
        aws s3 sync $CENSUS_BUILDER_WORKING_DIR {{inputs.parameters.target-uri}}

        # TODO validate the generated object

        echo "Done!"
    inputs:
      parameters:
      - name: build-tag
        value: "{{inputs.parameters.build-tag}}"
      - name: target-uri
        value: "{{inputs.parameters.target-uri}}"
      - name: branch
        value: "{{inputs.parameters.branch}}"
      - name: build-args
        value: "{{inputs.parameters.build-args}}"

    affinity:
      nodeAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          nodeSelectorTerms:
          - matchExpressions:
            - key: kubernetes.io/arch
              operator: In
              values:
              - amd64