GitLab CI - nf-core/modules

.gitlab-ci.yml

default:
  image: <DEFAULT IMAGE>
  tags:
    - <RUNNER TAGS>

stages:
  - lint
  - pipeline-generators
  - pipeline-triggers

lint-repo:
  stage: lint
  image: node:22
  script:
    - npm install -g [email protected]
    - npm install -g editorconfig-checker
    - prettier --check .
    - editorconfig-checker $(git ls-files | grep -v test)
  rules:
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_PIPELINE_SOURCE == "merge_request_event"'

generate-module-test-pipeline:
  stage: pipeline-generators
  image: <PYTHON IMAGE>
  script:
    - python ci/generate_ci.py
  artifacts:
    paths:
      - module-test-ci.yml
  rules:
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_PIPELINE_SOURCE == "merge_request_event"'
      changes:
        - modules/**/*

trigger-module-test-pipeline:
  stage: pipeline-triggers
  variables:
    PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE
  trigger:
    include:
      - artifact: module-test-ci.yml
        job: generate-module-test-pipeline
    strategy: depend
  rules:
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_PIPELINE_SOURCE == "merge_request_event"'
      changes:
        - modules/**/*

generate_ci.py

import pathlib

WORKFLOW_TEMPLATE = """\
---
default:
  tags:
    - <RUNNER TAGS>

stages:
  - lint-modules
  - nf-test-modules

.dind:
  variables:
    DOCKER_DRIVER: overlay2
    DOCKER_HOST: tcp://docker:2376/
    DOCKER_TLS_CERTDIR: "/certs"
    DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
    DOCKER_TLS_VERIFY: 1
  services:
    - name: docker:24.0.1-dind
      command:
        [
          "--experimental",
          "--registry-mirror",
          "<DOCKER URL>",
        ]

"""

def module_test_ci_job(module_path):
    module_name = str(module_path.relative_to(MODULES_PATH)).replace('/', '_')
    return f"""\
lint_{module_name}:
  image: <IMAGE WITH nf-core/tools>
  stage: lint-modules
  script:
    - nf-core modules <YOUR REMOTE OPTIONS e.g. --git-repo> lint {str(module_path.relative_to(MODULES_PATH))}
  rules:
    - if: $PARENT_PIPELINE_SOURCE == "web"
      when: manual
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_MERGE_REQUEST_ID'
      changes:
      - {module_path}/**/*
      
      
test_{module_name}:
  image: <IMAGE WITH DOCKER/SINGULARITY/nf-test>
  extends: .dind
  stage: nf-test-modules
  variables:
    NFT_DIFF: "pdiff"
    NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2"
  parallel:
    matrix:
      - PROFILE: ['singularity', 'docker']
  script:
    - export MODULES_TESTDATADIR=$(pwd)/test-data
    - >
      nf-test test
      --profile=$PROFILE
      --tap=test.tap
      --verbose
      {module_path}
  artifacts:
    when: always
    name: {module_name}_nftest_output
    paths:
      - test.tap
  rules:
    - if: $PARENT_PIPELINE_SOURCE == "web"
      when: manual
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_MERGE_REQUEST_ID'
      changes:
      - {module_path}/**/*
"""

MODULES_PATH = 'modules/<YOUR-ORG_PATH>'

def main():
    module_paths = [m.parent for m in pathlib.Path(MODULES_PATH).rglob('main.nf')]

    with open("module-test-ci.yml", "w") as cif:
        cif.write(WORKFLOW_TEMPLATE)
        cif.write("\n")
        for m in module_paths:
            job = module_test_ci_job(m)
            cif.write(job)
            cif.write("\n")


if __name__ == "__main__":
    main()