Example GitLab CI Pipeline using Terraform, etc.

gistfile0.txt

---
# Most pre-req tooling, etc. is installed using jumphosts.yml playbook
variables:
  ADMIN_EMAIL: [email protected]
  CLOUD_PROVIDER: Azure # Define Supported Cloud Provider (Azure)
  GIT_CRYPT_ENABLED: "true" # Must be lowercase (true|false)
  GIT_SUBMODULE_STRATEGY: recursive
  ORGANIZATION: example_org
  PROJECT_NAME: example_project
  TERRAFORM_VERSION: 0.12.28

# Use tags to execute in the correct environments
# dev -> dev
# staging -> stg
# production -> prd
stages:
  - build
  - reset_dev_environment
  - reset_staging_environment
  - reset_prod_environment
  - terraform_apply_dev
  - terraform_apply_staging
  - terraform_apply_prod
  - ansible_dev
  - ansible_staging
  - ansible_prod

after_script:
  # Encrypt files using git-crypt
  - git-crypt lock

before_script:
  # Record GitLab Runner address for troubleshooting
  - curl --silent ifconfig.co
  # Manage $HOME/.ssh directory
  - |
    if [ ! -d "$HOME/.ssh" ]; then
      mkdir "$HOME/.ssh"
    else
      if [ -f "$HOME/.ssh/known_hosts" ]; then
        cat /dev/null > "$HOME/.ssh/known_hosts"
      fi
    fi
  # Ensure pre-reqs are met
  - |
    if [ -f /etc/debian_version ]; then
      apt-get update
      apt-get install -y git-crypt python3-venv sshpass
    fi
  # Decrypt files using git-crypt
  - |
    set +e
    git stash --quiet
    set -e
    base64 --decode $UNLOCK_KEY > $HOME/.git-crypt.key
    git-crypt unlock $HOME/.git-crypt.key
    set +e
    git stash pop --quiet
    set -e
  # Define our repo root directory so we can jump around easily
  - REPO_ROOT="$PWD"
  # Defines Python virtual environment directory to use
  - VENV_DIR="$REPO_ROOT/venv"
  # Define Ansible directory where deployments occur
  - ANSIBLE_DIR="$REPO_ROOT/Deployment/Ansible"
  # Define default Ansible inventory directory
  - ANSIBLE_INVENTORY_DIR="$REPO_ROOT/Inventory/Ansible"
  # Define Terraform directory where deployments occur
  - TERRAFORM_DIR="$REPO_ROOT/Deployment/Terraform/$PROJECT_NAME/$CLOUD_PROVIDER"
  # Ensure that tfenv has the correct Terraform version we need
  - echo $TERRAFORM_VERSION > $HOME/.terraform-version
  # Define tfenv directory
  - TFENV_DIR="$REPO_ROOT/.tfenv"
  # Append tfenv path
  - export PATH="$TFENV_DIR/bin:$PATH"

setup_build_environment:
  stage: build
  artifacts:
    paths:
      - .tfenv/
      - venv/
  script:
    - python3 -m venv "$VENV_DIR"
    - source "$VENV_DIR/bin/activate"
    - pip3 install --upgrade pip pip-tools
    - pip3 install --upgrade cffi
    - pip-sync requirements.txt
    # Install tfenv
    - |
      if [ ! -d $TFENV_DIR ]; then
        git clone https://github.com/tfutils/tfenv.git $TFENV_DIR
      fi
    # Install our default Terraform version
    - tfenv install
    # Check our installed Terraform versions and which one is being used
    - tfenv list
    # Get a list of all Python packages installed in virtual environment
    - pip3 list
    # Check our Ansible version
    - ansible --version
    # Check our Terraform version
    - terraform version

.terraform: &terraform
  script:
    - source "$VENV_DIR/bin/activate"
    # Ensure that Org, workspace and local execution are set
    - python "$REPO_ROOT/Tooling/terraform_cloud_management.py" --organization "${ORGANIZATION}" --token "$TERRAFORM_CLOUD_API_TOKEN" --workspace "${PROJECT_NAME}" --email "${ADMIN_EMAIL}"
    # Change to our Terraform directory
    - cd "$TERRAFORM_DIR/environments/$ENVIRONMENT"
    # Initialize all modules, etc. required
    - terraform init
    # Generate Module services plan for the environment
    - terraform plan -var-file "$TERRAFORM_DIR/terraform.tfvars" -out tfplan
    # Apply any changes for our environment services module
    - terraform apply tfplan

.terraform-destroy: &terraform-destroy
  script:
    - cd "$TERRAFORM_DIR/environments/$ENVIRONMENT"
    - terraform init
    - terraform destroy -var-file "$TERRAFORM_DIR/terraform.tfvars" -auto-approve

.ansible: &ansible
  script:
    - source "$VENV_DIR/bin/activate"
    # Change to our Terraform directory
    - cd "$TERRAFORM_DIR/environments/$ENVIRONMENT"
    # Initialize all modules, etc. required
    - terraform init
    # Refresh Terraform state to use for generating inventory
    - terraform refresh -var-file "$TERRAFORM_DIR/terraform.tfvars"
    # Change to our Terraform to Ansible inventory generator
    - cd "$REPO_ROOT/Tooling/terraform-to-ansible"
    # Install any Python requirements
    - pip-sync requirements.txt
    # Generate Ansible inventory from the most current Terraform state, overwrite
    # existing inventory, and use private IPs for ansible_host
    - python -m terraform_to_ansible --tfstatedir "$TERRAFORM_DIR/environments/$ENVIRONMENT" --output "$ANSIBLE_INVENTORY_DIR/$ENVIRONMENT.yml" --ansibleHost private --force
    # Change to repository root
    - cd "$REPO_ROOT"
    # Install Python requirements
    - pip-sync requirements.txt
    # Change to our Ansible playbooks directory
    - cd "$ANSIBLE_DIR/playbooks"
    # Copy ansible.cfg to home directory
    # https://docs.ansible.com/ansible/devel/reference_appendices/config.html#cfg-in-%201111%20world-writable-dir
    - cp ansible.cfg "$HOME/.ansible.cfg"
    # Export ANSIBLE_CONFIG
    - export ANSIBLE_CONFIG="$HOME/.ansible.cfg"
    # Export ANSIBLE_ROLES_PATH
    - export ANSIBLE_ROLES_PATH="$ANSIBLE_DIR/roles"
    # Execute Ansible prep for environment
    - ansible-playbook -i "$ANSIBLE_INVENTORY_DIR/$ENVIRONMENT.yml" prep.yml
    # Do a quick Ansible ping for all hosts for environment
    - ansible all -i "$ANSIBLE_INVENTORY_DIR/$ENVIRONMENT.yml" -m ping
    # Execute Ansible playbook for environment
    - ansible-playbook -i "$ANSIBLE_INVENTORY_DIR/$ENVIRONMENT.yml" playbook.yml

terraform_dev:
  <<: *terraform
  stage: terraform_apply_dev
  # tags:
  #   - development
  variables:
    ENVIRONMENT: development

terraform_staging:
  <<: *terraform
  stage: terraform_apply_staging
  # tags:
  #   - staging
  variables:
    ENVIRONMENT: staging
  only:
    refs:
      - master

terraform_prod:
  <<: *terraform
  stage: terraform_apply_prod
  # tags:
  #   - production
  variables:
    ENVIRONMENT: production
  only:
    refs:
      - master

ansible_dev:
  <<: *ansible
  stage: ansible_dev
  # tags:
  #   - development
  timeout: 4 hours
  variables:
    ENVIRONMENT: development

ansible_staging:
  <<: *ansible
  stage: ansible_staging
  # tags:
  #   - staging
  timeout: 4 hours
  variables:
    ENVIRONMENT: staging
  only:
    refs:
      - master

ansible_prod:
  <<: *ansible
  stage: ansible_prod
  # tags:
  #   - production
  timeout: 4 hours
  variables:
    ENVIRONMENT: production
  only:
    refs:
      - master

reset_dev_environment:
  <<: *terraform-destroy
  stage: reset_dev_environment
  when: manual
  variables:
    ENVIRONMENT: development

reset_staging_environment:
  <<: *terraform-destroy
  stage: reset_staging_environment
  when: manual
  variables:
    ENVIRONMENT: staging

reset_prod_environment:
  <<: *terraform-destroy
  stage: reset_prod_environment
  when: manual
  variables:
    ENVIRONMENT: production