Gitlab CI on monorepo

gitlab-ci.yaml

image: google/cloud-sdk

stages:
  - test
  - build
  - pre_deploy
  - deploy

services:
  - docker:18.09.8-dind

variables:
  DOCKER_HOST: tcp://localhost:2375/

test_backend:
  stage: test
  script:
    - echo "Test for backend not found"
  only:
    refs:
      - master
    changes:
      - .gitlab-ci.yml
      - backend/**/*
      - deploy/*

test_frontend:
  stage: test
  script:
    - echo "Test for frontend not found"
  only:
    refs:
      - master
    changes:
      - .gitlab-ci.yml
      - frontend/**/*
      - deploy/*

build_backend_staging:
  stage: build
  script:
    - cat $STG_GCP_SVC_KEY | docker login -u _json_key --password-stdin https://gcr.io
    - docker build -t gcr.io/$STG_GCP_PROJECT_ID/dataops-be:$CI_COMMIT_SHORT_SHA backend
    - docker push gcr.io/$STG_GCP_PROJECT_ID/dataops-be:$CI_COMMIT_SHORT_SHA
  only:
    refs:
      - master
    changes:
      - .gitlab-ci.yml
      - backend/**/*
      - deploy/*

build_frontend_staging:
  stage: build
  script:
    - cat $STG_GCP_SVC_KEY | docker login -u _json_key --password-stdin https://gcr.io
    - docker build -t gcr.io/$STG_GCP_PROJECT_ID/dataops-fe:$CI_COMMIT_SHORT_SHA frontend
    - docker push gcr.io/$STG_GCP_PROJECT_ID/dataops-fe:$CI_COMMIT_SHORT_SHA
  only:
    refs:
      - master
    changes:
      - .gitlab-ci.yml
      - frontend/**/*
      - deploy/*

build_backend_production:
  stage: build
  script:
    - cat $PRD_GCP_SVC_KEY | docker login -u _json_key --password-stdin https://gcr.io
    - docker build -t gcr.io/$PRD_GCP_PROJECT_ID/dataops-be:$CI_COMMIT_TAG backend
    - docker push gcr.io/$PRD_GCP_PROJECT_ID/dataops-be:$CI_COMMIT_TAG
  only:
    refs:
      - tags
    changes:
      - .gitlab-ci.yml
      - backend/**/*
      - deploy/*

build_frontend_production:
  stage: build
  script:
    - cat $PRD_GCP_SVC_KEY | docker login -u _json_key --password-stdin https://gcr.io
    - docker build -t gcr.io/$PRD_GCP_PROJECT_ID/dataops-fe:$CI_COMMIT_TAG frontend
    - docker push gcr.io/$PRD_GCP_PROJECT_ID/dataops-fe:$CI_COMMIT_TAG
  only:
    refs:
      - tags
    changes:
      - .gitlab-ci.yml
      - frontend/**/*
      - deploy/*

pre_deploy_staging:
  stage: pre_deploy
  script:
    - connect_cluster $STG_GCP_SVC_KEY $STG_GCP_PROJECT_ID
    - pre_deployment $STG_DB_URL $STG_CACHE_URL $STG_CELERY_BROKER_URL $STG_CELERY_RESULT_BACKEND $STG_GCP_PROJECT_ID $STG_SQL_INSTANCE
    - migrate_db_schema
  only:
    refs:
      - master

pre_deploy_production:
  stage: pre_deploy
  script:
    - connect_cluster $PRD_GCP_SVC_KEY $PRD_GCP_PROJECT_ID
    - pre_deployment $PRD_DB_URL $PRD_CACHE_URL $PRD_CELERY_BROKER_URL $PRD_CELERY_RESULT_BACKEND $PRD_PROJECT_ID $PRD_SQL_INSTANCE
    - migrate_db_schema
  only:
    refs:
      - tags

deploy_be_staging:
  stage: deploy
  script:
    - connect_cluster $STG_GCP_SVC_KEY $STG_GCP_PROJECT_ID
    - be_deployment $STG_DB_URL $STG_CACHE_URL $STG_CELERY_BROKER_URL $STG_CELERY_RESULT_BACKEND $STG_GCP_PROJECT_ID $STG_SQL_INSTANCE
    - apply_and_rollout_backend
  when: manual
  only:
    refs:
      - master
    changes:
      - .gitlab-ci.yml
      - backend/**/*
      - deploy/*

deploy_fe_staging:
  stage: deploy
  when: manual
  script:
    - connect_cluster $STG_GCP_SVC_KEY $STG_GCP_PROJECT_ID
    - fe_deployment $STG_GCP_PROJECT_ID
    - apply_and_rollout_frontend
  only:
    refs:
      - master
    changes:
      - .gitlab-ci.yml
      - frontend/**/*
      - deploy/*

deploy_be_production:
  stage: deploy
  environment:
    name: production
  script:
    - connect_cluster $PRD_GCP_SVC_KEY $PRD_GCP_PROJECT_ID
    - be_deployment $PRD_DB_URL $PRD_CACHE_URL $PRD_CELERY_BROKER_URL $PRD_CELERY_RESULT_BACKEND $PRD_PROJECT_ID $PRD_SQL_INSTANCE
    - apply_and_rollout_backend
  when: manual
  only:
    refs:
      - tags
    changes:
      - .gitlab-ci.yml
      - backend/**/*
      - deploy/*

deploy_fe_production:
  stage: deploy
  when: manual
  script:
    - connect_cluster $PRD_GCP_SVC_KEY $PRD_GCP_PROJECT_ID
    - fe_deployment $PRD_GCP_PROJECT_ID
    - apply_and_rollout_backend
  only:
    refs:
      - tags
    changes:
      - .gitlab-ci.yml
      - frontend/**/*
      - deploy/*

.custom_functions: &custom_functions |
  export VERSION=$([ -z "$CI_COMMIT_TAG" ] && echo $CI_COMMIT_SHORT_SHA || echo $CI_COMMIT_TAG)

  # $1 = GCP service key
  # $2 = Project ID
  function connect_cluster() {
    cat $1 > key.json
    gcloud auth activate-service-account --key-file=key.json
    gcloud container clusters get-credentials kumparan --zone asia-southeast1-a --project $2
    kubectl config set-context --current --namespace=data-ops
  }

  # $1 = Database URL
  # $2 = Redis Cache URL
  # $3 = Celery broker URL
  # $4 = Celery result backend
  # $5 = Project ID
  # $6 = SQL Instance
  function pre_deployment() {
    sed 's/_DB_URL_/'"$1"'/g; s/_CACHE_URL_/'"$2"'/g; s/_CELERY_BROKER_URL_/'"$3"'/g; s/_CELERY_RESULT_BACKEND_/'"$4"'/g;' deploy/secrets.yaml > secrets.yaml;
    sed 's/_PROJECT_NAME_/'"$5"'/g; s/_VERSION_/'"$VERSION"'/g; s/_SQL_INSTANCE_/'"$6"'/g;' deploy/deployment-migrator.yaml > deployment-migrator.yaml;
  }

  # $1 = Database URL
  # $2 = Redis Cache URL
  # $3 = Celery broker URL
  # $4 = Celery result backend
  # $5 = Project ID
  # $6 = SQL Instance
  function be_deployment() {
    sed 's/_DB_URL_/'"$1"'/g; s/_CACHE_URL_/'"$2"'/g; s/_CELERY_BROKER_URL_/'"$3"'/g; s/_CELERY_RESULT_BACKEND_/'"$4"'/g;' deploy/secrets.yaml > secrets.yaml;
    sed 's/_PROJECT_NAME_/'"$5"'/g; s/_VERSION_/'"$VERSION"'/g; s/_SQL_INSTANCE_/'"$6"'/g;' deploy/deployment-be.yaml > deployment-be.yaml;
    sed 's/_PROJECT_NAME_/'"$5"'/g; s/_VERSION_/'"$VERSION"'/g; s/_SQL_INSTANCE_/'"$6"'/g;' deploy/deployment-worker.yaml > deployment-worker.yaml;
  }

  # $1 = Project ID
  function fe_deployment() {
    sed 's/_PROJECT_NAME_/'"$1"'/g; s/_VERSION_/'"$VERSION"'/g;' deploy/deployment-fe.yaml > deployment-fe.yaml;
  }

  function migrate_db_schema() {
    cat deployment-migrator.yaml

    kubectl apply -f deploy/configs.yaml
    kubectl apply -f secrets.yaml
    kubectl apply -f deployment-migrator.yaml

    kubectl rollout status -f deployment-migrator.yaml
    podname=$(kubectl get pods -l app=dataops-migrator --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' | head -1)
    kubectl wait --for=condition=Ready pod/$podname
    kubectl exec -ti $podname flask db upgrade
  }

  function apply_and_rollout_backend() {
    cat deployment-worker.yaml
    cat deployment-be.yaml

    kubectl delete deploy dataops-migrator-deployment
    kubectl apply -f deploy/configs.yaml
    kubectl apply -f secrets.yaml
    kubectl apply -f deployment-worker.yaml
    kubectl apply -f deployment-be.yaml
    kubectl apply -f deploy/services.yaml

    kubectl rollout status -f deployment-be.yaml
    kubectl rollout status -f deployment-worker.yaml
  }

  function apply_and_rollout_frontend() {
    cat deployment-fe.yaml

    kubectl apply -f deploy/configs.yaml
    kubectl apply -f deployment-fe.yaml
    kubectl apply -f deploy/services.yaml
    kubectl rollout status -f deployment-fe.yaml
  }

before_script:
  - *custom_functions