Skip to content

Instantly share code, notes, and snippets.

@theraaz
Last active July 13, 2019 15:31
Show Gist options
  • Select an option

  • Save theraaz/933ab38f1dcbd985f448439728bfef6e to your computer and use it in GitHub Desktop.

Select an option

Save theraaz/933ab38f1dcbd985f448439728bfef6e to your computer and use it in GitHub Desktop.
Kubernetes + Travis CI/CD with DigitalOcean - https://medium.com/p/c0c4058b7734
# Use Dockerized infrastructure
sudo: false
language: generic
cache:
bundler: true
git:
depth: 10
node_js:
- "9"
services:
- docker
# only whitelisting these branches for branch-update, so that if any update or PR
# merged in these branches then build otherwise ignore pushes to same branches
# https://stackoverflow.com/q/31882306/2037323
branches:
only:
- development
- production
- master
before_install:
# we'll use Platform repo to hold Raz Platform wide common configs etc
- sudo service mysql stop
- pip install --user awscli
- export PATH=$PATH:$HOME/.local/bin
- export IMAGE_NAME=raz4/raz4api
- export REPO_URL=123.dkr.ecr.eu-west-2.amazonaws.com # your AWS ECR repo
- export GIT_REPO_NAME=$(echo $TRAVIS_REPO_SLUG | grep -oE "[^/]+$")
- export GIT_SHA_TS=$(git log -1 --format=%ai)
- export GIT_SHA=$(git rev-parse HEAD)
# i use a separate repo for secrets and config for all the projects
- git clone https://$GITHUB_TOKEN@github.com/username/repo.git ../Platform
install:
- eval $(aws ecr get-login --region eu-west-2 --no-include-email)
- sudo apt-get update && sudo apt-get -y install gettext-base
# - docker pull $REPO_URL/$IMAGE_NAME:latest
- docker-compose -f ./dev/docker-compose.yml up --remove-orphans -d
- docker build --build-arg GIT_SHA=$GIT_SHA --build-arg GIT_SHA_TS="$GIT_SHA_TS" -t $REPO_URL/$IMAGE_NAME:latest .
- docker tag $REPO_URL/$IMAGE_NAME:latest $REPO_URL/$IMAGE_NAME:$TRAVIS_COMMIT
- docker tag $REPO_URL/$IMAGE_NAME:latest $REPO_URL/$IMAGE_NAME:$TRAVIS_BRANCH
script:
# next two lines optional. I'm running my tests here
- docker run --env-file ./dev/.test.env --net dev_raz_test_network $REPO_URL/$IMAGE_NAME npm run lint || travis_terminate 1
- docker run --env-file ./dev/.test.env -e CODECOV_TOKEN=$CODECOV_TOKEN --net dev_raz_test_network $REPO_URL/$IMAGE_NAME npm run test:cov || travis_terminate 1
- docker push $REPO_URL/$IMAGE_NAME:latest
- docker push $REPO_URL/$IMAGE_NAME:$TRAVIS_COMMIT
- docker push $REPO_URL/$IMAGE_NAME:$TRAVIS_BRANCH
- cp -r ../Platform/k8s/. ../$GIT_REPO_NAME/dev
deploy:
skip_cleanup: true
provider: script
script: ./dev/ci-deploy.sh
on:
all_branches: true
condition: $TRAVIS_BRANCH =~ ^development|master|production$
# https://gist.github.com/cablespaghetti/b5343b04dd5bdc68dcb62754986a34ed
# this file along with ecr-red-updater.sh is a one time run thing. it is used to setup a crontask in k8s
# to get auth details from AWS so that we can fetch docker repo. once called, no need to call again
# to use, just call the bash file ./ecr-cred-updater.sh [namespace-here]
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: ecr-cred-updater
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "create", "delete"]
- apiGroups: [""]
resources: ["serviceaccounts"]
verbs: ["get", "patch"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: ecr-cred-updater
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: ecr-cred-updater
subjects:
- kind: ServiceAccount
name: ecr-cred-updater
roleRef:
kind: Role
name: ecr-cred-updater
apiGroup: rbac.authorization.k8s.io
---
apiVersion: batch/v1
kind: Job
metadata:
name: ecr-cred-updater
spec:
backoffLimit: 4
template:
spec:
serviceAccountName: ecr-cred-updater
terminationGracePeriodSeconds: 0
restartPolicy: Never
containers:
- name: kubectl
image: xynova/aws-kubectl
command:
- "/bin/sh"
- "-c"
- |
AWS_ACCOUNT=AWS_ACCOUNT_ID_HERE
export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
export AWS_REGION=eu-west-2
DOCKER_REGISTRY_SERVER=https://${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com
DOCKER_USER=AWS
DOCKER_PASSWORD=`aws ecr get-login --region ${AWS_REGION} --registry-ids ${AWS_ACCOUNT} | cut -d' ' -f6`
kubectl delete secret --ignore-not-found eu-west-2-ecr-registry || true
kubectl create secret docker-registry eu-west-2-ecr-registry \
--docker-server=$DOCKER_REGISTRY_SERVER \
--docker-username=$DOCKER_USER \
--docker-password=$DOCKER_PASSWORD \
--docker-email=no@email.local
kubectl patch serviceaccount default -p '{"imagePullSecrets":[{"name":"eu-west-2-ecr-registry"}]}'
---
apiVersion: batch/v1beta1
kind: CronJob
metadata:
name: ecr-cred-updater
spec:
schedule: "* */8 * * *"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 4
template:
spec:
serviceAccountName: ecr-cred-updater
terminationGracePeriodSeconds: 0
restartPolicy: Never
containers:
- name: kubectl
image: xynova/aws-kubectl
command:
- "/bin/sh"
- "-c"
- |
AWS_ACCOUNT=AWS_ACCOUNT_ID_HERE
export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
export AWS_REGION=eu-west-2
DOCKER_REGISTRY_SERVER=https://${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com
DOCKER_USER=AWS
DOCKER_PASSWORD=`aws ecr get-login --region ${AWS_REGION} --registry-ids ${AWS_ACCOUNT} | cut -d' ' -f6`
kubectl delete secret --ignore-not-found eu-west-2-ecr-registry || true
kubectl create secret docker-registry eu-west-2-ecr-registry \
--docker-server=$DOCKER_REGISTRY_SERVER \
--docker-username=$DOCKER_USER \
--docker-password=$DOCKER_PASSWORD \
--docker-email=no@email.local
kubectl patch serviceaccount cicd -p '{"imagePullSecrets":[{"name":"eu-west-2-ecr-registry"}]}'
kubectl patch serviceaccount default -p '{"imagePullSecrets":[{"name":"eu-west-2-ecr-registry"}]}'
#! /bin/bash
# exit script when any command ran here returns with non-zero exit code
set -e
COMMIT_SHA1=$TRAVIS_COMMIT
# We must export it so it's available for envsubst
export COMMIT_SHA1=$TRAVIS_COMMIT
curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
chmod u+x ./kubectl
./dev/platform.sh
echo "$KUBERNETES_CLUSTER_CERTIFICATE" | base64 --decode > cert.crt
# now mapping branches to namespaces in k8s
# since we got staging namespace for master branch so setting that in variable
if [[ $TRAVIS_BRANCH == "master" ]]
then
k8s_namespace=staging
else
k8s_namespace=$TRAVIS_BRANCH
fi
# lower case name needed for kubernetes secrets name
GIT_REPO_NAME_LC=$(echo $GIT_REPO_NAME | tr "[:upper:]" "[:lower:]")
# create/update secrets
# https://stackoverflow.com/a/45881259/2037323
./kubectl --kubeconfig=/dev/null \
--server=$KUBERNETES_SERVER \
--certificate-authority=cert.crt \
--token=$KUBERNETES_TOKEN \
create secret generic "v4-secrets-$GIT_REPO_NAME_LC" \
--from-env-file=./dev/$k8s_namespace/$GIT_REPO_NAME/.env \
--dry-run -o yaml \
--namespace=$k8s_namespace | \
./kubectl --kubeconfig=/dev/null \
--server=$KUBERNETES_SERVER \
--certificate-authority=cert.crt \
--token=$KUBERNETES_TOKEN \
apply --namespace=$k8s_namespace -f -
# since the only way for envsubst to work on files is using input/output redirection,
# it's not possible to do in-place substitution, so we need to save the output to another file
# and overwrite the original with that one.
envsubst <./dev/deployment.$k8s_namespace.yml >./dev/deployment.yml.out
mv ./dev/deployment.yml.out ./dev/deployment.$k8s_namespace.yml
./kubectl \
--kubeconfig=/dev/null \
--server=$KUBERNETES_SERVER \
--certificate-authority=cert.crt \
--token=$KUBERNETES_TOKEN \
apply -f ./dev/deployment.$k8s_namespace.yml --record
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cicd
subjects:
- kind: ServiceAccount
name: cicd
namespace: default
roleRef:
kind: ClusterRole
name: cicd
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cicd
rules:
- apiGroups: ["", "apps", "batch", "extensions"]
resources: ["deployments", "services", "replicasets", "pods", "jobs", "cronjobs", "secrets", "roles"]
verbs: ["*"]
apiVersion: apps/v1
kind: Deployment
metadata:
name: raz4
namespace: development
labels:
app: raz4
spec:
replicas: 1
minReadySeconds: 60
selector:
matchLabels:
app: raz4api
template:
metadata:
labels:
app: raz4api
spec:
containers:
- name: raz4api
image: $REPO_URL/$IMAGE_NAME:$TRAVIS_COMMIT
ports:
- containerPort: $RAZ_API_PORT
name: http
envFrom:
- secretRef:
name: v4-secrets-smartapi
readinessProbe:
initialDelaySeconds: 100
periodSeconds: 5
timeoutSeconds: 5
failureThreshold: 3
httpGet:
path: /status
port: http
livenessProbe:
initialDelaySeconds: 100
periodSeconds: 5
timeoutSeconds: 5
failureThreshold: 3
httpGet:
path: /status
port: http
imagePullSecrets:
- name: eu-west-2-ecr-registry
apiVersion: apps/v1
kind: Deployment
metadata:
name: raz4
namespace: production
labels:
app: raz4
spec:
replicas: 3
minReadySeconds: 60 # wait for 60 seconds before considering pod as ready and continuing with next
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
maxSurge: 1
selector:
matchLabels:
app: raz4api
template:
metadata:
labels:
app: raz4api
spec:
containers:
- name: raz4api
image: $REPO_URL/$IMAGE_NAME:$TRAVIS_COMMIT
ports:
- containerPort: $RAZ_API_PORT
name: http
envFrom:
- secretRef:
name: v4-secrets-smartapi
readinessProbe:
initialDelaySeconds: 600
periodSeconds: 5
timeoutSeconds: 5
failureThreshold: 3
httpGet:
path: /status
port: http
livenessProbe:
initialDelaySeconds: 600
periodSeconds: 5
timeoutSeconds: 5
failureThreshold: 3
httpGet:
path: /status
port: http
imagePullSecrets:
- name: eu-west-2-ecr-registry
apiVersion: apps/v1
kind: Deployment
metadata:
name: raz4
namespace: staging
labels:
app: raz4
spec:
replicas: 1
minReadySeconds: 60
selector:
matchLabels:
app: raz4api
template:
metadata:
labels:
app: raz4api
spec:
containers:
- name: raz4api
image: $REPO_URL/$IMAGE_NAME:$TRAVIS_COMMIT
ports:
- containerPort: $RAZ_API_PORT
name: http
envFrom:
- secretRef:
name: v4-secrets-smartapi
readinessProbe:
initialDelaySeconds: 600
periodSeconds: 5
timeoutSeconds: 5
failureThreshold: 3
httpGet:
path: /status
port: http
livenessProbe:
initialDelaySeconds: 600
periodSeconds: 5
timeoutSeconds: 5
failureThreshold: 3
httpGet:
path: /status
port: http
imagePullSecrets:
- name: eu-west-2-ecr-registry
apiVersion: v1
kind: ServiceAccount
metadata:
name: cicd
namespace: default
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment