airflow.cfg

[core]
airflow_home = /etc/airflow
dags_folder = /home/airflow/gcs/dags
base_log_folder = /home/airflow/gcs/logs
plugins_folder = /home/airflow/gcs/plugins
remote_logging = True
remote_log_conn_id = google_cloud_default
remote_base_log_folder = gs://asia-northeast1-settings-te-xxxxxxxx-bucket/logs
executor = CeleryExecutor
dags_are_paused_at_creation = False
load_examples = False
donot_pickle = True
dagbag_import_timeout = 30
fernet_key = W5yfP4Cj6cuwOEcJteu3sd3-xxxxxxxxxxxxxxxxxxx=
parallelism = 30
dag_concurrency = 15
max_active_runs_per_dag = 15
enable_xcom_pickling = False

[webserver]
web_server_host = 0.0.0.0
web_server_port = 8080
secret_key = temporary_key
workers = 2
worker_class = sync
expose_config = True
web_server_name = settings-test

[celery]
celery_app_name = airflow.executors.celery_executor
worker_concurrency = 6
worker_log_server_port = 8793
broker_url = redis://airflow-redis-service:6379/0
result_backend = redis://airflow-redis-service:6379/0
flower_port = 5555
default_queue = default
ssl_active = False

[email]
email_backend = airflow.contrib.utils.sendgrid.send_email

[smtp]
smtp_host = localhost
smtp_starttls = True
smtp_ssl = False
smtp_user = airflow
smtp_port = 25
smtp_password = airflow
smtp_mail_from = [email protected]

[scheduler]
dag_dir_list_interval = 100

[kubernetes]
in_cluster = True

daemonset.yaml

apiVersion: v1
items:
- apiVersion: extensions/v1beta1
  kind: DaemonSet
  metadata:
    creationTimestamp: "2019-02-27T01:27:14Z"
    generation: 1
    name: composer-fluentd-daemon
    namespace: default
    resourceVersion: "985"
    selfLink: /apis/extensions/v1beta1/namespaces/default/daemonsets/composer-fluentd-daemon
    uid: d03578bb-3a2e-11e9-8976-42010a9200db
  spec:
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        name: composer-fluentd
    template:
      metadata:
        creationTimestamp: null
        labels:
          name: composer-fluentd
      spec:
        containers:
        - env:
          - name: FLUENTD_ARGS
            value: --no-supervisor -q
          - name: COMPOSER_ENVIRONMENT
            value: settings-test
          image: gcr.io/cloud-airflow-releaser/composer-fluentd:cloud_composer_service_2019-02-05-RC4
          imagePullPolicy: IfNotPresent
          livenessProbe:
            exec:
              command:
              - /bin/sh
              - -c
              - |
                LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-300}; STUCK_THRESHOLD_SECONDS=${STUCK_THRESHOLD_SECONDS:-900}; if [ ! -e /var/log/fluentd-buffers ]; then
                  exit 1;
                fi; LAST_MODIFIED_DATE=`stat /var/log/fluentd-buffers | grep Modify | sed -r "s/Modify: (.*)/\1/"`; LAST_MODIFIED_TIMESTAMP=`date -d "$LAST_MODIFIED_DATE" +%s`; if [ `date +%s` -gt `expr $LAST_MODIFIED_TIMESTAMP + $STUCK_THRESHOLD_SECONDS` ]; then
                  rm -rf /var/log/fluentd-buffers;
                  exit 1;
                fi; if [ `date +%s` -gt `expr $LAST_MODIFIED_TIMESTAMP + $LIVENESS_THRESHOLD_SECONDS` ]; then
                  exit 1;
                fi;
            failureThreshold: 3
            initialDelaySeconds: 300
            periodSeconds: 60
            successThreshold: 1
            timeoutSeconds: 1
          name: composer-fluentd
          resources: {}
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
          - mountPath: /var/log
            name: varlog
          - mountPath: /var/lib/docker/containers
            name: varlibdockercontainers
        dnsPolicy: ClusterFirst
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        terminationGracePeriodSeconds: 30
        volumes:
        - hostPath:
            path: /var/log
            type: ""
          name: varlog
        - hostPath:
            path: /var/lib/docker/containers
            type: ""
          name: varlibdockercontainers
    templateGeneration: 1
    updateStrategy:
      rollingUpdate:
        maxUnavailable: 1
      type: RollingUpdate
  status:
    currentNumberScheduled: 3
    desiredNumberScheduled: 3
    numberAvailable: 3
    numberMisscheduled: 0
    numberReady: 3
    observedGeneration: 1
    updatedNumberScheduled: 3
kind: List
metadata:
  resourceVersion: ""
  selfLink: ""

deployment.yaml

apiVersion: v1
items:
- apiVersion: extensions/v1beta1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2019-02-26T10:08:16Z"
    generation: 1
    labels:
      run: airflow-scheduler
    name: airflow-scheduler
    namespace: default
    resourceVersion: "1632"
    selfLink: /apis/extensions/v1beta1/namespaces/default/deployments/airflow-scheduler
    uid: 6f0307e9-39ae-11e9-bf53-xxxxxxxxxxxx
  spec:
    progressDeadlineSeconds: 2147483647
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        run: airflow-scheduler
    strategy:
      rollingUpdate:
        maxSurge: 1
        maxUnavailable: 1
      type: RollingUpdate
    template:
      metadata:
        creationTimestamp: null
        labels:
          run: airflow-scheduler
      spec:
        containers:
        - args:
          - scheduler
          env:
          - name: CLOUDSDK_METRICS_ENVIRONMENT
            value: 1.10.1-composer
          - name: GCS_BUCKET
            value: asia-northeast1-settings-te-xxxxxxxx-bucket
          - name: AIRFLOW_HOME
            value: /etc/airflow
          - name: DAGS_FOLDER
            value: /home/airflow/gcs/dags
          - name: SQL_DATABASE
            value: airflow-db
          - name: SQL_USER
            value: root
          - name: SQL_PASSWORD
          - name: AIRFLOW__WEBSERVER__BASE_URL
            value: https://nf2d2247d4fdcdaed-tp.appspot.com
          - name: AIRFLOW__CORE__SQL_ALCHEMY_CONN
            value: mysql+mysqldb://$(SQL_USER):$(SQL_PASSWORD)@airflow-sqlproxy-service/$(SQL_DATABASE)
          - name: GCP_PROJECT
            value: xxxxxxxxx
          - name: COMPOSER_LOCATION
            value: asia-northeast1
          - name: COMPOSER_PYTHON_VERSION
            value: "2"
          - name: COMPOSER_ENVIRONMENT
            value: settings-test
          - name: GKE_CLUSTER_NAME
            value: asia-northeast1-settings-te-xxxxxxxx-gke
          - name: CONTAINER_NAME
            value: airflow-scheduler
          image: gcr.io/cloud-airflow-releaser/airflow-worker-scheduler-1.10.1:cloud_composer_service_2019-02-05-RC4
          imagePullPolicy: IfNotPresent
          livenessProbe:
            exec:
              command:
              - python
              - /var/local/scheduler_checker.py
            failureThreshold: 3
            initialDelaySeconds: 120
            periodSeconds: 60
            successThreshold: 1
            timeoutSeconds: 1
          name: airflow-scheduler
          resources: {}
          securityContext:
            privileged: true
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
          - mountPath: /etc/airflow/airflow_cfg
            name: airflow-config
          - mountPath: /home/airflow/gcs
            name: gcsdir
        - args:
          - /home/airflow/gcs
          env:
          - name: GCS_BUCKET
            value: asia-northeast1-settings-te-xxxxxxxx-bucket
          image: gcr.io/cloud-airflow-releaser/gcs-syncd:cloud_composer_service_2019-02-05-RC4
          imagePullPolicy: IfNotPresent
          name: gcs-syncd
          resources:
            requests:
              cpu: 10m
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
          - mountPath: /home/airflow/gcs
            name: gcsdir
        dnsPolicy: ClusterFirst
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        terminationGracePeriodSeconds: 30
        volumes:
        - configMap:
            defaultMode: 420
            name: airflow-configmap
          name: airflow-config
        - emptyDir: {}
          name: gcsdir
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2019-02-26T10:08:16Z"
      lastUpdateTime: "2019-02-26T10:08:16Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: extensions/v1beta1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2019-02-26T10:08:16Z"
    generation: 1
    labels:
      run: airflow-sqlproxy
    name: airflow-sqlproxy
    namespace: default
    resourceVersion: "1636"
    selfLink: /apis/extensions/v1beta1/namespaces/default/deployments/airflow-sqlproxy
    uid: 6f0fd545-39ae-11e9-bf53-xxxxxxxxxxxx
  spec:
    progressDeadlineSeconds: 2147483647
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        run: airflow-sqlproxy
    strategy:
      rollingUpdate:
        maxSurge: 1
        maxUnavailable: 1
      type: RollingUpdate
    template:
      metadata:
        creationTimestamp: null
        labels:
          run: airflow-sqlproxy
      spec:
        containers:
        - command:
          - /cloud_sql_proxy
          - -instances=nf2d2247d4fdcdaed-tp:asia-northeast1:asia-northeast1-settings-te-xxxxxxxx-sql=tcp:0.0.0.0:3306
          image: gcr.io/cloudsql-docker/gce-proxy:1.09
          imagePullPolicy: IfNotPresent
          livenessProbe:
            exec:
              command:
              - /bin/sh
              - -c
              - netstat -tlnp | grep -i cloud_sql_proxy
            failureThreshold: 3
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: airflow-sqlproxy
          ports:
          - containerPort: 3306
            protocol: TCP
          resources: {}
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
          - mountPath: /etc/ssl/certs
            name: ssl-certs
        dnsPolicy: ClusterFirst
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        terminationGracePeriodSeconds: 30
        volumes:
        - hostPath:
            path: /etc/ssl/certs
            type: ""
          name: ssl-certs
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2019-02-26T10:08:16Z"
      lastUpdateTime: "2019-02-26T10:08:16Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: extensions/v1beta1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2019-02-26T10:08:16Z"
    generation: 1
    labels:
      run: airflow-worker
    name: airflow-worker
    namespace: default
    resourceVersion: "1698"
    selfLink: /apis/extensions/v1beta1/namespaces/default/deployments/airflow-worker
    uid: 6f196e83-39ae-11e9-bf53-xxxxxxxxxxxx
  spec:
    progressDeadlineSeconds: 2147483647
    replicas: 3
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        run: airflow-worker
    strategy:
      rollingUpdate:
        maxSurge: 1
        maxUnavailable: 1
      type: RollingUpdate
    template:
      metadata:
        creationTimestamp: null
        labels:
          run: airflow-worker
      spec:
        containers:
        - args:
          - worker
          env:
          - name: CLOUDSDK_METRICS_ENVIRONMENT
            value: 1.10.1-composer
          - name: GCS_BUCKET
            value: asia-northeast1-settings-te-xxxxxxxx-bucket
          - name: GCP_PROJECT
            value: xxxxxxxxx
          - name: COMPOSER_LOCATION
            value: asia-northeast1
          - name: COMPOSER_GKE_ZONE
            value: asia-northeast1-c
          - name: COMPOSER_GKE_NAME
            value: asia-northeast1-settings-te-xxxxxxxx-gke
          - name: COMPOSER_PYTHON_VERSION
            value: "2"
          - name: COMPOSER_ENVIRONMENT
            value: settings-test
          - name: AIRFLOW_HOME
            value: /etc/airflow
          - name: DAGS_FOLDER
            value: /home/airflow/gcs/dags
          - name: SQL_DATABASE
            value: airflow-db
          - name: SQL_USER
            value: root
          - name: SQL_PASSWORD
          - name: AIRFLOW__WEBSERVER__BASE_URL
            value: https://nf2d2247d4fdcdaed-tp.appspot.com
          - name: AIRFLOW__CORE__SQL_ALCHEMY_CONN
            value: mysql+mysqldb://$(SQL_USER):$(SQL_PASSWORD)@airflow-sqlproxy-service/$(SQL_DATABASE)
          - name: C_FORCE_ROOT
            value: "TRUE"
          image: gcr.io/cloud-airflow-releaser/airflow-worker-scheduler-1.10.1:cloud_composer_service_2019-02-05-RC4
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - pkill ForkPoolWorker
                - ;
                - kill
                - "1"
          livenessProbe:
            exec:
              command:
              - python
              - /var/local/worker_checker.py
            failureThreshold: 3
            initialDelaySeconds: 120
            periodSeconds: 60
            successThreshold: 1
            timeoutSeconds: 1
          name: airflow-worker
          ports:
          - containerPort: 8793
            protocol: TCP
          resources: {}
          securityContext:
            privileged: true
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
          - mountPath: /etc/airflow/airflow_cfg
            name: airflow-config
          - mountPath: /home/airflow/gcs
            name: gcsdir
        - args:
          - /home/airflow/gcs
          env:
          - name: GCS_BUCKET
            value: asia-northeast1-settings-te-xxxxxxxx-bucket
          image: gcr.io/cloud-airflow-releaser/gcs-syncd:cloud_composer_service_2019-02-05-RC4
          imagePullPolicy: IfNotPresent
          name: gcs-syncd
          resources:
            requests:
              cpu: 10m
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
          - mountPath: /home/airflow/gcs
            name: gcsdir
        dnsPolicy: ClusterFirst
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        terminationGracePeriodSeconds: 30
        volumes:
        - configMap:
            defaultMode: 420
            name: airflow-configmap
          name: airflow-config
        - emptyDir: {}
          name: gcsdir
  status:
    availableReplicas: 3
    conditions:
    - lastTransitionTime: "2019-02-26T10:10:30Z"
      lastUpdateTime: "2019-02-26T10:10:30Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    observedGeneration: 1
    readyReplicas: 3
    replicas: 3
    updatedReplicas: 3
kind: List
metadata:
  resourceVersion: ""
  selfLink: ""

service.yaml

apiVersion: v1
items:
- apiVersion: v1
  kind: Service
  metadata:
    creationTimestamp: "2019-02-26T10:08:16Z"
    labels:
      run: airflow-redis
    name: airflow-redis-service
    namespace: default
    resourceVersion: "1264"
    selfLink: /api/v1/namespaces/default/services/airflow-redis-service
    uid: 6f326dea-39ae-11e9-bf53-xxxxxxxxxxxx
  spec:
    clusterIP: xxx.xxx.xxx.xxx
    ports:
    - port: 6379
      protocol: TCP
      targetPort: 6379
    selector:
      run: airflow-redis
    sessionAffinity: None
    type: ClusterIP
  status:
    loadBalancer: {}
- apiVersion: v1
  kind: Service
  metadata:
    creationTimestamp: "2019-02-26T10:08:16Z"
    labels:
      run: airflow-sqlproxy
    name: airflow-sqlproxy-service
    namespace: default
    resourceVersion: "1278"
    selfLink: /api/v1/namespaces/default/services/airflow-sqlproxy-service
    uid: 6f454453-39ae-11e9-bf53-xxxxxxxxxxxx
  spec:
    clusterIP: xxx.xxx.xxx.xxx
    ports:
    - port: 3306
      protocol: TCP
      targetPort: 3306
    selector:
      run: airflow-sqlproxy
    sessionAffinity: None
    type: ClusterIP
  status:
    loadBalancer: {}
- apiVersion: v1
  kind: Service
  metadata:
    creationTimestamp: "2019-02-26T10:01:05Z"
    labels:
      component: apiserver
      provider: kubernetes
    name: kubernetes
    namespace: default
    resourceVersion: "6"
    selfLink: /api/v1/namespaces/default/services/kubernetes
    uid: 6e81176c-39ad-11e9-9eef-42010a920014
  spec:
    clusterIP: xxx.xxx.xxx.xxx
    ports:
    - name: https
      port: 443
      protocol: TCP
      targetPort: 443
    sessionAffinity: None
    type: ClusterIP
  status:
    loadBalancer: {}
kind: List
metadata:
  resourceVersion: ""
  selfLink: ""

statefulset.yaml

apiVersion: v1
items:
- apiVersion: apps/v1
  kind: StatefulSet
  metadata:
    creationTimestamp: "2019-02-27T02:32:11Z"
    generation: 1
    labels:
      run: airflow-redis
    name: airflow-redis
    namespace: default
    resourceVersion: "1643"
    selfLink: /apis/apps/v1/namespaces/default/statefulsets/airflow-redis
    uid: e2904c7f-3a37-11e9-b2b8-42010a92013a
  spec:
    podManagementPolicy: OrderedReady
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        run: airflow-redis
    serviceName: airflow-redis-service
    template:
      metadata:
        creationTimestamp: null
        labels:
          run: airflow-redis
      spec:
        containers:
        - command:
          - redis-server
          - --
          - save
          - "60"
          - "1"
          image: gcr.io/cloud-airflow-releaser/redis-5.0.3:cloud_composer_service_2019-02-05-RC4
          imagePullPolicy: IfNotPresent
          name: airflow-redis
          ports:
          - containerPort: 6379
            protocol: TCP
          resources: {}
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
          - mountPath: /data
            name: airflow-redis-volume
        dnsPolicy: ClusterFirst
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        terminationGracePeriodSeconds: 60
    updateStrategy:
      type: RollingUpdate
    volumeClaimTemplates:
    - metadata:
        creationTimestamp: null
        name: airflow-redis-volume
      spec:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: 2Gi
      status:
        phase: Pending
  status:
    collisionCount: 0
    currentReplicas: 1
    currentRevision: airflow-redis-55bb5cc5cf
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updateRevision: airflow-redis-55bb5cc5cf
    updatedReplicas: 1
kind: List
metadata:
  resourceVersion: ""
  selfLink: ""

• 以下のコマンドで出力

kubectl get Deployment -o yaml > deployment.yml
kubectl get Service -o yaml > service.yml
kubectl get Daemonset -o yaml > daemonset.yml
kubectl get Statefulset -o yaml > statefullset.yml