cluster-bug-reproduction.yaml

addons:
  rescheduler:
    enabled: false
clusterName: etcd-fail
controller:
  autoScalingGroup:
    maxSize: 3
    minSize: 1
    rollingUpdateMinInstancesInService: 2
  count: 1
  createTimeout: PT15M
  instanceType: m3.medium
  loadBalancer:
    private: false
    subnets:
    - name: Existing-Public-A
    - name: Existing-Public-B
    - name: Existing-Public-D
    - name: Existing-Public-E
  managedIamRoleName: K8sMaster
  rootVolume:
    iops: 0
    size: 30
    type: gp2
  subnets:
  - name: Existing-Private-A
  - name: Existing-Private-B
  - name: Existing-Private-D
  - name: Existing-Private-E
createRecordSet: true
etcd:
  count: 3
  customFiles:
    - path: "/etc/td-agent/td-agent.conf"
      permissions: 0644
      owner: "root"
      content: |
        <match fluent.**>
          type null
        </match>
        <source>
          @type forward
          port  24224
        </source>
        <source>
         @type monitor_agent
         bind 0.0.0.0
         port 24220
        </source>
        
        <source>
          @type systemd
          path /var/log/journal
          pos_file /var/log/fluentd-systemd-journal.pos
          tag systemd
          strip_underscores true
          read_from_head true
        </source>
        
        <filter docker.**>
          @type record_transformer
          <record>
            hostname ${hostname}
          </record>
        </filter>
        
        <filter docker.**>
          @type grep
          exclude1 log ^docker login.*$
        </filter>
        
        <match systemd>
          @type rewrite_tag_filter
          rewriterule1 SYSTEMD_UNIT   ^(.+).service$  systemd.$1
          rewriterule2 SYSTEMD_UNIT   !^(.+).service$ systemd.unmatched
        </match>
        
        <filter systemd.etcd-member>
          type parser
          format /^(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6,}) (?<severity>\w+) \| (?<package>\w+): (?<message>.*)$/
          time_format %Y-%m-%d %H:%M:%S.%N
          reserve_data true
          key_name MESSAGE
          suppress_parse_error_log false
        </filter>
        
        <filter systemd.docker>
          type parser
          format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
          reserve_data true
          key_name MESSAGE
          suppress_parse_error_log false
        </filter>
        
        <match **>
          type elasticsearch
          log_level info
          include_tag_key true
          hosts "https://#{ENV['ES_HOST']}:#{ENV['ES_PORT']}"
          scheme https
          ssl_verify false
          logstash_format true
          # Set the chunk limit the same as for fluentd-gcp.
          buffer_chunk_limit 2M
          # Cap buffer memory usage to 2MiB/chunk * 32 chunks = 64 MiB
          buffer_queue_limit 32
          flush_interval 5s
          # Never wait longer than 5 minutes between retries.
          max_retry_wait 30
          # Disable the limit on the number of retries (retry forever).
          disable_retry_limit
          # Use multiple threads for processing.
          num_threads 8
        </match>

    - path: "/etc/systemd/system/docker.service.d/20-docker-logstash-driver.conf"
      permissions: 0644
      owner: "root"
      content: |
        [Service]
        EnvironmentFile=-/etc/default/docker-daemon
    - path: "/etc/default/docker-daemon"
      permissions: 0644
      owner: "root"
      # fluentd-async-connect is important so as to not block containers from starting if logging somehow fails
      # OR: if td-agent is not yet up and running on "fluentd-address" host:port
      content: |
        DOCKER_OPTS='--log-driver=fluentd --log-opt fluentd-async-connect=true --log-opt fluentd-address=127.0.0.1:24224 --log-opt tag=docker.\{\{.Name\}\}.\{\{.FullID\}\}'
    - path: "/etc/docker-services"
      permissions: 0644
      owner: "root"
      content: |
        FLUENTD_TD_AGENT_IMAGE='gcr.io/google_containers/fluentd-elasticsearch:1.23'
#### If you want logs from FluentD shipped, fill in your ES_HOST and ES_PORT here:
#### If you do not want logs, then just remove the custom SystemD and cloud-init files
    - path: "/etc/default/td-agent"
      permissions: 0600
      owner: "root"
      content: |
        ES_HOST=your-elasticsearch-host.example.com
        ES_PORT=9200
    path: /etc/default/td-agent
    permissions: 384
  customSystemdUnits:
    - name: td-agent.service
      command: start
      enable: true
      content: |
        [Unit]
        Description=FluentD td-agent logger service (logs to ElasticSearch)
        Documentation=http://docs.fluentd.org/v0.12/articles/config-file
        After=docker.service
        [Service]
        TimeoutStartSec=infinity
        Restart=always
        EnvironmentFile=/etc/docker-services
        ExecStartPre=-/usr/bin/docker stop %p
        ExecStartPre=-/usr/bin/docker rm %p
        ExecStartPre=-/usr/bin/docker pull $FLUENTD_TD_AGENT_IMAGE
        ExecStart=/usr/bin/docker run --rm --name %p \
          -v /var/lib/docker/containers:/var/lib/docker/containers \
          -v /var/log:/var/log \
          -v /etc/td-agent:/etc/td-agent:ro  \
          -p 24220:24220 -p 24224:24224 -p 24224:24224/udp \
          --env-file=/etc/default/td-agent \
          --entrypoint=/bin/bash \
          --hostname="%H" \
          --health-cmd "curl -f http://localhost:24220/api/plugins.json" \
          --health-interval 10s \
          --health-timeout 1s \
          --health-retries 3 \
          $FLUENTD_TD_AGENT_IMAGE -c 'td-agent-gem install fluent-plugin-systemd -v 0.0.8 ;  td-agent'
        ExecStop=-/usr/bin/docker stop %p
        [Install]
        WantedBy=multi-user.target

  dataVolume:
    encrypted: true
    ephemeral: false
    iops: 0
    size: 30
    type: gp2
  disasterRecovery:
    automated: true
  instanceType: m3.medium
  memberIdentityProvider: eip
  rootVolume:
    iops: 0
    size: 30
    type: gp2
  snapshot:
    automated: true
  subnets:
  - name: Existing-Private-A
  - name: Existing-Private-B
  - name: Existing-Private-D
  - name: Existing-Private-E
  tenancy: default
  version: 3.1.3
experimental:
  awsNodeLabels:
    enabled: true
  clusterAutoscalerSupport:
    enabled: true
  kube2IamSupport:
    enabled: true
  nodeDrainer:
    enabled: true
  taints:
  - effect: NoSchedule
    key: dedicated
    value: search
  tlsBootstrap:
    enabled: true
#### Replace with your info
externalDNSName: etcd-fail.example.com
hostedZoneId: ZZAABBCCDDEE1
keyName: your-keyname
kmsKeyArn: arn:aws:kms:<region>:<ACCOUNT_ID>:key/<KMS_KEY_ID>

kubeResourcesAutosave:
  enabled: true
kubernetesVersion: v1.6.1_coreos.0
mapPublicIPs: false
recordSetTTL: 300
region: us-east-1
releaseChannel: stable

#### Your routeTableId here:
routeTableId: rtb-aabbccdd
sshAuthorizedKeys:
- <YOUR SSH PUBKEY HERE>
stackTags:
  Environment: Dev
  Name: Etcd-broken
subnets:
#### YOUR Pre-existing subnet IDs here
- availabilityZone: us-east-1a
  id: subnet-aaaaaaaa
  name: Existing-Public-A
- availabilityZone: us-east-1b
  id: subnet-bbbbbbbb
  name: Existing-Public-B
- availabilityZone: us-east-1d
  id: subnet-cccccccc
  name: Existing-Public-D
- availabilityZone: us-east-1e
  id: subnet-dddddddd
  name: Existing-Public-E
- availabilityZone: us-east-1a
  id: subnet-aaaapriv
  name: Existing-Private-A
- availabilityZone: us-east-1b
  id: subnet-bbbbpriv
  name: Existing-Private-B
- availabilityZone: us-east-1d
  id: subnet-ddddpriv
  name: Existing-Private-D
- availabilityZone: us-east-1e
  id: subnet-eeeepriv
  name: Existing-Private-E
tlsCADurationDays: 5475
tlsCertDurationDays: 3650
useCalico: false
### Your Pre-existing VPC CIDR here
vpcCIDR: 10.100.2.0/16
### Your Pre-existing VPC ID here
vpcId: vpc-12345678
worker:
  nodePools:
  - autoScalingGroup:
      maxSize: 20
      minSize: 5
      rollingUpdateMinInstancesInService: 5
    awsNodeLabels:
      enabled: true
    clusterAutoscalerSupport:
      enabled: true
    instanceType: c4.large
    kube2IamSupport:
      enabled: true
    loadBalancer:
      enabled: true
    managedIamRoleName: K8sWorker
    name: c4-large
    nodeDrainer:
      enabled: true
    rootVolume:
      iops: 0
      size: 30
      type: gp2
    subnets:
    - name: Existing-Private-A
    - name: Existing-Private-B
    - name: Existing-Private-D
    - name: Existing-Private-E
  - autoScalingGroup:
      maxSize: 20
      minSize: 4
      rollingUpdateMinInstancesInService: 4
    awsNodeLabels:
      enabled: true
    clusterAutoscalerSupport:
      enabled: true
      kube2IamSupport:
        enabled: true
    instanceType: m4.2xlarge
    managedIamRoleName: K8sWorker
    name: m4-2xlarge
    nodeDrainer:
      enabled: true
    rootVolume:
      iops: 0
      size: 30
      type: gp2
    subnets:
    - name: Existing-Private-A
    - name: Existing-Private-B
    - name: Existing-Private-D
    - name: Existing-Private-E