nomad-consul-production-minmal-setup-part2.md

Practice Consul, Nomad in Production Part 2 - Setup DNS and Make Basic Jobs

Complete the 1st part before you go on SET UP CLUSTER

DEPLOY TRAEFIK

1. Create service Policy

cd ~ &&  echo "
service \"traefik\" {
        policy = \"write\"
}
service \"traefik-sidecar-proxy\" {
        policy = \"write\"
}
service_prefix \"\" {
        policy = \"read\"
}
node_prefix \"\" {
        policy = \"read\"
}
"| tee traefik-policy.hcl

2. Deploy and create token from policy

consul acl policy create -name "traefik-service" -description "Traefik Proxy Service Access Policy" -rules @traefik-policy.hcl

consul acl token create -description "Traefik Service Token" -policy-name "traefik-service"| tee traefik-service.token

##### OUTPUT #####
AccessorID:       53078cee-9f5d-e78d-7928-8850b9934485
SecretID:         aae2f550-a21c-b20f-90ab-f939e1d35536 <=================== Save the key for deployment
Description:      Traefik Service Token
Local:            false
Create Time:      2022-11-21 15:35:03.275568004 +0700 +07
Policies:
   d0a1ca0f-ee56-b0bd-dbb4-80f661ce393c - traefik-service

3. Deloy nomad job with describe

job "proxy" {
  datacenters = ["saigon"]
  type        = "system"

  group "traefik" {
    count = 1

    network {
      port "http" {
        static = 8888
      }
      port "admin" {
        static = 9999
      }
    }

    service {
      name     = "traefik"
      provider = "consul"
      port     = "http"
      check {
        name     = "alive"
        type     = "tcp"
        port     = "http"
        interval = "10s"
        timeout  = "2s"
      }
    }

    task "traefik" {
      driver = "docker"
      constraint {
        attribute = "${node.class}"
        operator  = "="
        value     = "core"
      }
      config {
        image        = "traefik:2.9"
        ports        = ["admin", "http"]
        network_mode = "host"
        args = [
          "--log.level=DEBUG",
          "--api.dashboard=true",
          "--api.insecure=true",
          "--entrypoints.web.address=:${NOMAD_PORT_http}",
          "--entrypoints.traefik.address=:${NOMAD_PORT_admin}",
          "--providers.consulCatalog.exposedByDefault=true",
          "--providers.consulCatalog.prefix=traefik",
          "--providers.consulcatalog.endpoint.address=10.238.22.182:8500",
          "--providers.consulcatalog.endpoint.token=aae2f550-a21c-b20f-90ab-f939e1d35536",
          "--providers.consulcatalog.endpoint.scheme=http"
        ]
      }
      resources {
        cpu        = 100
        memory     = 512
        memory_max = 2048
      }
    }
  }
}

5. Access via link

• http://10.238.22.122:9999
• http://10.238.22.182:9999
• http://10.238.22.50:9999

job "appservice" {
  datacenters = ["saigon"]
  type        = "service"
  group "demo-backend" {
    count = 3
    scaling {
      min     = 2
      max     = 5
      enabled = true
      policy {
        evaluation_interval = "2s"
        cooldown            = "5s"
        check "cpu_usage" {
          source = "prometheus"
          query  = "avg(nomad_client_allocs_cpu_total_percent{task='server'})"
          strategy "target-value" {
            target = 10
          }
        }
      }
    }
    task "server" {
      driver = "docker"
      config {
        image              = "tuyendev/nomad-app-demo:v1"
        image_pull_timeout = "10m"
        port_map {
          http = 8080
        }
      }
      resources {
        memory = 4096
        cpu    = 100
        network {
          mbits = 10
          port "http" {}
        }
      }
      service {
        name = "demo-backend"
        port = "http"
        tags = [
          "backend-demo",
          "traefik.enable=true",
          "traefik.http.routers.http.rule=Path(`/`)",
        ]
        check {
          type     = "http"
          path     = "/management/health"
          interval = "2s"
          timeout  = "2s"
        }
      }
    }
  }
}

job "prometheus" {
  datacenters = ["saigon"]
  type        = "system"

  group "monitoring" {
    count = 1

    network {
      port "prometheus_ui" {
        static = 9090
      }
    }

    task "prometheus" {
      driver = "docker"
      constraint {
        attribute = "${node.class}"
        operator  = "="
        value     = "monitor"
      }
      config {
        image        = "prom/prometheus:latest"
        ports        = ["prometheus_ui"]
        network_mode = "host"

        args = [
          "--config.file=/etc/prometheus/config/prometheus.yml",
          "--storage.tsdb.path=/prometheus",
          "--web.listen-address=0.0.0.0:${NOMAD_PORT_prometheus_ui}",
          "--web.console.libraries=/usr/share/prometheus/console_libraries",
          "--web.console.templates=/usr/share/prometheus/consoles",
        ]

        volumes = [
          "local/config:/etc/prometheus/config",
        ]
      }

      template {
        data = <<EOH
---
global:
  scrape_interval:     1s
  evaluation_interval: 1s

scrape_configs:
  - job_name: consul
    metrics_path: /v1/agent/metrics
    params:
      format: ['prometheus']
    static_configs:
    - targets: ['10.238.22.248:8500', '10.238.22.166:8500', '10.238.22.201:8500']

  - job_name: nomad
    metrics_path: /v1/metrics
    params:
      format: ['prometheus']
    static_configs:
    - targets: ['10.238.22.248:4646', '10.238.22.166:4646', '10.238.22.201:4646', 'sg-monitor-1.node.saigon.consul:4646']
  - job_name: 'service-backend'
    scrape_interval: 5s
    metrics_path: /management/prometheus
    params:
      format: ['prometheus']
    consul_sd_configs:
    - server: '10.238.22.248:8500'
      services:
        - 'demo-backend'
    - server: '10.238.22.166:8500'
      services:
        - 'demo-backend'
    - server: '10.238.22.201:8500'
      services:
        - 'demo-backend'
EOH

        change_mode   = "signal"
        change_signal = "SIGHUP"
        destination   = "local/config/prometheus.yml"
      }

      resources {
        cpu    = 100
        memory = 256
      }

      service {
        name = "prometheus"
        port = "prometheus_ui"

        check {
          type     = "http"
          path     = "/-/healthy"
          interval = "10s"
          timeout  = "2s"
        }
      }
    }
  }
}

job "autoscaler" {
  type = "service"

  datacenters = ["saigon"]

  group "autoscaler" {
    count = 1

    network {
      port "http" {}
      port "promtail" {}
    }

    task "autoscaler" {
      driver = "docker"
      constraint {
        attribute = "${node.class}"
        operator  = "="
        value     = "monitor"
      }
      config {
        image   = "hashicorp/nomad-autoscaler:0.3.7"
        command = "nomad-autoscaler"
        ports   = ["http"]

        args = [
          "agent",
          "-config",
          "${NOMAD_TASK_DIR}/config.hcl",
          "-http-bind-address",
          "0.0.0.0",
          "-http-bind-port",
          "${NOMAD_PORT_http}",
        ]
      }

      template {
        data = <<EOF
nomad {
  address = "http://10.238.22.201:4646"
  token = "dd63c934-dfb6-fea2-4821-ce8bdc503828"
}

telemetry {
  prometheus_metrics = true
  disable_hostname   = true
}

apm "prometheus" {
  driver = "prometheus"
  config = {
    address = "http://10.238.22.205:9090"
  }
}

strategy "target-value" {
  driver = "target-value"
}
          EOF

        destination = "${NOMAD_TASK_DIR}/config.hcl"
      }

      resources {
        cpu    = 50
        memory = 128
      }

      service {
        name = "autoscaler"
        port = "http"

        check {
          type     = "http"
          path     = "/v1/health"
          interval = "3s"
          timeout  = "1s"
        }
      }
    }

    task "promtail" {
      driver = "docker"

      lifecycle {
        hook    = "prestart"
        sidecar = true
      }

      config {
        image = "grafana/promtail:1.5.0"
        ports = ["promtail"]

        args = [
          "-config.file",
          "local/promtail.yaml",
        ]
      }

      template {
        data = <<EOH
server:
  http_listen_port: {{ env "NOMAD_PORT_promtail" }}
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

client:
  url: http://{{ range $i, $s := service "loki" }}{{ if eq $i 0 }}{{.Address}}:{{.Port}}{{end}}{{end}}/api/prom/push

scrape_configs:
- job_name: system
  entry_parser: raw
  static_configs:
  - targets:
      - localhost
    labels:
      task: autoscaler
      __path__: /alloc/logs/autoscaler*
  pipeline_stages:
  - match:
      selector: '{task="autoscaler"}'
      stages:
      - regex:
          expression: '.*policy_id=(?P<policy_id>[a-zA-Z0-9_-]+).*source=(?P<source>[a-zA-Z0-9_-]+).*strategy=(?P<strategy>[a-zA-Z0-9_-]+).*target=(?P<target>[a-zA-Z0-9_-]+).*Group:(?P<group>[a-zA-Z0-9]+).*Job:(?P<job>[a-zA-Z0-9_-]+).*Namespace:(?P<namespace>[a-zA-Z0-9_-]+)'
      - labels:
          policy_id:
          source:
          strategy:
          target:
          group:
          job:
          namespace:
EOH

        destination = "local/promtail.yaml"
      }

      resources {
        cpu    = 50
        memory = 32
      }

      service {
        name = "promtail"
        port = "promtail"

        check {
          type     = "http"
          path     = "/ready"
          interval = "10s"
          timeout  = "2s"
        }
      }
    }
  }
}