Scale Testing Resources

00-README.md

My tests and test files

Content:

• scale-test.sh
• glooResources.yaml
• appResources.yaml

Notes:

• hardcoded cluster names (mgmt,worker-1,worker-2)
• replicas set to 0 by default
• no global workspace
• gateways-namespace destination rule for cross cluster fix

Client tests and test files

Content

• client-test.sh
• client-gloo.yaml
• client-workspace.yaml
• client-app.yaml

Notes:

• can enable/disable datadog annotations
• hardcoded cluster awscmhqa2
• some resources labelled with gloo_load_test: "true"

appResources.yaml

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: frontend-$num
  namespace: client-namespace-$num
  labels:
    account: frontend
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: frontend-$num
  namespace: client-namespace-$num
  labels:
    app: frontend-$num
    version: v1
spec:
  replicas: 0
  selector:
    matchLabels:
      app: frontend-$num
      version: v1
  template:
    metadata:
      labels:
        app: frontend-$num
        version: v1
    spec:
      securityContext:
        runAsUser: 1000
      serviceAccountName: frontend-$num
      containers:
      - name: frontend
        image: gcr.io/istio-release/app:1.13.7
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 8080
        command:
        - "/bin/sh"
        - "-c"
        - "--"
        args:
        - "while true; do curl -I -s http://echo-$num.global:8080; sleep 1; done"
---
apiVersion: v1
kind: Service
metadata:
  name: backend-$num
  namespace: server-namespace-$num
  labels:
    app: backend-$num
    service: backend-$num
spec:
  ports:
  - port: 8080
    name: http
  - port: 9080
    name: grpc
  selector:
    app: backend-$num
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: backend-$num
  namespace: server-namespace-$num
  labels:
    account: backend
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: backend-$num
  namespace: server-namespace-$num
  labels:
    app: backend-$num
    version: v1
spec:
  replicas: 0
  selector:
    matchLabels:
      app: backend-$num
      version: v1
  template:
    metadata:
      labels:
        app: backend-$num
        version: v1
    spec:
      securityContext:
        runAsUser: 1000
      serviceAccountName: backend-$num
      containers:
      - name: backend
        image: gcr.io/istio-release/app:1.13.7
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 8080
        args:
          - --port
          - "8080"
          - --grpc
          - "9080"
          - --version
          - v1
          - --cluster
          - $cluster
---
apiVersion: networking.istio.io/v1alpha3
kind: DestinationRule
metadata:
  name: echo-$num
  namespace: istio-gateways
spec:
  host: echo-$num.global
  trafficPolicy:
    loadBalancer:
      localityLbSetting:
        enabled: true
        failoverPriority:
        - "topology.istio.io/network"
        - "topology.kubernetes.io/region"
        - "topology.kubernetes.io/zone"
        - "topology.istio.io/subzone"
    outlierDetection:
      consecutive5xxErrors: 5
      interval: 1m
      baseEjectionTime: 30s

client-app.yaml

---
apiVersion: v1
kind: Namespace
metadata:
  name: gloo-poc-loadtest-server--${NUM}
  labels:
    gloo_load_test: "true"
    istio.io/rev: default
---
apiVersion: v1
kind: Namespace
metadata:
  name: gloo-poc-loadtest-client--${NUM}
  labels:
    gloo_load_test: "true"
    istio.io/rev: default
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: gloo-poc-loadtest-server--${NUM}
  namespace: gloo-poc-loadtest-server--${NUM}
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: gloo-poc-loadtest-client--${NUM}
  namespace: gloo-poc-loadtest-client--${NUM}
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: server
  namespace: gloo-poc-loadtest-server--${NUM}
  labels:
    app: server
    version: v1
spec:
  replicas: 1
  selector:
    matchLabels:
      app: server
      version: v1
  template:
    metadata:
      labels:
        app: server
        version: v1
    spec:
      serviceAccountName: gloo-poc-loadtest-server--${NUM}
      securityContext:
        runAsUser: 65534
        runAsGroup: 65534
      containers:
      - name: main
        image: gcr.io/istio-release/app:1.14.4
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 8080
        - containerPort: 8083
        - containerPort: 8084
        args:
          - --port
          - "8080"
          - --grpc
          - "8083"
          - --version
          - v1
          - --cluster
          - ${CLUSTER}
        resources:
          requests:
            cpu: '100m'
            memory: '128Mi'
            ephemeral-storage: 1Gi
          limits:
            cpu: '500m'
            memory: '128Mi'
            ephemeral-storage: 1Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: client
  namespace: gloo-poc-loadtest-client--${NUM}
  labels:
    app: client
    version: v1
spec:
  replicas: 1
  selector:
    matchLabels:
      app: client
      version: v1
  template:
    metadata:
      labels:
        app: client
        version: v1
${DD_ANNOTATIONS}
    spec:
      serviceAccountName: gloo-poc-loadtest-client--${NUM}
      securityContext:
        runAsUser: 65534
        runAsGroup: 65534
      containers:
      - name: main
        image: gcr.io/istio-release/app:1.14.4
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 8080
        - containerPort: 8083
        - containerPort: 8084
        command:
          - '/bin/bash'
          - '-c'
        args:
          - 'while true; do client ${URL} --log_rotate_max_size 512 ; sleep 1; done && exit -1'
        resources:
          requests:
            cpu: '100m'
            memory: '128Mi'
            ephemeral-storage: 2Gi
          limits:
            cpu: '500m'
            memory: '128Mi'
            ephemeral-storage: 2Gi
---
apiVersion: v1
kind: Service
metadata:
  name: server
  namespace: gloo-poc-loadtest-server--${NUM}
  labels:
    app: server
    service: server
spec:
  ports:
  - port: 8080
    name: http
  - port: 8083
    name: grpc
  - port: 8084
    name: egress
  selector:
    app: server
    version: v1
  type: ClusterIP
  clusterIP: None

client-gloo.yaml

# Routes
---
apiVersion: networking.gloo.solo.io/v2
kind: VirtualDestination
metadata:
  name: server
  namespace: gloo-poc-loadtest-server--${NUM}
spec:
  hosts:
  - infra.echo-${NUM}.indeed.mesh
  ports:
  - number: 8080
    protocol: HTTP
    targetPort:
      name: http
  - number: 8083
    protocol: GRPC
    targetPort:
      name: grpc
  - number: 8084
    protocol: HTTP
    targetPort:
      name: egress
  services:
  - namespace: gloo-poc-loadtest-server--${NUM}
    name: server
---
apiVersion: networking.gloo.solo.io/v2
kind: RouteTable
metadata:
  name: server
  namespace: gloo-poc-loadtest-server--${NUM}
spec:
  hosts:
  - 'infra.echo-${NUM}.indeed.mesh'
  workloadSelectors:
    - selector:
        workspace: gloo-poc-loadtest-server--${NUM}
  http:
  - forwardTo:
      destinations:
      - kind: VIRTUAL_DESTINATION
        port:
          number: 8083
        ref:
          name:  server
          namespace: gloo-poc-loadtest-server--${NUM}
          cluster: awscmhqa2
    matchers:
      - headers:
          - name: content-type
            value: application/grpc
    name: gloo-poc-loadtest-server--${NUM}-grpc
  - forwardTo:
      destinations:
        - kind: VIRTUAL_DESTINATION
          port:
            number: 8080
          ref:
            name: server
            namespace: gloo-poc-loadtest-server--${NUM}
            cluster: awscmhqa2
#    matchers:
#      - headers:
#          - name: content-type
#            value: application/grpc
#            invertMatch: true
    name: gloo-poc-loadtest-server--${NUM}-http
  virtualGateways: []
---
apiVersion: resilience.policy.gloo.solo.io/v2
kind: FailoverPolicy
metadata:
  name: server
  namespace: gloo-poc-loadtest-server--${NUM}
spec:
  applyToDestinations:
    - kind: VIRTUAL_DESTINATION
      selector:
        namespace: gloo-poc-loadtest-server--${NUM}
  config:
    # enable default locality based load balancing
    localityMappings: []
---
apiVersion: resilience.policy.gloo.solo.io/v2
kind: OutlierDetectionPolicy
metadata:
  name: server
  namespace: gloo-poc-loadtest-server--${NUM}
spec:
  applyToDestinations:
    - kind: VIRTUAL_DESTINATION
      selector:
        namespace: gloo-poc-loadtest-server--${NUM}
  config:
    consecutiveErrors: 2
    interval: 5s
    baseEjectionTime: 15s
    maxEjectionPercent: 100

client-test.sh

#!/bin/bash

set -euo pipefail

DELETE=false
WORKSPACE=false
GLOO=false
APP=false
METRICS=false
START=$1; shift
END=$1; shift

while [[ "$#" -gt 0 ]]; do
    case $1 in
        -d) DELETE=true ;;
        -w) WORKSPACE=true ;;
        -g) GLOO=true ;;
        -a) APP=true ;;
        -m) METRICS=true ;;
        *) echo "Unknown parameter passed: $1"; exit 1 ;;
    esac
    shift
done

CMD="apply"
if [[ ${DELETE} == true ]]; then
    CMD="delete"
fi

DD_ANNOTATIONS=$(cat <<EOF
      annotations:
        ad.datadoghq.com/istio-proxy.check_names: '["openmetrics"]'
        ad.datadoghq.com/istio-proxy.init_configs: '[{}]'
        ad.datadoghq.com/istio-proxy.instances: '[{"openmetrics_endpoint": "http://%%host%%:%%port%%/stats/prometheus", "metrics": ["istio_request_duration_milliseconds.*"], "histogram_buckets_as_distributions": true, "tag_by_endpoint": false, "exclude_labels": ["source_principal", "destination_principal", "source_workload_namespace", "destination_workload_namespace", "destination_service_namespace"], "ignore_tags": ["kube_replica_set:.*", "security-group:.*", "machinename:.*", "name:.*"]}]'
EOF
)

if [[ ${METRICS} == true ]]; then
    export DD_ANNOTATIONS
fi

for i in $(seq $START $END);
do
    echo $i
    export NUM=$i
    export URL="http://infra.echo-${NUM}.indeed.mesh:8084"
    if [[ $((i%2)) -eq 0 ]]; then
        export URL="grpc://infra.echo-${NUM}.indeed.mesh:8084"
    fi
    if [[ ${WORKSPACE} == true ]]; then
        envsubst < workspace.yaml | kubectl --context awscmhqa2 ${CMD} --wait=false -f - || true
    fi
    if [[ ${GLOO} == true ]]; then
        envsubst < gloo.yaml | kubectl --context awscmhqa2 ${CMD} --wait=false -f - || true
    fi
    if [[ ${APP} == true ]]; then
        CLUSTER="awscmhdev" envsubst < app.yaml | kubectl --context awscmhdev ${CMD} --wait=false -f - || true
        CLUSTER="awscmhdev2" envsubst < app.yaml | kubectl --context awscmhdev2 ${CMD} --wait=false -f - || true
    fi
done

client-workspace.yaml

# Server
---
apiVersion: v1
kind: Namespace
metadata:
  name: gloo-poc-loadtest-server--${NUM}
  labels:
    gloo_load_test: "true"
---
apiVersion: admin.gloo.solo.io/v2
kind: Workspace
metadata:
  name: gloo-poc-loadtest-server--${NUM}
  namespace: gloo-mesh
  labels:
    gloo_load_test: "true"
spec:
  workloadClusters:
  - name: 'awscmhqa2'
    namespaces:
    - name: 'gloo-poc-loadtest-server--${NUM}'
  - name: '*'
    namespaces:
    - name: 'gloo-poc-loadtest-server--${NUM}'
---
apiVersion: admin.gloo.solo.io/v2
kind: WorkspaceSettings
metadata:
  name: gloo-poc-loadtest-server--${NUM}
  namespace: gloo-poc-loadtest-server--${NUM}
spec:
  exportTo:
  - workspaces:
    - name: gloo-poc-loadtest-client--${NUM}
  importFrom:
  - workspaces:
    - name: istio-gateways
# Client
---
apiVersion: v1
kind: Namespace
metadata:
  name: gloo-poc-loadtest-client--${NUM}
  labels:
    gloo_load_test: "true"
---
apiVersion: admin.gloo.solo.io/v2
kind: Workspace
metadata:
  name: gloo-poc-loadtest-client--${NUM}
  namespace: gloo-mesh
  labels:
    gloo_load_test: "true"
spec:
  workloadClusters:
  - name: 'awscmhqa2'
    namespaces:
    - name: 'gloo-poc-loadtest-client--${NUM}'
  - name: '*'
    namespaces:
    - name: 'gloo-poc-loadtest-client--${NUM}'
---
apiVersion: admin.gloo.solo.io/v2
kind: WorkspaceSettings
metadata:
  name: gloo-poc-loadtest-client--${NUM}
  namespace: gloo-poc-loadtest-client--${NUM}
  labels:
    gloo_load_test: "true"
spec:
  importFrom:
  - workspaces:
    - name: istio-gateways
    - name: gloo-poc-loadtest-server--${NUM}

glooResources.yaml

---
apiVersion: admin.gloo.solo.io/v2
kind: Workspace
metadata:
  name: client-workspace-$num
  namespace: gloo-mesh
spec:
  workloadClusters:
  - name: '*'
    namespaces:
    - name: client-namespace-$num
---
apiVersion: admin.gloo.solo.io/v2
kind: Workspace
metadata:
  name: server-workspace-$num
  namespace: gloo-mesh
spec:
  workloadClusters:
  - name: '*'
    namespaces:
    - name: server-namespace-$num
---
apiVersion: admin.gloo.solo.io/v2
kind: WorkspaceSettings
metadata:
  name: client-workspace-$num
  namespace: client-namespace-$num
spec:
  importFrom:
  - workspaces:
    - name: server-workspace-$num
  options:
    eastWestGateways:
    - selector:
        labels:
          istio: eastwestgateway
    federation:
      enabled: false
    serviceIsolation:
      enabled: false
---
apiVersion: admin.gloo.solo.io/v2
kind: WorkspaceSettings
metadata:
  name: server-workspace-$num
  namespace: server-namespace-$num
spec:
  exportTo:
  - workspaces:
    - name: client-workspace-$num
  options:
    eastWestGateways:
    - selector:
        labels:
          istio: eastwestgateway
    federation:
      enabled: false
    serviceIsolation:
      enabled: false
---
apiVersion: networking.gloo.solo.io/v2
kind: VirtualDestination
metadata:
  name: echo-$num
  namespace: server-namespace-$num
spec:
  hosts:
  - echo-$num.global
  ports:
  - number: 8080
    protocol: HTTP
    targetPort:
      number: 8080
  - number: 9080
    protocol: GRPC
    targetPort:
      number: 9080
  services:
  - cluster: worker-1
    name: backend-$num
    namespace: server-namespace-$num
  - cluster: worker-2
    name: backend-$num
    namespace: server-namespace-$num
---
apiVersion: networking.gloo.solo.io/v2
kind: RouteTable
metadata:
  name: echo-$num
  namespace: server-namespace-$num
spec:
  hosts:
  - echo-$num.global
  http:
  - forwardTo:
      destinations:
      - kind: VIRTUAL_DESTINATION
        port:
          number: 8080
        ref:
          name: echo-$num
          namespace: server-namespace-$num
    labels:
      protocol: http
    matchers:
    - port: 8080
    name: echo-http
  - forwardTo:
      destinations:
      - kind: VIRTUAL_DESTINATION
        port:
          number: 9080
        ref:
          name: echo-$num
          namespace: server-namespace-$num
    labels:
      protocol: grpc
    matchers:
    - port: 9080
    name: echo-grpc

scale-test.sh

#!/bin/bash
operation=create
start=0
finish=10
glooResources=./glooResources.yaml
appResources=./appResources.yaml

usage () {
  echo "USAGE: $0 [--operation] [--context] [--start] [--finish] [--resources]"
  echo "  [-o|--operation kube-operation] kubectl operation (e.g. apply,delete,create)"
  echo "  [-s|--start start-number] starting index"
  echo "  [-f|--finish finish-number] finishing index"
  echo "  [-g|--glooResources path] path to gloo resource template file"
  echo "  [-a|--appResources path] path to app resource template file"
  echo "  [-h|--help] Usage message"
}

POSITIONAL=()
while [[ $# -gt 0 ]]; do
    key="$1"
    case $key in
        -o|--operation)
        operation="$2"
        shift # past argument
        shift # past value
        ;;
        -s|--start)
        start="$2"
        shift # past argument
        shift # past value
        ;;
        -f|--finish)
        finish="$2"
        shift # past argument
        shift # past value
        ;;
        -g|--glooResources)
        glooResources="$2"
        shift # past argument
        shift # past value
        ;;
        -a|--appResources)
        appResources="$2"
        shift # past argument
        shift # past value
        ;;
        -h|--help)
        help="true"
        shift
        ;;
        *)
        usage
        exit 1
        ;;
    esac
done

if [[ $help ]]; then
    usage
    exit 0
fi

for i in $(seq $start $finish)
do
  export num=$(printf %03d $i)
  for cluster in mgmt worker-1 worker-2
  do
    kubectl --context $cluster $operation -f - <<-EOF
                        apiVersion: v1
                        kind: Namespace
                        metadata:
                          labels:
                            istio.io/rev: default
                          name: client-namespace-$num
                        EOF
    kubectl --context $cluster $operation -f - <<-EOF
                        apiVersion: v1
                        kind: Namespace
                        metadata:
                          labels:
                            istio.io/rev: default
                          name: server-namespace-$num
                        EOF
  done
  for cluster in mgmt
  do
    envsubst < $glooResources | kubectl --context $cluster $operation -f -
  done
  for cluster in worker-1 worker-2
  do
    export cluster
    envsubst < $appResources | kubectl --context $cluster $operation -f -
  done

done