policy exception

gistfile1.txt

apiVersion: kyverno.io/v2
kind: PolicyException
metadata:
  name: toleration-exceptions
  namespace: kube-system
spec:
  background: true
  match:
    any:
    - kinds:
        - Pod
        - Deployment
      names:
        - kube-system
  exceptions:
    - policyName: all-pods-need-tolerations
      ruleNames:
        - check-pod-tolerations

      - name: Build and push image
        uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6
        with:
          context: ${{ env.BUILD_CONTEXT }}
          file: ${{ env.DOCKERFILE }}
          push: true
          provenance: false
          tags: |
            ${{ steps.meta.outputs.image }}:latest
            ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.tag_sha }}
            ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.tag_date }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

name: Invoke Airflow Customer Onboard Lambda

on:
  issue_comment:
    types: [created]

jobs:
  invoke_lambda:
    name: Trigger Airflow Customer Onboard Lambda Function
    if: startsWith(github.event.comment.body, 'deploy airflow_customer_onboard') && github.event.issue.pull_request
    runs-on: ubuntu-latest
    permissions:
      contents: read
      id-token: write
      pull-requests: write

    steps:
      - name: Checkout Repository
        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5

      - name: Extract Environment & Payload from Comment
        id: extract_env

        run: |
          COMMENT_BODY="${{ github.event.comment.body }}"
          # Extract ENV
          ENV=$(echo "$COMMENT_BODY" | grep -oE '\--env (dev|prod|test)' | awk '{print $2}')
          if [[ -z "$ENV" ]]; then
            echo "::error::No valid environment found in comment! Use --env dev/prod/test"
            exit 1
          fi
          # Extract JSON payload (everything after --payload, possibly quoted)
          PAYLOAD_JSON=$(echo "$COMMENT_BODY" | grep -oP '(?<=--payload )(\{.*\}|\'.*\'|".*")')
          if [[ -z "$PAYLOAD_JSON" ]]; then
            echo "::error::No JSON payload specified in the comment"
            exit 1
          fi
          echo "Environment detected: $ENV"
          echo "Payload detected: $PAYLOAD_JSON"
          echo "env=$ENV" >> "$GITHUB_ENV"
          echo "payload=$PAYLOAD_JSON" >> "$GITHUB_ENV"

Updated file

name: Invoke Airflow Customer Onboard Lambda

on:
  issue_comment:
    types: [created]

jobs:
  invoke_lambda:
    name: Trigger Airflow Customer Onboard Lambda Function
    if: startsWith(github.event.comment.body, 'deploy airflow_customer_onboard') && github.event.issue.pull_request
    runs-on: ubuntu-latest
    permissions:
      contents: read
      id-token: write
      pull-requests: write

    steps:
      - name: Checkout Repository
        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5

      - name: Extract Environment & Payload from Comment
        id: extract_env

        run: |
          COMMENT_BODY="${{ github.event.comment.body }}"
          # Extract ENV
          ENV=$(echo "$COMMENT_BODY" | grep -oE '\--env (dev|prod|test)' | awk '{print $2}')
          if [[ -z "$ENV" ]]; then
            echo "::error::No valid environment found in comment! Use --env dev/prod/test"
            exit 1
          fi
          # Extract JSON payload (everything after --payload, possibly quoted)
          PAYLOAD_JSON=$(echo "$COMMENT_BODY" | grep -oP ' --payload.*?$' | awk '{print $2}' )
          if [[ -z "$PAYLOAD_JSON" ]]; then
            echo "::error::No JSON payload specified in the comment"
            exit 1
          fi
          echo "Environment detected: $ENV"
          echo "Payload detected: $PAYLOAD_JSON"

          echo $PAYLOAD_JSON
          echo "$PAYLOAD_JSON"

        
          echo "env=$ENV" >> "$GITHUB_ENV"
          echo "payload=$PAYLOAD_JSON" >> "$GITHUB_ENV"

      - name: Read ENV and Payload from env
        id: extract_env_from_env

        run: |
          aws lambda invoke --function-name dev-lambda --payload '${{env.payload}}' --debug demo-outfile
          echo "data --payload ${{env.payload}}"
        shell: bash

Comment

deploy airflow_customer_onboard --env dev --payload {\"env\":\"dev\",\"domain_name\":\"ts\",\"team_name\":\"asher\"}

      - name: Comment Back to PR with Results
        if: success()
        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6
        with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
             const output = process.env.output || 'No output captured';

             github.rest.issues.createComment({

               issue_number: context.issue.number,

               owner: context.repo.owner,

               repo: context.repo.repo,

               body: `Lambda invocation results:\n${output}`

             })
      - name: Propagate Script Failure to Workflow
        if: failure()
        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6
        with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
             const output = process.env.output || 'No output captured';

             github.rest.issues.createComment({

               issue_number: context.issue.number,

               owner: context.repo.owner,

               repo: context.repo.repo,

               body: `Lambda invocation results:\n${output}. Check the GitHub actions`

             })

name: Invoke Airflow Customer Onboard Lambda

on:
  issue_comment:
    types: [created]

jobs:
  invoke_lambda:
    name: Trigger Airflow Customer Onboard Lambda Function
    if: startsWith(github.event.comment.body, 'deploy airflow_customer_onboard') && github.event.issue.pull_request
    runs-on: ubuntu-latest
    permissions:
      contents: read
      id-token: write
      pull-requests: write

    steps:
      - name: Checkout Repository
        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5

      - name: Verify PR comment
        id: verify_comment
        env:
           GH_TOKEN: ${{ secrets.GHA_TOKEN }}
        run: |
          COMMENT_BODY="${{ github.event.comment.body }}"
          export PR_NUMBER="${{ github.event.pull_request.number || github.event.issue.number }}"
          # Verify Comment
          if [[ "$COMMENT_BODY" != *"--env"* ]] || [[ "$COMMENT_BODY" != *"--payload"* ]]; then
              echo -e "**Usage:** \`deploy airflow_customer_onboard --env dev --payload {\\\"env\\\":\\\"dev\\\",\\\"domain_name\\\":\\\"ts\\\",\\\"team_name\\\":\\\"asher\\\"} \` \n **NOTE: Please make sure to escape the double quotes in the JSON payload**" > msg
              export msg=$(cat msg) 
              gh pr comment $PR_NUMBER --body "$msg"
              exit 1
          fi

      - name: Extract Environment & Payload from Comment
        id: extract_env

        run: |
          COMMENT_BODY="${{ github.event.comment.body }}"
          # Extract ENV
          ENV=$(echo "$COMMENT_BODY" | grep -oE '\--env (dev|prod|test)' | awk '{print $2}')
          if [[ -z "$ENV" ]]; then
            echo "::error::No valid environment found in comment! Use --env dev/prod/test"
            exit 1
          fi
          # Extract JSON payload (everything after --payload, possibly quoted)
          PAYLOAD_JSON=$(echo "$COMMENT_BODY" | grep -oP ' --payload.*?$' | awk '{print $2}' )
          if [[ -z "$PAYLOAD_JSON" ]]; then
            echo "::error::No JSON payload specified in the comment"
            exit 1
          fi
          echo "Environment detected: $ENV"
          echo "Payload detected: $PAYLOAD_JSON"

          echo $PAYLOAD_JSON
          echo "$PAYLOAD_JSON"

        
          echo "env=$ENV" >> "$GITHUB_ENV"
          echo "payload=$PAYLOAD_JSON" >> "$GITHUB_ENV"

      - name: Read ENV and Payload from env
        id: extract_env_from_env

        run: |
          aws lambda invoke --function-name dev-lambda --payload '${{env.payload}}' --debug demo-outfile
          echo "data --payload ${{env.payload}}"
        shell: bash

[celery]
flower_url_prefix = 
worker_concurrency = 16

[celery_kubernetes_executor]
kubernetes_queue = kubernetes

[core]
parallelism = 512
max_active_tasks_per_dag = 128
max_active_runs_per_dag = 64
colored_console_log = False
dags_folder = /opt/airflow/dags
executor = CeleryExecutor
load_examples = False
remote_logging = False
 

pip install --upgrade cryptography --platform manylinux2014_x86_64 --only-binary=:all: -t .

kube-prometheus-stack:
  alertmanager:
    ingress:
      enabled: true

      ingressClassName: internal

      hosts:
      - k8s1-am-alertmanager.indexexchange.com

      paths:
      - /

      pathType: Prefix

  grafana:
    enabled: true

    ingress:
      enabled: true

      ingressClassName: internal

      hosts:
      - k8s1-am-grafana.indexexchange.com

      ## Path for grafana ingress
      path: /

  kubeEtcd:
    ## If your etcd is not deployed as a pod, specify IPs it can be found on
    ##
    endpoints:
    - 10.129.5.1
    - 10.129.5.2
    - 10.129.5.3

  prometheus:
    ingress:
      enabled: true

      ingressClassName: internal

      hosts:
      - k8s1-am-prometheus.indexexchange.com

      paths:
      - /

      pathType: Prefix

    prometheusSpec:
      externalLabels:
        cluster: k8s1-am

- apiVersion: monitoring.coreos.com/v1
  kind: ServiceMonitor
  metadata:
    labels:
      argocd.argoproj.io/instance: data-systems-airflow-dev
      mimir_matcher: datasystems-dev-airflow
      prom: local
    name: data-systems-airflow-dev-statsd-monitor
    namespace: data-systems-airflow-dev
  spec:
    endpoints:
    - interval: 10s
      metricRelabelings:
      - action: replace
        replacement: datasystems
        targetLabel: team
      - action: replace
        replacement: dev
        targetLabel: env
      - action: replace
        replacement: airflow
        targetLabel: group
      path: /metrics
      port: statsd-scrape
    namespaceSelector:
      matchNames:
      - data-systems-airflow-dev
    selector:
      matchLabels:
        component: statsd
        env: dev
        team: datasystems
        tier: airflow