External data pipeline local setup

docker-compose.yml

version: "3"
services:
  # Auxiliar services the external data pipeline needs
  db:
    ports:
      - 5432:5432
    image: registry.redhat.io/rhscl/postgresql-13-rhel7
    environment:
      - POSTGRESQL_USER=user
      - POSTGRESQL_PASSWORD=password
      - POSTGRESQL_ADMIN_PASSWORD=admin
      - POSTGRESQL_DATABASE=aggregator
  kafka:
    image: quay.io/ccxdev/kafka-no-zk:latest
    ports:
      - 9092:9092
    environment:
      - KAFKA_ADVERTISED_HOST_NAME=kafka
      - KAFKA_CREATE_TOPICS="ccx.ocp.results:1:1"
  minio:
    image: quay.io/ccxdev/minio-server-ubi:latest
    command:
      - server
      - /data
      - --console-address
      - ":9001"
    ports:
      - 9000:9000
      - 9001:9001
    environment:
      - MINIO_ACCESS_KEY=minio
      - MINIO_SECRET_KEY=minio123
  createbuckets:
    image: minio/mc
    depends_on:
      - minio
    entrypoint:
      - /bin/sh
      - -c
      - "sleep 5;
      /usr/bin/mc config host add minio http://minio:9000 minio minio123;
      /usr/bin/mc mb minio/insights-upload-perma;
      /usr/bin/mc mb minio/insights-upload-rejected;
      exit 0;
      "
  ingress:
    # you can build the image from these sources
    # https://github.com/RedHatInsights/insights-ingress-go
    image: localhost/ingress:latest
    ports:
      - 3000:3000
    environment:
      - INGRESS_STAGEBUCKET=insights-upload-perma
      - INGRESS_REJECTBUCKET=insights-upload-rejected
      - INGRESS_VALIDTOPICS=testareno,platform.upload.testareno
      - OPENSHIFT_BUILD_COMMIT=woopwoop
      - INGRESS_MINIODEV=true
      - INGRESS_MINIOACCESSKEY=minio
      - INGRESS_MINIOSECRETKEY=minio123
      - INGRESS_MINIOENDPOINT=minio:9000
      - INGRESS_KAFKA_BROKERS=kafka:9092
    depends_on:
      - kafka
      - minio
  # Core services
  ccx-data-pipeline:
    image: quay.io/cloudservices/ccx-data-pipeline:latest
    environment:
      - CDP_PUBLISHER_SERVER=kafka:9092
      - CDP_CONSUMER_SERVER=kafka:9092
      - CDP_INCOMING_TOPIC=platform.upload.testareno
    depends_on:
      - kafka
  db-writer:
    image: quay.io/cloudservices/insights-results-aggregator:latest
    environment:
      - INSIGHTS_RESULTS_AGGREGATOR__STORAGE__DB_DRIVER=postgres
      - INSIGHTS_RESULTS_AGGREGATOR__STORAGE__PG_PARAMS=sslmode=disable
      - INSIGHTS_RESULTS_AGGREGATOR__STORAGE__PG_USERNAME=user
      - INSIGHTS_RESULTS_AGGREGATOR__STORAGE__PG_PASSWORD=password
      - INSIGHTS_RESULTS_AGGREGATOR__STORAGE__PG_HOST=db
      - INSIGHTS_RESULTS_AGGREGATOR__STORAGE__PG_PORT=5432
      - INSIGHTS_RESULTS_AGGREGATOR__STORAGE__PG_DB_NAME=aggregator
      - INSIGHTS_RESULTS_AGGREGATOR__SERVER__API_SPEC_FILE=/openapi/openapi.json
      - INSIGHTS_RESULTS_AGGREGATOR__SERVER__ADDRESS=:8085
      - INSIGHTS_RESULTS_AGGREGATOR__BROKER__ADDRESS=kafka:9092
      - INSIGHTS_RESULTS_AGGREGATOR__BROKER__TOPIC=ccx.ocp.results
      - INSIGHTS_RESULTS_AGGREGATOR__BROKER__SERVICE_NAME=insights-results-db-writer
      - INSIGHTS_RESULTS_AGGREGATOR__BROKER__GROUP=test-consumer-group
      - INSIGHTS_RESULTS_AGGREGATOR__BROKER__ENABLED=true
      - INSIGHTS_RESULTS_AGGREGATOR__BROKER__ENABLE_ORG_WHITELIST=false
      - INSIGHTS_RESULTS_AGGREGATOR__BROKER__TIMEOUT=300s
      - HABERDASHER_EMITTER=stderr
    command: bash -c "/insights-results-aggregator migration latest && /insights-results-aggregator"
    depends_on:
      - db
      - kafka
  aggregator:
    image: quay.io/cloudservices/insights-results-aggregator:latest
    environment:
      - INSIGHTS_RESULTS_AGGREGATOR__STORAGE__DB_DRIVER=postgres
      - INSIGHTS_RESULTS_AGGREGATOR__STORAGE__PG_PARAMS=sslmode=disable
      - INSIGHTS_RESULTS_AGGREGATOR__STORAGE__PG_USERNAME=user
      - INSIGHTS_RESULTS_AGGREGATOR__STORAGE__PG_PASSWORD=password
      - INSIGHTS_RESULTS_AGGREGATOR__STORAGE__PG_HOST=db
      - INSIGHTS_RESULTS_AGGREGATOR__STORAGE__PG_PORT=5432
      - INSIGHTS_RESULTS_AGGREGATOR__STORAGE__PG_DB_NAME=aggregator
      - INSIGHTS_RESULTS_AGGREGATOR__SERVER__API_PREFIX=/api/v1/
      - INSIGHTS_RESULTS_AGGREGATOR__SERVER__API_SPEC_FILE=/openapi/openapi.json
      - INSIGHTS_RESULTS_AGGREGATOR__SERVER__ADDRESS=:8082
    ports:
      - 8082:8082
    depends_on:
      - db-writer
      - db
  content-service:
    image: quay.io/cloudservices/ccx-insights-content-service
    environment:
      - INSIGHTS_CONTENT_SERVICE__SERVER__ADDRESS=:8081
      - INSIGHTS_CONTENT_SERVICE__SERVER__API_PREFIX=/api/v1/
      - INSIGHTS_CONTENT_SERVICE__SERVER__API_SPEC_FILE=/openapi/openapi.json
      - INSIGHTS_CONTENT_SERVICE__GROUPS__PATH=/groups/groups_config.yaml
  smart-proxy:
    image: quay.io/cloudservices/ccx-smart-proxy:latest
    environment:
      - INSIGHTS_RESULTS_SMART_PROXY__SERVER__ADDRESS=:8080
      - INSIGHTS_RESULTS_SMART_PROXY__SERVER__API_V1_PREFIX=/api/v1/
      - INSIGHTS_RESULTS_SMART_PROXY__SERVER__API_V2_PREFIX=/api/v2/
      - INSIGHTS_RESULTS_SMART_PROXY__SERVER__API_V1_SPEC_FILE=/openapi/v1/openapi.json
      - INSIGHTS_RESULTS_SMART_PROXY__SERVER__API_V2_SPEC_FILE=/openapi/v2/openapi.json
      - INSIGHTS_RESULTS_SMART_PROXY__SERVICES__GROUPS_POLL_TIME=60s
      - INSIGHTS_RESULTS_SMART_PROXY__SERVICES__CONTENT=http://content-service:8081/api/v1/
      - INSIGHTS_RESULTS_SMART_PROXY__SERVICES__AGGREGATOR=http://aggregator:8082/api/v1/
      - INSIGHTS_RESULTS_SMART_PROXY__SERVER__AUTH=true
      - INSIGHTS_RESULTS_SMART_PROXY__SERVER__AUTH_TYPE=xrh
      - INSIGHTS_RESULTS_SMART_PROXY__SERVER__USE_HTTPS=false
    ports:
      - 8080:8080
    depends_on:
      - content-service
      - aggregator