dbt_materialize_redpanda.yml

dbt_materialize_redpanda.yml

version: '3.7'
services:
  redpanda:
    image: docker.vectorized.io/vectorized/redpanda:v22.1.7
    container_name: redpanda
    hostname: redpanda
    command:
     - redpanda start
     - --overprovisioned
     - --smp 1
     - --memory 1G
     - --reserve-memory 0M
     - --node-id 0
     - --check=false
     - --kafka-addr PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
     - --advertise-kafka-addr PLAINTEXT://redpanda:29092,OUTSIDE://127.0.0.1:9092
     - --pandaproxy-addr 0.0.0.0:8082
     - --advertise-pandaproxy-addr redpanda:8082
     - --set redpanda.enable_transactions=true
     - --set redpanda.enable_idempotence=true
     - --set redpanda.auto_create_topics_enabled=true
     - --set redpanda.cluster_id=redpanda
     - '--set pandaproxy_client.brokers={"address":"redpanda","port":29092}'
     - '--set schema_registry_client.brokers={"address":"redpanda","port":29092}'
    ports:
     - 9092:9092
     - 8081:8081
     - 8082:8082
     - 9644:9644
     - 29092:29092
    networks:
      - streaming_network
    healthcheck: { test: curl -f localhost:9644/v1/status/ready, interval: 1s, start_period: 30s }
  kafka-create-topics:
    image: docker.vectorized.io/vectorized/redpanda:v22.1.7
    entrypoint: []
    networks:
      - streaming_network
    depends_on:
      - redpanda
    command: "sh -c 'echo Waiting for Kafka to be ready... && \
                       while ! rpk cluster health -e --api-urls redpanda:9644; \
                        do sleep 1; done && \
                        
                       rpk topic create my_topic1 --brokers redpanda:29092 -p 3 -r 1 && \
                       rpk topic create my_topic2 --brokers redpanda:29092 -p 3 -r 1 && \
                       
                       echo FINISH'"
  materialized:
    image: materialize/materialized:v0.26.4
    container_name: materialized
    hostname: materialized
    command: -w1
    networks:
      - streaming_network
    ports:
      - 6875:6875
    environment:
      MZ_LOG_FILTER: WARN
    depends_on:
      - redpanda
networks:
  streaming_network:
    driver: bridge
    name: streaming_network