Created
August 4, 2022 13:55
-
-
Save DennisFederico/fe36adc245216d966936316bcfcdad78 to your computer and use it in GitHub Desktop.
Control Center, Connect and KSQLDB using Kafka and SR in Confluent Cloud
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| version: '2' | |
| services: | |
| connect_1: | |
| image: confluentinc/cp-server-connect:7.2.1 | |
| hostname: connect-1 | |
| container_name: connect-1 | |
| ports: | |
| - "8083:8083" | |
| environment: | |
| CONNECT_BOOTSTRAP_SERVERS: $BOOTSTRAP_SERVERS | |
| CONNECT_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG | |
| CONNECT_SECURITY_PROTOCOL: SASL_SSL | |
| CONNECT_SASL_MECHANISM: PLAIN | |
| CONNECT_GROUP_ID: "connect-onprem" | |
| CONNECT_CONFIG_STORAGE_TOPIC: connect-onprem-configs | |
| CONNECT_OFFSET_STORAGE_TOPIC: connect-onprem-offsets | |
| CONNECT_STATUS_STORAGE_TOPIC: connect-onprem-status | |
| CONNECT_REPLICATION_FACTOR: 3 | |
| CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 3 | |
| CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 3 | |
| CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 3 | |
| CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.storage.StringConverter" | |
| CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" | |
| CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true" | |
| CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: $SCHEMA_REGISTRY_URL | |
| CONNECT_VALUE_CONVERTER_BASIC_AUTH_CREDENTIALS_SOURCE: $BASIC_AUTH_CREDENTIALS_SOURCE | |
| CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO: $SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO | |
| CONNECT_REST_ADVERTISED_HOST_NAME: "connect-1" | |
| CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" | |
| CONNECT_LOG4J_ROOT_LOGLEVEL: INFO | |
| CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR" | |
| # CLASSPATH required due to CC-2422 | |
| CLASSPATH: /usr/share/java/monitoring-interceptors/* | |
| # Connect producer | |
| CONNECT_PRODUCER_SECURITY_PROTOCOL: SASL_SSL | |
| CONNECT_PRODUCER_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG | |
| CONNECT_PRODUCER_SASL_MECHANISM: PLAIN | |
| CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" | |
| CONNECT_PRODUCER_CONFLUENT_MONITORING_INTERCEPTOR_SECURITY_PROTOCOL: SASL_SSL | |
| CONNECT_PRODUCER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG | |
| CONNECT_PRODUCER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_MECHANISM: PLAIN | |
| # Connect consumer | |
| CONNECT_CONSUMER_SECURITY_PROTOCOL: SASL_SSL | |
| CONNECT_CONSUMER_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG | |
| CONNECT_CONSUMER_SASL_MECHANISM: PLAIN | |
| CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" | |
| CONNECT_CONSUMER_CONFLUENT_MONITORING_INTERCEPTOR_SECURITY_PROTOCOL: SASL_SSL | |
| CONNECT_CONSUMER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG | |
| CONNECT_CONSUMER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_MECHANISM: PLAIN | |
| volumes: | |
| - $PWD/connect-plugins:/usr/share/confluent-hub-components | |
| connect_2: | |
| image: confluentinc/cp-server-connect:7.2.1 | |
| hostname: connect-2 | |
| container_name: connect-2 | |
| ports: | |
| - "8383:8083" | |
| environment: | |
| CONNECT_BOOTSTRAP_SERVERS: $BOOTSTRAP_SERVERS | |
| CONNECT_GROUP_ID: "connect-onprem" | |
| CONNECT_CONFIG_STORAGE_TOPIC: connect-onprem-configs | |
| CONNECT_OFFSET_STORAGE_TOPIC: connect-onprem-offsets | |
| CONNECT_STATUS_STORAGE_TOPIC: connect-onprem-status | |
| CONNECT_REPLICATION_FACTOR: 3 | |
| CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 3 | |
| CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 3 | |
| CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 3 | |
| CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.storage.StringConverter" | |
| CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" | |
| CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true" | |
| CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: $SCHEMA_REGISTRY_URL | |
| CONNECT_VALUE_CONVERTER_BASIC_AUTH_CREDENTIALS_SOURCE: $BASIC_AUTH_CREDENTIALS_SOURCE | |
| CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO: $SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO | |
| CONNECT_REST_ADVERTISED_HOST_NAME: "connect-2" | |
| #CONNECT_REST_ADVERTISED_PORT: 8383 | |
| CONNECT_REST_PORT: 8083 | |
| CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" | |
| CONNECT_LOG4J_ROOT_LOGLEVEL: INFO | |
| CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR" | |
| # CLASSPATH required due to CC-2422 | |
| CLASSPATH: /usr/share/java/monitoring-interceptors/* | |
| # Connect worker | |
| CONNECT_SECURITY_PROTOCOL: SASL_SSL | |
| CONNECT_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG | |
| CONNECT_SASL_MECHANISM: PLAIN | |
| # Connect producer | |
| CONNECT_PRODUCER_SECURITY_PROTOCOL: SASL_SSL | |
| CONNECT_PRODUCER_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG | |
| CONNECT_PRODUCER_SASL_MECHANISM: PLAIN | |
| CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" | |
| CONNECT_PRODUCER_CONFLUENT_MONITORING_INTERCEPTOR_SECURITY_PROTOCOL: SASL_SSL | |
| CONNECT_PRODUCER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG | |
| CONNECT_PRODUCER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_MECHANISM: PLAIN | |
| # Connect consumer | |
| CONNECT_CONSUMER_SECURITY_PROTOCOL: SASL_SSL | |
| CONNECT_CONSUMER_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG | |
| CONNECT_CONSUMER_SASL_MECHANISM: PLAIN | |
| CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" | |
| CONNECT_CONSUMER_CONFLUENT_MONITORING_INTERCEPTOR_SECURITY_PROTOCOL: SASL_SSL | |
| CONNECT_CONSUMER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG | |
| CONNECT_CONSUMER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_MECHANISM: PLAIN | |
| volumes: | |
| - $PWD/connect-plugins:/usr/share/confluent-hub-components | |
| ksqldb-server: | |
| image: confluentinc/cp-ksqldb-server:7.2.1 | |
| hostname: ksqldb-server | |
| container_name: ksqldb-server | |
| ports: | |
| - "8089:8089" | |
| environment: | |
| KSQL_HOST_NAME: ksqldb-server | |
| KSQL_CONFIG_DIR: "/etc/ksql" | |
| KSQL_LISTENERS: "http://0.0.0.0:8089" | |
| KSQL_AUTO_OFFSET_RESET: "earliest" | |
| KSQL_COMMIT_INTERVAL_MS: 0 | |
| KSQL_CACHE_MAX_BYTES_BUFFERING: 0 | |
| KSQL_KSQL_SCHEMA_REGISTRY_URL: $SCHEMA_REGISTRY_URL | |
| KSQL_KSQL_SCHEMA_REGISTRY_BASIC_AUTH_CREDENTIALS_SOURCE: $BASIC_AUTH_CREDENTIALS_SOURCE | |
| KSQL_KSQL_SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO: $SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO | |
| KSQL_BOOTSTRAP_SERVERS: $BOOTSTRAP_SERVERS | |
| KSQL_SECURITY_PROTOCOL: "SASL_SSL" | |
| KSQL_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG | |
| KSQL_SASL_MECHANISM: "PLAIN" | |
| KSQL_KSQL_STREAMS_PRODUCER_RETRIES: 2147483647 | |
| KSQL_KSQL_STREAMS_PRODUCER_CONFLUENT_BATCH_EXPIRE_MS: 9223372036854775807 | |
| KSQL_KSQL_STREAMS_PRODUCER_REQUEST_TIMEOUT_MS: 300000 | |
| KSQL_KSQL_STREAMS_PRODUCER_MAX_BLOCK_MS: 9223372036854775807 | |
| KSQL_KSQL_STREAMS_PRODUCER_DELIVERY_TIMEOUT_MS: 2147483647 | |
| KSQL_KSQL_STREAMS_REPLICATION_FACTOR: 3 | |
| KSQL_KSQL_INTERNAL_TOPIC_REPLICAS: 3 | |
| KSQL_KSQL_LOGGING_PROCESSING_TOPIC_REPLICATION_FACTOR: 3 | |
| KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: 'true' | |
| KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: 'true' | |
| # Producer Confluent Monitoring Interceptors for Control Center streams monitoring | |
| KSQL_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" | |
| KSQL_PRODUCER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_MECHANISM: PLAIN | |
| KSQL_PRODUCER_CONFLUENT_MONITORING_INTERCEPTOR_SECURITY_PROTOCOL: "SASL_SSL" | |
| KSQL_PRODUCER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG | |
| # Consumer Confluent Monitoring Interceptors for Control Center streams monitoring | |
| KSQL_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" | |
| KSQL_CONSUMER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_MECHANISM: PLAIN | |
| KSQL_CONSUMER_CONFLUENT_MONITORING_INTERCEPTOR_SECURITY_PROTOCOL: "SASL_SSL" | |
| KSQL_CONSUMER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG | |
| KSQL_KSQL_CONNECT_URL: "http://connect:8083" | |
| ksqldb-cli: | |
| image: confluentinc/cp-ksqldb-cli:7.2.1 | |
| container_name: ksqldb-cli | |
| entrypoint: /bin/sh | |
| tty: true | |
| control-center: | |
| image: confluentinc/cp-enterprise-control-center:7.2.1 | |
| hostname: control-center | |
| container_name: control-center | |
| ports: | |
| - "9021:9021" | |
| environment: | |
| CONTROL_CENTER_BOOTSTRAP_SERVERS: $BOOTSTRAP_SERVERS | |
| CONTROL_CENTER_KSQL_KSQLDB1_URL: "http://ksqldb-server:8089" | |
| CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: "http://localhost:8089" | |
| # CONTROL_CENTER_KSQL_KSQLCLOUD_URL: $KSQLDB_ENDPOINT:443 | |
| # CONTROL_CENTER_KSQL_KSQLCLOUD_ADVERTISED_URL: $KSQLDB_ENDPOINT | |
| # CONTROL_CENTER_KSQL_KSQLCLOUD_BASIC_AUTH_USER_INFO: $KSQLDB_BASIC_AUTH_USER_INFO | |
| CONTROL_CENTER_SCHEMA_REGISTRY_URL: $SCHEMA_REGISTRY_URL | |
| CONTROL_CENTER_SCHEMA_REGISTRY_BASIC_AUTH_CREDENTIALS_SOURCE: $BASIC_AUTH_CREDENTIALS_SOURCE | |
| CONTROL_CENTER_SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO: $SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO | |
| CONTROL_CENTER_CONNECT_CONNECT-DEFAULT_CLUSTER: "http://connect-1:8083,http://connect-2:8083" | |
| CONTROL_CENTER_STREAMS_SECURITY_PROTOCOL: SASL_SSL | |
| CONTROL_CENTER_STREAMS_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG | |
| CONTROL_CENTER_STREAMS_SASL_MECHANISM: PLAIN | |
| CONTROL_CENTER_REPLICATION_FACTOR: 3 | |
| CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_REPLICATION: 3 | |
| CONTROL_CENTER_INTERNAL_TOPICS_REPLICATION: 3 | |
| CONTROL_CENTER_COMMAND_TOPIC_REPLICATION: 3 | |
| CONTROL_CENTER_METRICS_TOPIC_REPLICATION: 3 | |
| CONFLUENT_METRICS_TOPIC_REPLICATION: 3 | |
| CONTROL_CENTER_STREAMS_NUM_STREAM_THREADS: 3 | |
| CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1 | |
| CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1 | |
| # Workaround for MMA-3564 | |
| CONTROL_CENTER_METRICS_TOPIC_MAX_MESSAGE_BYTES: 8388608 | |
| PORT: 9021 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment