Created
September 12, 2020 23:21
-
-
Save scheung38/b4be143611fb1d87d32f3b1000954654 to your computer and use it in GitHub Desktop.
Kafka connect issue
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
version: "3" | |
services: | |
zookeeper: | |
image: confluentinc/cp-zookeeper:5.5.0 | |
hostname: zookeeper | |
container_name: zookeeper | |
ports: | |
- "2181:2181" | |
environment: | |
ZOOKEEPER_CLIENT_PORT: 2181 | |
ZOOKEEPER_TICK_TIME: 2000 | |
broker: | |
image: confluentinc/cp-enterprise-kafka:5.5.0 | |
hostname: broker | |
container_name: broker | |
depends_on: | |
- zookeeper | |
ports: | |
- "29092:29092" | |
environment: | |
KAFKA_BROKER_ID: 1 | |
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' | |
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT | |
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9092,PLAINTEXT_HOST://localhost:29092 | |
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 | |
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 | |
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 | |
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 | |
schema-registry: | |
image: confluentinc/cp-schema-registry:5.5.0 | |
hostname: schema-registry | |
container_name: schema-registry | |
depends_on: | |
- zookeeper | |
- broker | |
ports: | |
- "8081:8081" | |
environment: | |
SCHEMA_REGISTRY_HOST_NAME: schema-registry | |
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' | |
ksqldb-server: | |
image: confluentinc/ksqldb-server:0.10.1 | |
hostname: ksqldb-server | |
container_name: ksqldb-server | |
depends_on: | |
- broker | |
- schema-registry | |
- kafka-connect | |
ports: | |
- "8088:8088" | |
volumes: | |
- "./confluent-hub-components/:/usr/share/kafka/plugins/" | |
environment: | |
KSQL_LISTENERS: "http://0.0.0.0:8088" | |
KSQL_BOOTSTRAP_SERVERS: "broker:9092" | |
KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" | |
KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true" | |
KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true" | |
KSQL_CONNECT_GROUP_ID: "ksql-connect-cluster" | |
KSQL_CONNECT_BOOTSTRAP_SERVERS: "broker:9092" | |
KSQL_CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter" | |
KSQL_CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" | |
KSQL_CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" | |
KSQL_CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" | |
KSQL_CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false" | |
KSQL_CONNECT_CONFIG_STORAGE_TOPIC: "ksql-connect-configs" | |
KSQL_CONNECT_OFFSET_STORAGE_TOPIC: "ksql-connect-offsets" | |
KSQL_CONNECT_STATUS_STORAGE_TOPIC: "ksql-connect-statuses" | |
KSQL_CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 | |
KSQL_CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 | |
KSQL_CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 | |
KSQL_CONNECT_PLUGIN_PATH: "/usr/share/kafka/plugins" | |
ksqldb-cli: | |
image: confluentinc/ksqldb-cli:0.10.1 | |
container_name: ksqldb-cli | |
depends_on: | |
- broker | |
- ksqldb-server | |
entrypoint: /bin/sh | |
tty: true | |
ksql-python: | |
build: | |
context: . | |
network_mode: host | |
depends_on: | |
- ksqldb-server | |
- broker | |
- schema-registry | |
environment: | |
KSQL_SERVER: localhost:8088 | |
STREAMS_BOOTSTRAP_SERVERS: localhost:29092 | |
volumes: | |
- ./:/app | |
command: tail -f /dev/null | |
kafka-connect: | |
image: confluentinc/cp-kafka-connect-base:5.5.0 | |
container_name: kafka-connect | |
depends_on: | |
- zookeeper | |
- broker | |
- schema-registry | |
ports: | |
- 8083:8083 | |
environment: | |
CONNECT_BOOTSTRAP_SERVERS: "broker:29092" | |
CONNECT_REST_PORT: 8083 | |
CONNECT_GROUP_ID: compose-connect-group | |
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs | |
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets | |
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status | |
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter | |
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter | |
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081' | |
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" | |
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" | |
CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect" | |
CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO" | |
CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN: "[%d] %p %X{connector.context}%m (%c:%L)%n" | |
CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR" | |
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1" | |
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1" | |
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1" | |
CONNECT_PLUGIN_PATH: '/usr/share/java,/usr/share/confluent-hub-components/' | |
command: | |
- bash | |
- -c | |
- | | |
echo "Installing connector plugins" | |
confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:5.5.0 | |
confluent-hub install --no-prompt jcustenborder/kafka-connect-spooldir:2.0.43 | |
# | |
# ----------- | |
# Launch the Kafka Connect worker | |
/etc/confluent/docker/run & | |
# | |
# Don't exit | |
sleep infinity | |
volumes: | |
- $PWD/data:/data | |
postgres: | |
# *-----------------------------* | |
# To connect to the DB: | |
# docker exec -it postgres bash -c 'psql -U $POSTGRES_USER $POSTGRES_DB' | |
# *-----------------------------* | |
image: postgres:12 | |
container_name: postgres | |
environment: | |
- POSTGRES_USER=postgres | |
- POSTGRES_PASSWORD=postgres | |
kafkacat: | |
image: edenhill/kafkacat:1.5.0 | |
container_name: kafkacat | |
links: | |
- broker | |
- schema-registry | |
entrypoint: | |
- /bin/sh | |
- -c | |
- | | |
apk add jq; | |
while [ 1 -eq 1 ];do sleep 60;done | |
kafka-rest: | |
image: confluentinc/cp-kafka-rest:5.5.0 | |
hostname: kafka-rest | |
ports: | |
- "8082:8082" | |
depends_on: | |
- schema-registry | |
environment: | |
KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper:2181 | |
KAFKA_REST_SCHEMA_REGISTRY_URL: schema-registry:8081 | |
KAFKA_REST_HOST_NAME: kafka-rest | |
KAFKA_REST_LISTENERS: http://kafka-rest:8082 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
$docker-compose up -d
inside cp-kafka-connect-base shell:
[main] ERROR io.confluent.admin.utils.ClusterStatus - Error while getting broker list.
java.util.concurrent.ExecutionException: org.apache.kafka.common.errors.TimeoutException: Call(callName=listNodes, deadlineMs=1599952674475) timed out at 1599952674476 after 1 attempt(s)
at org.apache.kafka.common.internals.KafkaFutureImpl.wrapAndThrow(KafkaFutureImpl.java:45)
at org.apache.kafka.common.internals.KafkaFutureImpl.access$000(KafkaFutureImpl.java:32)
at org.apache.kafka.common.internals.KafkaFutureImpl$SingleWaiter.await(KafkaFutureImpl.java:89)
at org.apache.kafka.common.internals.KafkaFutureImpl.get(KafkaFutureImpl.java:260)
at io.confluent.admin.utils.ClusterStatus.isKafkaReady(ClusterStatus.java:149)
at io.confluent.admin.utils.cli.KafkaReadyCommand.main(KafkaReadyCommand.java:150)
Caused by: org.apache.kafka.common.errors.TimeoutException: Call(callName=listNodes, deadlineMs=1599952674475) timed out at 1599952674476 after 1 attempt(s)
Caused by: org.apache.kafka.common.errors.TimeoutException: Timed out waiting for a node assignment.
[kafka-admin-client-thread | adminclient-1] WARN org.apache.kafka.clients.NetworkClient - [AdminClient clientId=adminclient-1] Connection to node 1 (localhost/127.0.0.1:29092) could not be established. Broker may not be available.
[main] INFO io.confluent.admin.utils.ClusterStatus - Expected 1 brokers but found only 0. Trying to query Kafka for metadata again ...
[main] ERROR io.confluent.admin.utils.ClusterStatus - Expected 1 brokers but found only 0. Brokers found []