Created
August 29, 2017 13:08
-
-
Save rmoff/f32543f78d821b25502f6db49eee9259 to your computer and use it in GitHub Desktop.
Kafka Connect JDBC source with JSON converter
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"name": "jdbc_source_mysql_foobar_01", | |
"config": { | |
"_comment": "The JDBC connector class. Don't change this if you want to use the JDBC Source.", | |
"connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector", | |
"_comment": "How to serialise the value of keys - here use the Json converter. We want to retain the schema in the message (which will generate a schema/payload JSON document) and so set schemas.enable=true", | |
"key.converter": "org.apache.kafka.connect.json.JsonConverter", | |
"key.converter.schemas.enable":"true", | |
"_comment": "As above, but for the value of the message. Note that these key/value serialisation settings can be set globally for Connect and thus omitted for individual connector configs to make them shorter and clearer", | |
"value.converter": "org.apache.kafka.connect.json.JsonConverter", | |
"value.converter.schemas.enable": "true", | |
"_comment": " --- JDBC-specific configuration below here --- ", | |
"_comment": "JDBC connection URL. This will vary by RDBMS. Consult your manufacturer's handbook for more information", | |
"connection.url": "jdbc:mysql://localhost:3306/demo?user=rmoff&password=pw", | |
"_comment": "Which table(s) to include", | |
"table.whitelist": "foobar", | |
"_comment": "Pull all rows based on an timestamp column. You can also do bulk or incrementing column-based extracts. For more information, see http://docs.confluent.io/current/connect/connect-jdbc/docs/source_config_options.html#mode", | |
"mode": "timestamp", | |
"_comment": "Which column has the timestamp value to use? ", | |
"timestamp.column.name": "update_ts", | |
"_comment": "If the column is not defined as NOT NULL, tell the connector to ignore this ", | |
"validate.non.null": "false", | |
"_comment": "The Kafka topic will be made up of this prefix, plus the table name ", | |
"topic.prefix": "mysql-" | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment