Cheat sheet for Standalone Kafka Connect with JSON deserialisation

kafka-connect-json.sh

# Enter Kafka Connect Shell Client
make connect-shell

# Kafka Connect with connect-standalone setting, connecting via postgres
connect-standalone dev/connect-standalone.properties dev/source-postgres-incremental.properties

# Expected console-consumer output from topic: app-users
# {
#   "schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"first_name"},{"type":"string","optional":false,"field":"last_name"},{"type":"string","optional":false,"field":"email"}],"optional":false,"name":"users"},
#   "payload":{"id":101,"first_name":"Cass","last_name":"Truss","email":"[email protected]"}
# }
# {
#   "schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"first_name"},{"type":"string","optional":false,"field":"last_name"},{"type":"string","optional":false,"field":"email"}],"optional":false,"name":"users"},
#   "payload":{"id":102,"first_name":"Joete","last_name":"Scowcroft","email":"[email protected]"}
# }
# {
#   "schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"first_name"},{"type":"string","optional":false,"field":"last_name"},{"type":"string","optional":false,"field":"email"}],"optional":false,"name":"users"},
#   "payload":{"id":103,"first_name":"Celine","last_name":"Wortley","email":"[email protected]"}
# }
# {
#   "schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"first_name"},{"type":"string","optional":false,"field":"last_name"},{"type":"string","optional":false,"field":"email"}],"optional":false,"name":"users"},
#   "payload":{"id":104,"first_name":"Othilia","last_name":"Gumme","email":"[email protected]"}
# }
# {
#   "schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"first_name"},{"type":"string","optional":false,"field":"last_name"},{"type":"string","optional":false,"field":"email"}],"optional":false,"name":"users"},
#   "payload":{"id":105,"first_name":"Maynord","last_name":"Buttrick","email":"[email protected]"}
# }
# ...

# Exit Docker when Kafka Connect is done
(Ctrl-C)
exit