loading click-ingest with data

script.sh

 kcat -b localhost:29092 -t catchall_legacy -P -l test_events.jsonl

test_events.jsonl

{"timestamp": 1696964534, "id": "0", "account": {"id": "000000001"}, "recipient": "[email protected]", "event": "delivered"}
{"timestamp": 1696964535, "id": "1", "account": {"id": "000000001"}, "recipient": "[email protected]", "event": "delivered"}
{"timestamp": 1697040168, "id": "2", "account": {"id": "000000001"}, "recipient": "[email protected]", "event": "delivered"}
{"timestamp": 1697050168, "id": "3", "account": {"id": "000000002"}, "recipient": "[email protected]", "event": "delivered"}
{"timestamp": 1697053778, "id": "4", "account": {"id": "000000002"}, "recipient": "[email protected]", "event": "delivered"}
{"timestamp": 1697057388, "id": "5", "account": {"id": "000000002"}, "recipient": "[email protected]", "event": "delivered"}
{"timestamp": 1697040168, "id": "6", "account": {"id": "000000002"}, "recipient": "[email protected]", "event": "delivered"}
{"timestamp": 1696940168, "id": "7", "account": {"id": "000000002"}, "recipient": "[email protected]", "event": "delivered"}
{"timestamp": 1697050268, "id": "3", "account": {"id": "000000002"}, "recipient": "[email protected]", "event": "opened"}
{"timestamp": 1697050278, "id": "3", "account": {"id": "000000002"}, "recipient": "[email protected]", "event": "clicked"}
{"timestamp": 1697060168, "id": "1", "account": {"id": "000000001"}, "recipient": "[email protected]", "event": "opened"}
{"timestamp": 1697120168, "id": "2", "account": {"id": "000000001"}, "recipient": "[email protected]", "event": "opened"}
{"timestamp": 1697120178, "id": "2", "account": {"id": "000000001"}, "recipient": "[email protected]", "event": "clicked"}
{"timestamp": 1697120168, "id": "2", "account": {"id": "000000001"}, "recipient": "[email protected]", "event": "opened"}
{"timestamp": 1697120179, "id": "6", "account": {"id": "000000001"}, "recipient": "[email protected]", "event": "clicked"}
{"timestamp": 1697120169, "id": "6", "account": {"id": "000000001"}, "recipient": "[email protected]", "event": "opened"}
{"timestamp": 1696940168, "id": "7", "account": {"id": "000000002"}, "recipient": "[email protected]", "event": "opened"}

Observe prod email events: kcat -b kafka.service.prod.mailforce:9092 -t usw1_pipeline_events

staging topics: kcat -b kafka-001.node.staging.mailforce:9092 -L | grep topic

pipeline.jsonl
{"event": "accepted", "id": "WoJnqPRrTYilPUwR9zCgNy", "timestamp": 1704468093.6670388, "account": {"name": "Test Account", "id": "621279c94cb74f695b9670f8"}, "domain": {"name": "example.com", "id": "628d49d589e2a96a9951e975"}, "flags": {"is-authenticated": true}, "message": {"headers": {"from": "Caleb Test <[email protected]>"}}, "method": "HTTP", "originating-ip": "35.203.254.224", "recipient": "[email protected]", "recipient-domain": "gmail.com", "tags": null, "user-variables": "{}"}

Hit staging with a pipeline event: kcat -b kafka-001.node.staging.mailforce:9092 -t use4_pipeline_events -P -l pipeline.jsonl

local: kcat -b localhost:29092 -t catchall_legacy -P -l pipeline.jsonl