taufiqibrahim

kafkacat -C -b <AWS_MSK_BOOTSTRAP_SERVERS_URLS> -t demo-walkthrough-orders -o -20 -e | jq -c
% Reached end of topic demo-walkthrough-orders [0] at offset 10: exiting
{"OrderNo":"SO-001","OrderDate":"1/6/19","Region":"East","Customer":"Jones","Item":"Pencil","Units":95,"Unit Cost":1.99,"Total":189.05}
{"OrderNo":"SO-002","OrderDate":"1/23/19","Region":"Central","Customer":"Kivell","Item":"Binder","Units":50,"Unit Cost":19.99,"Total":999.5}
{"OrderNo":"SO-003","OrderDate":"2/9/19","Region":"Central","Customer":"Jardine","Item":"Pencil","Units":36,"Unit Cost":4.99,"Total":179.64}
{"OrderNo":"SO-004","OrderDate":"2/26/19","Region":"Central","Customer":"Gill","Item":"Pen","Units":27,"Unit Cost":19.99,"Total":539.73}
{"OrderNo":"SO-005","OrderDate":"3/15/19","Region":"West","Customer":"Sorvino","Item":"Pencil","Units":56,"Unit Cost":2.99,"Total":167.44}
{"OrderNo":"SO-006","OrderDate":"4/1/19","Region":"East","Customer":"Jones","Item":"Binder","Units":60,"Unit Cost":4.99,"Total":299.4}
{"OrderNo":"SO-007","OrderDat

taufiqibrahim

kafkacat -C -b <AWS_MSK_BOOTSTRAP_SERVERS_URLS> -t demo-walkthrough-p01OrdersEnrichCustomerLocation -o -20 -e | jq -c
% Reached end of topic demo-walkthrough-p01OrdersEnrichCustomerLocation [0] at offset 10: exiting
{"OrderNo":"SO-001","OrderDate":"1/6/19","Region":"East","Customer":"Jones","Item":"Pencil","Units":95,"Unit Cost":1.99,"Total":189.05,"address":"1745 T Street Southeast","longitude":"-76.979235","latitude":"38.867033"}
{"OrderNo":"SO-002","OrderDate":"1/23/19","Region":"Central","Customer":"Kivell","Item":"Binder","Units":50,"Unit Cost":19.99,"Total":999.5,"address":"6007 Applegate Lane","longitude":"-85.649851","latitude":"38.134301"}
{"OrderNo":"SO-003","OrderDate":"2/9/19","Region":"Central","Customer":"Jardine","Item":"Pencil","Units":36,"Unit Cost":4.99,"Total":179.64,"address":"560 Penstock Drive","longitude":"-121.077583","latitude":"39.213076"}
{"OrderNo":"SO-004","OrderDate":"2/26/19","Region":"Central","Customer":"Gill","Item":"Pen","Units":27,"Unit Cost":19.99,"Total":539.73,"address":

taufiqibrahim

# install docker
sudo apt-get update
sudo apt-get install \
    ca-certificates \
    curl \
    gnupg \
    lsb-release -y

# Add Docker’s official GPG key
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg

taufiqibrahim

# Create directory for ftp
export NONROOTUSER=$USER
sudo mkdir -p /ftpdata
sudo chown -R $NONROOTUSER:$NONROOTUSER /ftpdata
mkdir -p /ftpdata/ftpuser/buckets/$BUCKET_NAME
mkdir -p /ftpdata/ftpuser/passwd

taufiqibrahim

version: '3'
services:
  ftpd_server:
    image: stilliard/pure-ftpd
    container_name: pure-ftpd
    ports:
      - "21:21"
      - "30000-30009:30000-30009"
    volumes: # remember to replace /folder_on_disk/ with the path to where you want to store the files on the host machine
      - "/ftpdata/ftpuser/buckets:/home/ftpuser/"

taufiqibrahim

# Install gcsfuse
export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s`
echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -

sudo apt-get update
sudo apt-get install gcsfuse

# Add this to /etc/fstab
# Create a file /ftpdata/key.json which contains service account JSON key

taufiqibrahim

"""
Requirements:
pip install tabula-py
pip install tabulate

Read all *.pdf files inside PATH_DIR, transpose and compile into single CSV file
"""

import pandas as pd
import os

taufiqibrahim

version: '3.9'
services:
  mysql:
    image: quay.io/debezium/example-mysql
    container_name: mysql
    ports:
      - 3306:3306
    environment:
      - MYSQL_ROOT_PASSWORD=debezium
      - MYSQL_USER=mysqluser

taufiqibrahim

debezium.sink.type=pubsub
debezium.sink.pubsub.project.id=<GOOGLE_CLOUD_PROJECT_ID>
debezium.source.connector.class=io.debezium.connector.mysql.MySqlConnector
debezium.source.database.hostname=mysql
debezium.source.database.port=3306
debezium.source.database.user=root
debezium.source.database.password=debezium
debezium.source.database.server.id=1001
debezium.source.database.server.name=mysql
debezium.source.database.include.list=inventory

taufiqibrahim

gcloud pubsub schemas create mysql.inventory.products-schema \
        --type=AVRO \
        --definition='
{
 "type" : "record",
 "name" : "MysqlInventoryProductsSchema",
 "fields" : [
    {
        "type": "int",
        "optional": false,