rssanders3

{
  "type": "record",
  "name": "Order",
  "fields": [
    {
      "name": "orderId",
      "type": "long"
    },
    {
      "name": "orderDate",

rssanders3

+--------------------+--------------+
| Confluent Platform | Apache Kafka |
+--------------------+--------------+
| 2.0.x              | 0.9.0.x      |
+--------------------+--------------+
| 3.0.x              | 0.10.0.x     |
+--------------------+--------------+
| 3.1.x              | 0.10.1.x     |
+--------------------+--------------+
| 3.2.x              | 0.10.2.x     |

rssanders3

[Confluent.dist]
name=Confluent repository (dist)
baseurl=https://packages.confluent.io/rpm/{CONFLUENT_VERSION}/7
gpgcheck=1
gpgkey=https://packages.confluent.io/rpm/{CONFLUENT_VERSION}/archive.key
enabled=1

[Confluent]
name=Confluent repository
baseurl=https://packages.confluent.io/rpm/{CONFLUENT_VERSION}

rssanders3

import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkConf, SparkContext
from pyspark.sql import SparkSession
from awsglue.context import GlueContext
from awsglue.job import Job

## @params: [JOB_NAME]
args = getResolvedOptions(sys.argv, ['JOB_NAME'])

rssanders3

# Setup Test Data

df = spark.read.format("csv").option("header", "true").load(f"s3://{AWS_BUCKET}/data/input_data.csv")
df.registerTempTable("input_data")

df = spark.read.format("csv").option("header", "true").load(f"s3://{AWS_BUCKET}/data/input_data_updates.csv")
df.registerTempTable("input_data_updates")


## CREATE TABLE