taiwotman

COPY seattle_emergence_calls from 's3://seattle-realtime-emergence-fire-call/Seattle_Real_Time_Fire_911_Calls.csv'  FORMAT CSV 
        IGNOREHEADER 1
        CREDENTIALS 'aws_iam_role=your_iam_role' 
        REGION 'us-east-1';

taiwotman

from pyspark.sql import SparkSession
from pyspark.sql.types import DoubleType
from pyspark.sql.functions import col

jar_path = 'rds_jar_driver.jar'
spark = SparkSession \
    .builder \
    .appName("AWS REDSHIFT PYSPARK APP") \
    .config("spark.jars", jar_path)\
    .config('spark.driver.extraClassPath', jar_path) \

taiwotman

#Transformation
df = df.withColumn("latitude", col("latitude").cast(DoubleType()))\
       .withColumn("longitude", col("longitude").cast(DoubleType()))

df.printSchema()

df.show(10)


#More Insight

taiwotman

taiwotman

## Key Docker Commands

docker build --pull --no-cache --tag=<image-name> .

docker-compose up

docker-compose down

taiwotman

version: "3.9"
services:
    postgres:
      image: postgres:10.5
      restart: always
      environment:
        - POSTGRES_USER=<username>
        - POSTGRES_PASSWORD=<password>
      ports:
        - '5433:5432'

taiwotman

psycopg2==2.9.3
pytest== 7.1.2

taiwotman

FROM python:latest

WORKDIR app
COPY . /app

RUN python3 -m pip install -r requirements.txt

taiwotman

$ docker build -t <image-name> .

taiwotman

docker compose up run