aws_redshift_pyspark.py

from pyspark.sql import SparkSession
from pyspark.sql.types import DoubleType
from pyspark.sql.functions import col

jar_path = 'rds_jar_driver.jar'
spark = SparkSession \
    .builder \
    .appName("AWS REDSHIFT PYSPARK APP") \
    .config("spark.jars", jar_path)\
    .config('spark.driver.extraClassPath', jar_path) \
    .getOrCreate()


JDBC_URL='jdbc:redshift://your_redshift_cluster.us-east-1.redshift.amazonaws.com:5439/dev_redshift?user=awsuser&password=yourpassword'
TEMP_DIR='s3://somebucket/tempdir/'
TABLE_NAME='seattle_emergence_calls'



df = spark.read.format("jdbc") \
  .option("url",JDBC_URL ) \
  .option("Tempdir", TEMP_DIR) \
  .option("dbtable", TABLE_NAME) \
  .option("forward_spark_s3_credentials", "true") \
  .load()