spark.py

spark.py

%spark.pyspark

spark = SparkSession.builder \
    .master("spark://localhost:7077") \
    .appName("ConexionACluster") \
    .getOrCreate()

df = spark.createDataFrame([("A", 1), ("B", 2)], ["col1", "col2"])

jdbc_url = "jdbc:postgresql://host.docker.internal:5432/northwind"
properties = {
    "user": "postgres",
    "password": "123456",
    "driver": "org.postgresql.Driver"
}

df2 = spark.read.jdbc(url=jdbc_url, table="orders", properties=properties)

print(df2.count())
    
df.show()