Skip to content

Instantly share code, notes, and snippets.

@maltzsama
Last active March 23, 2023 14:21
Show Gist options
  • Save maltzsama/53d175926875d9922d0e416081e2b21d to your computer and use it in GitHub Desktop.
Save maltzsama/53d175926875d9922d0e416081e2b21d to your computer and use it in GitHub Desktop.
para inserir o tipo json no postgresql
from pyspark.sql import SparkSession
def init_spark():
spark = SparkSession.builder.config("spark.jars", "/postgresql-42.5.4.jar")\
.master("local").appName("PySpark_Postgres_test").getOrCreate()
sc = spark.sparkContext
return spark,sc
def main():
spark, sc = init_spark()
nums = sc.parallelize([1])
print(nums.map(lambda x: x*x).collect())
data = [("Alice", 25, '{"a": "a"}'), ("Bob", 30, '{"a": "b"}'), ("Charlie", 35, '{"a": "c"}')]
rdd = spark.sparkContext.parallelize(data)
data = rdd.toDF(["nome", "id", "a"])
data.show()
host = "localhost"
database = "teste"
user = "postgre"
password = "passwd"
url = f"jdbc:postgresql://{host}/{database}"
properties = {
"user": user,
"password": password,
"driver": "org.postgresql.Driver",
"stringtype":"unspecified"
}
data.write.jdbc(url=url, table="acc.teste", mode="append", properties=properties)
if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment