All the ways to create a Spark Dataframe (pyspark)

all the ways to create spark dataframe.py

data = [
  ("a", "Alice", 34),
  ("b", "Bob", 36),
  ("c", "Charlie", 30),
]

df = spark.createDataFrame(data, schema=["id", "name", "age"])

dict.py

from pyspark.sql import Row

data = [
  {"id": "a", "name": "Alice", "age": 34},
  {"id": "b", "name": "Bob", "age": 36},
  {"id": "c", "name": "Charlie", "age": 30},
]

df = spark.createDataFrame(Row(**x) for x in data)

pandas.py

import pandas as pd 

data = [['a', 'Alice', 34], ['b', 'Bob', 36], ['c', 'Charlie', 30]] 

df = spark.createDataFrame(pd.DataFrame(data, columns = ['id', 'name', 'age']))

rdd.py

from pyspark.sql import Row
l = [('a',"Alice", 33),("b", "Bob", 36),("c", "Charlie", 30)]
rdd = sc.parallelize(l)
rows = rdd.map(lambda x: Row(id=x[0], name=x[1], age=int(x[2])))
df = spark.createDataFrame(rows)

spark.read.json.py

alice = '''{"id":"a", "name": "Alice", "age":34}'''
bob = '''{"id":"b", "name": "Bob", "age":36}'''
charlie = '''{"id":"c", "name": "Charlie", "age":30}'''

rdd = sc.parallelize([alice, bob, charlie])
df = spark.read.json(rdd)

spark.sql.py

df = spark.sql("""
select 'a' id, 'Alice' name, 34 age
union
select 'b', 'Bob', 36 age
union
select 'c', 'Charlie', 30 age
""")