bepcyc · June 19, 2020 15:39
diff --git a/spark_extract_json_field_schema.scala b/spark_extract_json_field_schema.scala
 val path = "s3://some/dir"
 val df = spark.read.parquet(path)
 val df2 = df.select($"value") // suppose value is a string with JSON
 val ds = df2.as[String]
 val dsj = spark.read.json(ds)
 val schema = dsj.schema // here is your schema
 println(schema.json)
 println(schema.toDDL)
	val path = "s3://some/dir"
	val df = spark.read.parquet(path)
	val df2 = df.select($"value") // suppose value is a string with JSON
	val ds = df2.as[String]
	val dsj = spark.read.json(ds)
	val schema = dsj.schema // here is your schema
	println(schema.json)
	println(schema.toDDL)