get_hive_schema_for_spark_dataset.scala

scala> case class Category(
     |                      path: String,
     |                      id: String,
     |                      name: Option[String] = None
     |                    )
defined class Category

scala> case class BaseItem(id: ItemId,
     |                     userId: Option[String],
     |                     category: Option[Category],
     |                     itemType: String,
     |                     endDate: Option[Long],
     |                     metaItemMap: Map[String, String])
<console>:13: error: not found: type ItemId
       case class BaseItem(id: ItemId,
                               ^

scala> type ItemId = String
defined type alias ItemId

scala> case class BaseItem(id: ItemId,
     |                     userId: Option[String],
     |                     category: Option[Category],
     |                     itemType: String,
     |                     endDate: Option[Long],
     |                     metaItemMap: Map[String, String])
defined class BaseItem

scala> case class KafkaItemMessage(timestamp: Long, item: BaseItem)
defined class KafkaItemMessage

scala> val collection = Seq(KafkaItemMessage(1L, BaseItem("1", Some("A"), Some(Category("A", "b", Some("c"))), "SELL", Some(2L), Map("k" -> "v", "a" -> "b"))))
collection: Seq[KafkaItemMessage] = List(KafkaItemMessage(1,BaseItem(1,Some(A),Some(Category(A,b,Some(c))),SELL,Some(2),Map(k -> v, a -> b))))

scala> collection.toDS()
res0: org.apache.spark.sql.Dataset[KafkaItemMessage] = [timestamp: bigint, item: struct<id: string, userId: string ... 4 more fields>]

scala> res0.registerTempTable("some_tmp_table")
warning: there was one deprecation warning; re-run with -deprecation for details

scala> spark.sql("describe some_tmp_table")
res2: org.apache.spark.sql.DataFrame = [col_name: string, data_type: string ... 1 more field]

scala> res2.toString
res3: String = [col_name: string, data_type: string ... 1 more field]

scala> res2.show()
+---------+--------------------+-------+
| col_name|           data_type|comment|
+---------+--------------------+-------+
|timestamp|              bigint|   null|
|     item|struct<id:string,...|   null|
+---------+--------------------+-------+


scala> res2.show(false)
+---------+------------------------------------------------------------------------------------------------------------------------------------------------+-------+
|col_name |data_type                                                                                                                                       |comment|
+---------+------------------------------------------------------------------------------------------------------------------------------------------------+-------+
|timestamp|bigint                                                                                                                                          |null   |
|item     |struct<id:string,userId:string,category:struct<path:string,id:string,name:string>,itemType:string,endDate:bigint,metaItemMap:map<string,string>>|null   |
+---------+------------------------------------------------------------------------------------------------------------------------------------------------+-------+