Skip to content

Instantly share code, notes, and snippets.

@francescofrontera
Last active February 15, 2019 14:28
Show Gist options
  • Save francescofrontera/cd457b32c6b1e255dfc35ea63f777ba7 to your computer and use it in GitHub Desktop.
Save francescofrontera/cd457b32c6b1e255dfc35ea63f777ba7 to your computer and use it in GitHub Desktop.
Serialize Enum using spark standar implicits
sealed case class EvaluableData(
value: BigDecimal,
threshold: Threshold,
serviceClass: String,
serviceInstance: String,
triggeredEvent: String
)
sealed case class Threshold(criteria: Criteria.CriteriaValue,
critical: BigDecimal,
major: BigDecimal,
minor: BigDecimal)
object Criteria {
sealed case class CriteriaValue(value: String)
object LessThan extends CriteriaValue("LessThan")
object MoreThan extends CriteriaValue("MoreThan")
val values: Vector[CriteriaValue] = Vector(LessThan, MoreThan)
}
object SerializationTest extends App with SparkSessionWrapper {
import spark.implicits._
val dataType = EvaluableData(BigDecimal(10),
Threshold(Criteria.LessThan,
BigDecimal(10),
BigDecimal(4),
BigDecimal(1)),
"a",
"b",
"c") :: Nil
spark
.createDataset(dataType)
.show()
}
trait SparkSessionWrapper extends Serializable {
lazy val spark: SparkSession = {
SparkSession.builder().master("local").appName("spark session").getOrCreate()
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment