Skip to content

Instantly share code, notes, and snippets.

@DmytroMitin
Created February 23, 2023 15:35
Show Gist options
  • Select an option

  • Save DmytroMitin/137c622e6782d9b7a2dacab059fcfdfb to your computer and use it in GitHub Desktop.

Select an option

Save DmytroMitin/137c622e6782d9b7a2dacab059fcfdfb to your computer and use it in GitHub Desktop.
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.functions.typedLit
object App13 {
val spark: SparkSession = SparkSession.builder.master("local[*]").appName("Example").getOrCreate
import spark.implicits._
val columns = Seq("language", "users_count")
val data = Seq(("Java", "20000"), ("Python", "100000"), ("Scala", "3000"))
val df = spark.createDataFrame(data).toDF(columns: _*)
val df_with_col = df.withColumn("raw", typedLit[Option[Int]](None))
df_with_col.show()
def main(args: Array[String]): Unit = ()
}
sbt.version = 1.8.2
ThisBuild / version := "0.1.0-SNAPSHOT"
ThisBuild / scalaVersion := "2.13.10"
lazy val root = (project in file("."))
.settings(
name := "scalademo35"
)
libraryDependencies += "org.apache.spark" %% "spark-sql" % "3.3.1"
libraryDependencies += scalaOrganization.value % "scala-compiler" % scalaVersion.value
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment