Skip to content

Instantly share code, notes, and snippets.

@frgomes
Last active February 25, 2023 01:42
Show Gist options
  • Save frgomes/a32dccdaa602f4c8e6fa8196bc73637e to your computer and use it in GitHub Desktop.
Save frgomes/a32dccdaa602f4c8e6fa8196bc73637e to your computer and use it in GitHub Desktop.
Scala / Spark: allows obtaining schema from runtime TypeTag obtained from class loaded over the wire
import scala.reflect.runtime.universe._
import scala.reflect.api
def runtimeTypeTagOf(name: String, parent: ClassLoader, url: java.net.URL): TypeTag[_] =
runtimeTypeTagOf(name, parent, Seq(url))
def runtimeTypeTagOf(name: String, parent: ClassLoader, urls: Seq[java.net.URL]): TypeTag[_] =
runtimeTypeTagOf(name, new java.net.URLClassLoader(urls.toArray, parent))
def runtimeTypeTagOf(name: String, cl: ClassLoader): TypeTag[_] = {
val c = Class.forName(name, true, cl)
val mirror: Mirror = runtimeMirror(cl)
val sym = mirror.staticClass(name)
val tpe = sym.selfType
TypeTag(mirror, new api.TypeCreator {
def apply[U <: api.Universe with Singleton](m: api.Mirror[U]) =
if (m eq mirror)
tpe.asInstanceOf[U # Type]
else
throw new IllegalArgumentException(s"TypeTag cannot be migrated to other mirrors: ${mirror}")
})
}
def runtimeTypeOf(name: String, parent: ClassLoader, url: java.net.URL): `Type` =
runtimeTypeOf(runtimeTypeTagOf(name, parent, url))
def runtimeTypeOf(name: String, parent: ClassLoader, urls: Seq[java.net.URL]): `Type` =
runtimeTypeOf(runtimeTypeTagOf(name, parent, urls))
def runtimeTypeOf(name: String, cl: ClassLoader): `Type` =
runtimeTypeOf(runtimeTypeTagOf(name, cl))
def runtimeTypeOf(tag: TypeTag[_]): `Type` =
tag.in(tag.mirror).tpe.dealias
@frgomes
Copy link
Author

frgomes commented Feb 25, 2023

// Example of usage

val name = "com.example.entities.Person"
val url = new java.net.URL("http://example.com/entities.jar")

import org.apache.spark.sql.SparkSession
val spark: SparkSession = ???
val runtimeType = runtimeTypeOf(name, spark.getClass.getClassLoader, url)

import org.apache.spark.sql.types._
import org.apache.spark.sql.catalyst.ScalaReflection

val schema: StructType = ScalaReflection.schemaFor(runtimeType).dataType.asInstanceOf[StructType]

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment