TomLous

val targetDockerJarPath = "/opt/spark/jars"

// For building the docker image
lazy val dockerSettings = Seq(
  imageNames in docker := Seq(
    ImageName(s"$domain/${name.value}:latest"),
    ImageName(s"$domain/${name.value}:${version.value}"),
  ),
  buildOptions in docker := BuildOptions(
    cache = false,

TomLous

val domain = "graphiq"

// For building the FAT jar
lazy val assemblySettings = Seq(
  assembly / assemblyOption := (assemblyOption in assembly).value.copy(includeScala = false),
  assembly / assemblyOutputPath := baseDirectory.value / "output" / s"${domain}-${name.value}.jar"
)

TomLous

val selectAllQuery = SparQLQuery("SELECT ?s ?p ?o WHERE {?s ?p ?0 .}")

// ???
for{
  config <- [load config]
  connection <- [connect using config]
  json <- [query using connection]
}

TomLous

val sparkVersion = "2.4.4"

val sparkLibs = Seq(
  "org.apache.spark" %% "spark-core" % sparkVersion % "provided",
  "org.apache.spark" %% "spark-sql" % sparkVersion % "provided"
)

lazy val commonSettings = Seq(
  organization := "xyz.graphiq",
  scalaVersion := "2.12.10",

TomLous

import org.apache.spark.sql._
import org.apache.spark.sql.functions._
import xyz.graphiq.model._

object BasicSparkJob extends App {

  val spark: SparkSession = SparkSession
    .builder()
    .config(
      new SparkConf().setIfMissing("master", "local[*]")

TomLous

defaults write com.apple.finder AppleShowAllFiles YES
defaults write com.apple.Finder AppleShowAllFiles true

# TODO: Copy id_rsa keys to .ssh (copy from file)
chmod 600 ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa.pub

# Homebrew
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"

TomLous

 "display_name": "PySpark",
 "language": "python",
 "argv": [
  "[python bin]",
  "-m",
  "ipykernel",
  "-f",
  "{connection_file}"
 ],

TomLous

import com.mongodb.spark.MongoSpark
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.functions._

val spark = SparkSession.builder()
  .master("local[2]")
  .appName("test")
  .config("spark.mongodb.input.uri", "mongodb://127.0.0.1/dbname")
  .config("spark.mongodb.output.uri", "mongodb://127.0.0.1/dbname")
  .getOrCreate()

TomLous

    import scala.util._
    import scala.concurrent._
    import scala.concurrent.duration.Duration
    import scala.concurrent.ExecutionContext.Implicits.global

    import com.datlinq.datafiniti.response.DatafinitiError
    import com.datlinq.datafiniti.config.DatafinitiAPITypes.Businesses

    import cats.implicits._
    import org.json4s._

TomLous

import doobie.imports._
import fs2.interop.cats._
import cats.implicits._

// Connection to localhost
val xa = DriverManagerTransactor[IOLite]("com.mysql.cj.jdbc.Driver", "jdbc:mysql://localhost:3306/test?useUnicode=true&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=UTC", "root", "")

// Equivalent of table structure
case class Item(id: String, name: String, fetced: Boolean)