connection.R

library(sparklyr)
Sys.setenv(PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin")
Sys.setenv(SPARK_HOME_VERSION = "2.3.0")

# Run before starting a new session: pkill kubectl

sc <- spark_connect(
  master = "k8s://http://10.1.1.1:8080",
  config = list(
    sparklyr.shell.master = "k8s://http://10.1.1.1:8080",
    "sparklyr.shell.deploy-mode" = "cluster",
    sparklyr.gateway.remote = TRUE,
    sparklyr.shell.name = "sparklyr",
    sparklyr.shell.class = "sparklyr.Shell",
    sparklyr.shell.conf = c(
      "spark.kubernetes.container.image=localhost:50000/spark:latest",
      "spark.kubernetes.driver.pod.name=spark-driver13",
      "spark.kubernetes.authenticate.driver.serviceAccountName=spark",
      "spark.executor.instances=2"
    ),
    sparklyr.app.jar = "local:///opt/spark/sparklyr/sparklyr-2.3-2.11.jar",
    sparklyr.events.aftersubmit = function() {
      # wait for pods to launch
      message("Trying to connect")
      Sys.sleep(15)
      # configure port forwarding
      system2(
        "kubectl",
        c("port-forward", "spark-driver13", "8880:8880", "8881:8881", "4040:4040"),
        wait = FALSE
      )
    }
  ),
  spark_home = "/opt/spark", version = "2.3.0"
)

spark_mtcars <- sdf_copy_to(sc, mtcars)

spark_mtcars