Jupyter kernel for PySpark on Yarn

kernel.json

{
  "display_name": "PySpark",
  "language": "python",
    "argv": [
      "/home/rnederhand/venv/bin/python",
      "-m",
      "ipykernel",
      "-f",
      "{connection_file}"
    ],
    "env": {
      "HADOOP_CONF_DIR": "/etc/hadoop/conf",
      "SPARK_HOME": "/usr/hdp/current/spark-client",
      "PYTHONPATH": "/usr/hdp/2.3.4.0-3485/spark/python/:/usr/hdp/2.3.4.0-3485/spark/python/lib/py4j-0.9-src.zip",
      "PYTHONSTARTUP": "/usr/hdp/current/spark-client/python/pyspark/shell.py",
      "PYSPARK_SUBMIT_ARGS": "--master yarn-client --queue analysis pyspark-shell"
    }
}