kernel.json

{
 "argv": [
  "spark-launch-ipykernel",
  "-f",
  "{connection_file}"
 ],
 "display_name": "Python 3 (ipykernel)",
 "language": "python",
 "metadata": {
  "debugger": true
 }
}

spark-launch-ipykernel

#!/bin/sh
BINDIR="$(dirname "$0")"
PYSPARK_PYTHON="${BINDIR}/python"
PYSPARK_DRIVER_PYTHON="${BINDIR}/python"
SPARK_HOME="${HOME}/opt/spark-3.2.1-bin-hadoop3.2"
JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk-11.0.15+10/Contents/Home
export PYSPARK_PYTHON
export PYSPARK_DRIVER_PYTHON
export SPARK_HOME
export JAVA_HOME
IPYKERNEL_LAUNCHER="$("${PYSPARK_PYTHON}" -c 'import ipykernel_launcher; print(ipykernel_launcher.__file__)')"
"${SPARK_HOME}/bin/spark-submit" \
    "--conf" "spark.driver.memory=4g" \
    "--conf" "spark.executor.memory=4g" \
    "${IPYKERNEL_LAUNCHER}" "$@"