pyspark.sh

?

!apt-get install openjdk-8-jdk-headless -qq > /dev/null
!wget -q https://downloads.apache.org/spark/spark-3.0.1/spark-3.0.1-bin-hadoop2.7.tgz
!tar xf spark-3.0.1-bin-hadoop2.7.tgz
!pip install -q findspark
!pip install -q pyarrow

import os
os.environ["JAVA_HOME"] = "/usr/lib/jvm/java-8-openjdk-amd64"
os.environ["SPARK_HOME"] = "/content/spark-3.0.1-bin-hadoop2.7"

check spark version
!spark-3.0.1-bin-hadoop2.7/bin/spark-shell -version

import findspark
findspark.init("spark-3.0.1-bin-hadoop2.7")

from pyspark.sql import SparkSession

spark = SparkSession \
    .builder \
    .appName("Python Spark SQL basic example") \
    .config("spark.some.config.option", "some-value") \
    .getOrCreate()

sdf = spark.read.csv("/content/sample_data/california_housing_train.csv", header=True)

pdf = sdf.select("*").toPandas()

sdf.createOrReplaceTempView("california_housing")

sqlDF = spark.sql("SELECT sum(population) FROM california_housing WHERE total_rooms > 1000")
sqlDF.head()

http://spark.apache.org/docs/latest/sql-getting-started.html

http://spark.apache.org/docs/latest/ml-guide.html

!apt-get install openjdk-8-jdk-headless -qq > /dev/null
!wget -q https://downloads.apache.org/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz
!tar xf spark-2.4.7-bin-hadoop2.7.tgz

!pip install -q findspark
!pip install -q pyarrow

!rm -rf *.tgz
!rm -rf spark-3.0.1-bin-hadoop2.7/

https://relational.fit.cvut.cz/dataset/CCS

https://github.com/hackathonBI/CCS

!wget https://github.com/welly87/spark-load/raw/master/mysql-connector-java-8.0.14.jar
!mv /content/mysql-connector-java-8.0.14.jar /content/spark-2.4.7-bin-hadoop2.7/jars/mysql-connector-java-8.0.14.jar