pyspark.sh

?

pdf = sdf.select("*").toPandas()

sdf.createOrReplaceTempView("california_housing")

sqlDF = spark.sql("SELECT sum(population) FROM california_housing WHERE total_rooms > 1000")
sqlDF.head()

http://spark.apache.org/docs/latest/sql-getting-started.html

http://spark.apache.org/docs/latest/ml-guide.html

!apt-get install openjdk-8-jdk-headless -qq > /dev/null
!wget -q https://downloads.apache.org/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz
!tar xf spark-2.4.7-bin-hadoop2.7.tgz

!pip install -q findspark
!pip install -q pyarrow

!rm -rf *.tgz
!rm -rf spark-3.0.1-bin-hadoop2.7/

https://relational.fit.cvut.cz/dataset/CCS

https://github.com/hackathonBI/CCS

!wget https://github.com/welly87/spark-load/raw/master/mysql-connector-java-8.0.14.jar
!mv /content/mysql-connector-java-8.0.14.jar /content/spark-2.4.7-bin-hadoop2.7/jars/mysql-connector-java-8.0.14.jar