install_spark.sh

#!/bin/bash

SPARK_VERSION='spark-3.1.2'
SPARK_URL=https://archive.apache.org/dist/spark/$SPARK_VERSION/$SPARK_VERSION-bin-without-hadoop.tgz

echo "Downloading pre-built PySpark..."
wget $SPARK_URL -P /tmp
echo "Done!"

# Unpack PySpark
echo "Creating folder /home/ec2-user/SageMaker/spark..."
mkdir -p /home/ec2-user/SageMaker/spark
echo "Untar spark bundle..."
tar -xvf /tmp/$SPARK_VERSION-bin-without-hadoop.tgz -C /home/ec2-user/SageMaker/spark --strip-components=1
echo "Done"