tommydangerous

# https://www.digitalocean.com/community/tutorials/how-to-install-ruby-on-rails-with-rvm-on-ubuntu-16-04
$ gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
$ cd /tmp
$ curl -sSL https://get.rvm.io -o rvm.sh
$ ./rvm.sh
$ cat /tmp/rvm.sh | bash -s stable --rails
$ source /home/ubuntu/.rvm/scripts/rvm
$ rvm list known
$ rvm install 2.5

tommydangerous

# https://www.anaconda.com/download/#linux
$ ssh jpy
$ mkdir tmp
$ cd tmp
$ wget https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh
$ bash Anaconda3-5.0.1-Linux-x86_64.sh
$ vi ~/.bashrc

# export PATH=~/anaconda3/bin:$PATH

tommydangerous

 Cannot start service app: connection error: desc = "transport: dial unix /var/run/docker/containerd/docker-containerd.sock: connect: connection refused"

$ sudo systemctl daemon-reload
$ sudo systemctl restart docker

tommydangerous

from pyspark.sql import SparkSession


def load_data(spark, s3_location):
    """
    spark:
        Spark session
    s3_location:
        S3 bucket name and object prefix
    """

tommydangerous

from pyspark.sql.functions import pandas_udf, PandasUDFType


@pandas_udf(
    SCHEMA_COMING_SOON,
    PandasUDFType.GROUPED_MAP,
)
def custom_transformation_function(df):
    pass

tommydangerous

from pyspark.sql.functions import pandas_udf, PandasUDFType
from pyspark.sql.types import (
    IntegerType,
    StringType,
    StructField,
    StructType,
)


"""

tommydangerous

from pyspark.sql.functions import pandas_udf, PandasUDFType
from pyspark.sql.types import (
    IntegerType,
    StringType,
    StructField,
    StructType,
)


"""

tommydangerous

from pyspark.sql import SparkSession
from pyspark.sql.functions import pandas_udf, PandasUDFType
from pyspark.sql.types import (
    IntegerType,
    StringType,
    StructField,
    StructType,
)

tommydangerous

from sklearn.model_selection import train_test_split
import pandas as pd

df = pd.read_csv('/content/titanic_survival.csv')
label_feature_name = 'Survived'

X = df.drop(columns=[label_feature_name])
y = df[label_feature_name]

tommydangerous

X_train_raw, X_test_raw, y_train, y_test = train_test_split(
    X,
    y,
    stratify=y,
    test_size=0.2,
)