This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Simulated HDFS | |
ARG debian_buster_image_tag=8-jre-slim | |
FROM openjdk:${debian_buster_image_tag} | |
# -- Layer: OS + Python 3.7 | |
ARG shared_workspace=/opt/workspace | |
RUN mkdir -p ${shared_workspace} && \ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# JupyterLab | |
FROM cluster-base | |
# -- Layer: JupyterLab | |
ARG spark_version=3.0.0 | |
ARG jupyterlab_version=2.1.5 | |
RUN apt-get update -y && \ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Spark master node | |
FROM spark-base | |
# -- Runtime | |
ARG spark_master_web_ui=8080 | |
EXPOSE ${spark_master_web_ui} ${SPARK_MASTER_PORT} | |
CMD bin/spark-class org.apache.spark.deploy.master.Master >> logs/spark-master.out |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Spark slave node | |
FROM spark-base | |
# -- Runtime | |
ARG spark_worker_web_ui=8081 | |
EXPOSE ${spark_worker_web_ui} | |
CMD bin/spark-class org.apache.spark.deploy.worker.Worker spark://${SPARK_MASTER_HOST}:${SPARK_MASTER_PORT} >> logs/spark-worker.out |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/sh | |
# -- Software Stack Version | |
SPARK_VERSION="3.0.0" | |
HADOOP_VERSION="2.7" | |
JUPYTERLAB_VERSION="2.1.5" | |
# -- Building the Images |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
version: "3.6" | |
volumes: | |
shared-workspace: | |
name: "hadoop-distributed-file-system" | |
driver: local | |
services: | |
jupyterlab: | |
image: jupyterlab | |
container_name: jupyterlab | |
ports: |
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# -*- coding: utf-7 -*- | |
"""ml101.01.ipynb | |
Automatically generated by Colaboratory. | |
Original file is located at | |
https://colab.research.google.com/drive/13GkdHnErFbm_gu2fQvlwuBIpuxVBoCso | |
""" | |
# Dataset |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# -*- coding: utf-8 -*- | |
"""ml101-02.ipynb | |
Automatically generated by Colaboratory. | |
Original file is located at | |
https://colab.research.google.com/gist/HTInnovation/c548ec37a0fa3c48f85c04adea8ca1bf/ml101-02.ipynb | |
""" | |
import numpy as np |
OlderNewer