Skip to content

Instantly share code, notes, and snippets.

@j3speaks
j3speaks / Dockerfile
Created May 17, 2019 20:28
Dockerize Spark
FROM openjdk:8-alpine
RUN apk --update add wget tar bash python
RUN wget http://apache.mirror.anlx.net/spark/spark-2.4.2/spark-2.4.2-bin-hadoop2.7.tgz
RUN tar -xzf spark-2.4.2-bin-hadoop2.7.tgz && mv spark-2.4.2-bin-hadoop2.7 spark && rm spark-2.4.2-bin-hadoop2.7.tgz
RUN printf "#!/bin/sh\n/spark/bin/spark-class org.apache.spark.deploy.master.Master --host \$SPARK_MASTER_HOST --port \$SPARK_MASTER_PORT --webui-port \$SPARK_MASTER_WEBUI_PORT" > /start-master.sh
RUN chmod +x /start-master.sh
RUN printf "#!/bin/sh\n/spark/bin/spark-class org.apache.spark.deploy.worker.Worker --webui-port \$SPARK_WORKER_WEBUI_PORT \$SPARK_MASTER_URL" > /start-worker.sh
RUN chmod +x /start-worker.sh
COPY cloudera-10k.txt /cloudera-10k.txt
COPY employee.txt /spark/examples/src/main/scala/org/apache/spark/examples/sql/employee.txt
@j3speaks
j3speaks / caesar_cipher.py
Created May 17, 2019 20:24
Caesar Cipher written in Python
# Constant Values
DEFAULT_SHIFT = 1
MAX_ALPHABET_COUNT = 26
UPPERCASE_A_ASCII_CODE = 65
LOWERCASE_A_ASCII_CODE = 97
def encode(plaintext, shift=DEFAULT_SHIFT):
if not plaintext.isalpha():
raise ValueError("Invalid 'plaintext' argument value %s; it is not all alpha." % plaintext)
@j3speaks
j3speaks / bridge-torch-puzzle.go
Created May 17, 2019 20:09
Bridge and Torch Logic Puzzle solved in Go
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"sort"
"strings"
)
@j3speaks
j3speaks / scs-tp_test.go
Last active May 17, 2019 17:30
Bridge and Torch Logic Puzzle (written in Go)
package main
import (
"fmt"
"sort"
"testing"
)
func TestNumberOfParticipants(t *testing.T) {
// Initialize our Participants array
@j3speaks
j3speaks / input.json
Last active May 16, 2019 14:59
Bridge and Torch Logic Puzzle (written in Go)
{
"participants":[
{
"name": "Sam",
"speed": 5
},
{
"name": "Mouna",
"speed": 1
},
@j3speaks
j3speaks / scs-tp.go
Last active May 17, 2019 16:40
Bridge and Torch Logic Puzzle (written in Go)
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"sort"
"strings"
)
@j3speaks
j3speaks / docker-compose.yml
Last active May 7, 2019 02:22
docker-compose.yml for running a Spark cluster in Docker
version: "3.7"
services:
spark-master:
image: $MYNAME/spark:latest
container_name: spark-master
hostname: spark-master
build:
context: .
dockerfile: Dockerfile
ports:
@j3speaks
j3speaks / Dockerfile
Last active May 7, 2019 02:23
Spark in a Docker container
FROM openjdk:8-alpine
RUN apk --update add wget tar bash
RUN wget http://apache.mirror.anlx.net/spark/spark-2.4.2/spark-2.4.2-bin-hadoop2.7.tgz
RUN tar -xzf spark-2.4.2-bin-hadoop2.7.tgz && mv spark-2.4.2-bin-hadoop2.7 spark && rm spark-2.4.2-bin-hadoop2.7.tgz
RUN printf "#!/bin/sh\n/spark/bin/spark-class org.apache.spark.deploy.master.Master --host \$SPARK_MASTER_HOST --port \$SPARK_MASTER_PORT --webui-port \$SPARK_MASTER_WEBUI_PORT" > /start-master.sh
RUN chmod +x /start-master.sh
RUN printf "#!/bin/sh\n/spark/bin/spark-class org.apache.spark.deploy.worker.Worker --webui-port \$SPARK_WORKER_WEBUI_PORT \$SPARK_MASTER_URL" > /start-worker.sh
RUN chmod +x /start-worker.sh