jovianlin

# The PiStation Case (for the Raspberry Pi 4)
# is a well built fun case, especially for those
# with a passion for retro-gaming consoles.
# Personally, I love it.
#
# - https://shop.pimoroni.com/products/pistation-case
#
# But installing the 'safe shutdown' feature may get you frustrated.
# The hardware's brilliant, but the shutdown script installation
# needs a little work. There are a number of

jovianlin

class MaxHeapObj:
    def __init__(self,val): self.val = val
    def __lt__(self,other): return self.val > other.val
    def __eq__(self,other): return self.val == other.val
    def __str__(self): return str(self.val)

class MinHeap:
    def __init__(self): self.h = []
    def heappush(self,x): heapq.heappush(self.h,x)
    def heappop(self): return heapq.heappop(self.h)

jovianlin

from __future__ import print_function
import sys

if sys.version_info[0] >= 3:
  from urllib.request import urlretrieve
else:
  from urllib import urlretrieve

local_filename, headers = urlretrieve('http://jovianlin.com')
html = open(local_filename)

jovianlin

import random, sys, time
from termcolor import colored
       
colors = ['grey', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', None]
generated = ''
sys.stdout.write(generated)

for i in range(100):
    sys.stdout.write(colored('%02d ' % i, random.choice(colors)))
    sys.stdout.flush()

jovianlin

import sys

def is_kernel():
    """
    Determine if we're in an IPython notebook session
    Source: Source: http://stackoverflow.com/a/34092072
    -------------------------------------------------
    You can't detect that the frontend is a notebook with perfect precision, 
    because an IPython Kernel can have one or more different Jupyter frontends 
    with different capabilities (terminal console, qtconsole, notebook, etc.). 

jovianlin

from pyspark import SparkContext
from pyspark.sql import HiveContext
from graphframes.examples import Graphs

sc = SparkContext()
sc.setLogLevel("ERROR")
sqlContext = HiveContext(sc)


g = Graphs(sqlContext).friends()  # Get example graph

jovianlin

# Soure http://stackoverflow.com/questions/30214474/how-to-run-multiple-jobs-in-one-sparkcontext-from-separate-threads-in-pyspark
# Prereqs:
# set 
# spark.dynamicAllocation.enabled         true
# spark.shuffle.service.enabled           true
# in spark-defaults.conf

import threading
from pyspark import SparkContext, SparkConf

jovianlin

# Write DataFrame to Disk
spark_df.coalesce(1).write.csv( '<saved_output/YOUR_FOLDER_NAME>', header=True, mode='overwrite' )

# Read from Disk to DataFrame
new_spark_df = sqlContext.read.csv(s3_path,      header=True, inferSchema=False) # For S3
new_spark_df = sqlContext.read.csv('<LOCATION>', header=True, inferSchema=False) # mode='FAILFAST'

# SORTING
from pyspark.sql.functions import col
col_name = 'restaurant_id'

jovianlin

#############
# VARIABLES #
#############
access_key_id = '<ACCESS KEY ID>'
secret_access_key = '<SOME SECRET SHIT>'

bucket_name = 'my-awesome-bucket'
folder_name = 'upload_folder'
file_name   = 'uploaded_doge_shit.jpg'

jovianlin

find . -name ".ipynb_checkpoints" | grep ipynb