dslaw

Add packages to the Spark session from within python. They'll be automatically downloaded if they're not already available.

# Using `JsonSerDe` as an example.
os.environ["PYSPARK_SUBMIT_ARGS"] = (
   '--packages "org.apache.hive.hcatalog.data.JsonSerDe" pyspark-shell'
)
spark = SparkSession.builder.config().getOrCreate()

	#!/usr/bin/env python3

	""" Binary search tree with method to invert in-place. """

	class Node(object):
	def __init__(self, x):
	self.value = x
	self.left = None
	self.right = None

	#include "btree.h"


	template <typename T>
	void Btree<T>::insert(const T &x) {
	if (root == nullptr) {
	node_ptr node = std::make_unique<Node>(x);
	root = std::move(node);
	return;
	}

	def chunk(xs, n):
	""" Split an iterable into chunks.

	Parameters
	----------
	xs : iterable
	Iterable to be split.
	n : int
	Chunk size.

	from collections import OrderedDict


	class LRUCache(object):
	def __init__(self, max_size=4):
	if max_size <= 0:
	raise ValueError

	self.max_size = max_size
	self._items = OrderedDict()

	from scipy.stats import norm

	import numpy as np
	import matplotlib.pyplot as plt


	plt.style.use("ggplot")


	def save_and_close(fig, filename):

	# open terminal
	cmd - return : osascript -e 'tell application "iTerm" to create window with default profile'

	# enter fullscreen mode for the focused container
	# cmd + f clobbers text search in firefox.
	shift + cmd - f : yabai -m window --toggle zoom-fullscreen

	# kill focused window
	shift + cmd - q : yabai -m window --close

	"""Parse data element numbers as trees."""
	# XXX: Second spectral digit is broken - both values are 0!

	from dataclasses import dataclass
	from lxml import html
	from typing import Any, Dict, List


	DELIMITER = "\x97" #
	MIN_COLSPAN = 1

	import boto3


	s3_client = boto3.client("s3")

	# List available operations.
	s3_client._service_model.operation_names

	operation_name = "ListObjectsV2"
	params = {