Ruben Berenguel rberenguel

Keybase proof

I hereby claim:

I am rberenguel on github.
I am rberenguel (https://keybase.io/rberenguel) on keybase.
I have a public key ASDoKc_uTbL4lQBaJ4bNh9G-5OkiK2gSxIFF6h0xvuMy8wo

To claim this, I am signing this object:

To take full advantage of the workshop you'll need

PySpark installed (anything more recent than 2.3 should be fine)
Jupyter installed
Pandas and Arrow installed
All able to talk to each other
One or more datasets

You can clone this repository to have the notebook and slides (some things may still change until Saturday, like uploading and upgating the compiled slides, but the notebook is essentially finished).

	import org.apache.spark.sql.types._ // You'll need this to evaluate its output

	object StructFmt {
	def asScala(field: StructField): String = field.dataType match {
	case struct: StructType => s"""StructField("${field.name}",""" + asScala(struct) + s", ${field.nullable})"
	case _ => s"""StructField("${field.name}", ${field.dataType}, ${field.nullable})"""
	}

	def asScala(struct: StructType): String = "StructType(Seq(" + (for(field <- struct) yield asScala(field)).mkString(",") + "))"
	}

	case class Foo(id: String, value: Int)
	case class Bar(theId: String, value: Int)

	val ds = List(Foo("Alice", 42), Foo("Bob", 43)).toDS

	import org.apache.spark.sql.{DataFrame, Dataset}

	val renamedDF: DataFrame = ds.select($"id".as("theId"), $"value")
	val renamedDS: Dataset[Bar] = renamedDF.toDF("theId", "value").as[Bar]

	# -- coding: utf-8 --
	#
	# Author: Vegard Nossum <[email protected]>

	import math
	import os
	import sys

	import cairo

	case class Country(name: String) {
	private val ThreeLetterValidCountries = List("FOO", "BAR")
	def valid: Option[Country] =
	if (ThreeLetterValidCountries.contains(name.toUpperCase)) Some(this) else None
	}
	object Country {
	def apply(name: String): Country = new Country(name.toUpperCase)
	}

	%!PS-Adobe-2.0

	%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
	% %
	% Mandelbrot set via PostScript code. Not optimized %
	% in any way. Centered in A4 paper. Escape time, B&W %
	% %
	%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

	/fun {

	%!PS-Adobe-3.0
	%%BeginFeature: *PageSize A4
	<< /PageSize [595 842] >> setpagedevice
	%%EndFeature

	%/PageSize A4

	/CoordX 595 def
	/CoordY 842 def
	/RadiMax 95 def

	%!PS-Adobe-2.0

	%%% Start of L-system definition

	/STARTK { FK +K +K FK +K +K FK} def
	/FK {
	dup 0 eq
	{ DK } % if the recursion order ends, draw forward
	{
	1 sub % recurse

	;; Copyright 2009 Rubén Berenguel

	;; ruben /at/ maia /dot/ ub /dot/ es

	;; This program is free software: you can redistribute it and/or
	;; modify it under the terms of the GNU General Public License as
	;; published by the Free Software Foundation, either version 3 of the
	;; License, or (at your option) any later version.

	;; This program is distributed in the hope that it will be useful,