squito

import scala.collection.mutable.Map

import org.apache.spark.{Accumulator, AccumulatorParam, SparkContext}
import org.apache.spark.scheduler.{SparkListenerStageCompleted, SparkListener}
import org.apache.spark.SparkContext._


/**
 * just print out the values for all accumulators from the stage.
 * you will only get updates from *named* accumulators, though

wpf500

#!/usr/bin/python
import sys, boto.logs, time
from datetime import datetime

log_group = 'LOG GROUP NAME'
timestamp = int(datetime.strptime(sys.argv[1], '%Y-%m-%dT%H:%M:%S').strftime('%s')) * 1000

logs = boto.logs.connect_to_region('eu-west-1')

if len(sys.argv) > 2:

mkrcah

import sys
from kazoo.client import KazooClient

if len(sys.argv) not in (2,3):
    print('Usage: zkDelAll.py [path] [host:port=localhost:2181]')
    exit(1)

host = sys.argv[2] if len(sys.argv) == 3 else 'localhost:2181'
path = sys.argv[1]

samklr

val ts = sc.parallelize(0 to 100, 10)
val window = 3

class StraightPartitioner(p: Int) extends Partitioner {
  def numPartitions = p
  def getPartition(key: Int) = key * p/0.5
}

val partitioned = ts.mapPartitionsWithIndex((i, p) => {
  val overlap = p.take(window - 1).toArray

mik01aj

#!/bin/bash

# Usage: ./gen.sh collected-stacks.txt

TMPSTACKS=/tmp/flamegraph-stacks-collapsed.txt
TMPPALETTE=/tmp/flamegraph-palette.map

./stackcollapse-jstack.pl $1 > $TMPSTACKS

# 1st run - hot: default

nicolasembleton

#!/bin/bash

sudo kextunload -b com.apple.iokit.BroadcomBluetoothHostControllerUSBTransport
sudo kextload -b com.apple.iokit.BroadcomBluetoothHostControllerUSBTransport

acolyer

Internet Scale Services Checklist

A checklist for designing and developing internet scale services, inspired by James Hamilton's 2007 paper "On Desgining and Deploying Internet-Scale Services."

• http://mvdirona.com/jrh/talksandpapers/jamesrh_lisa.pdf

Basic tenets

• Does the design expect failures to happen regularly and handle them gracefully?
• Have we kept things as simple as possible?

bigsnarfdude

// transcribed from an Apache Spark 1.0 spark-shell session
// using data from http://chriswhong.com/open-data/foil_nyc_taxi/
// and the QTree algorithm for approximate quantiles over large datasets
// each of the distanceRange and minutesRange calculations below takes about 15 minutes on my four-core SSD-based Macbook Pro
 
import com.twitter.algebird._
import com.twitter.algebird.Operators._
implicit val qtSemigroupD = new QTreeSemigroup[Double](6)
 
val in = sc.textFile("trip_data") // a directory containing all the trip_data*.csv files downloaded from the above link

debasishg

1. Feature Learning

• Learning Feature Representations with K-means by Adam Coates and Andrew Y. Ng
• The devil is in the details: an evaluation of recent feature encoding methods by Chatfield et. al.
• Emergence of Object-Selective Features in Unsupervised Feature Learning by Coates, Ng
• Scaling Learning Algorithms towards AI Benjio & LeCun
• A Theory of Feature Learning by Brendan van Rooyen, Robert C. Williamson

2. Deep Learning

• Dropout: A Simple Way to Prevent Neural Networks from Overfitting by Nitish Srivastava, Geoffrey Hinton, Alex Krizhevsky, Ilya Sutskever and Ruslan Salakhutdinov
• [Understanding

asimjalis

import java.io.*;

import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.lib.output.*;
import org.apache.hadoop.mapreduce.lib.partition.*;
import org.apache.hadoop.mapreduce.lib.reduce.*;