senderista

class HTTPStream(object):
    """Given a URL which returns chunk-encoded results,
    constructs an iterator over those results (assuming
    individual records are newline-separated)."""
    def __init__(self, url, username="", password=""):
        self.prev_linepart = ""
        self.http_client = HTTPClient()
        self.http_client.fetch(
            url=url,
            auth_username=username,

senderista

class MockStream(object):
    """Given a URL which returns chunk-encoded results,
    constructs an iterator over those results (assuming
    individual records are newline-separated)."""
    def __init__(self, url, username="", password=""):
        self.prev_linepart = ""
        _handle_data("abc\ndef\nghi")
        _handle_data("jkl\nmno\npqr")
        _handle_data("stu\nvwx\nyz")

senderista

BOLD=`echo -e '\033[1m'`
BLACK=`echo -e '\033[30m'`
RED=`echo -e '\033[31m'`
GREEN=`echo -e '\033[32m'`
YELLOW=`echo -e '\033[33m'`
BLUE=`echo -e '\033[34m'`
MAGENTA=`echo -e '\033[35m'`
CYAN=`echo -e '\033[36m'`
WHITE=`echo -e '\033[37m'`
BG_BLACK=`echo -e '\033[40m'`

senderista

#!/usr/bin/env python

import sys
from time import sleep
from twitter import *

t = Twitter(
    auth=OAuth(
        token='XXXXXX',
        token_secret='XXXXXX',

senderista

#!/usr/bin/env python

import sys
import boto.kinesis

region, stream = sys.argv[1:3]
kinesis = boto.kinesis.connect_to_region(region)
response = kinesis.describe_stream(stream)
if response['StreamDescription']['StreamStatus'] == 'ACTIVE':
    shard_id = response['StreamDescription']['Shards'][0]['ShardId']

senderista

worker_processes 1;
error_log logs/error.log;
events {
    worker_connections 1024;
}
http {
    lua_shared_dict token_bucket 10m;

    init_by_lua '
        ngx.shared.token_bucket:set("tokens_avail", 0)

senderista

var PoissonProcess = require('poisson-process');
var sleep = require('sleep');
var http = require('http');
var port = process.env.PORT || 1337;
var TIMESLICE_MILLIS = 1;
var MEAN_SLEEP_MILLIS = 500;

function sleep_for_timeslice(total_ms, ms_left, req_timestamp, res) {
    if (ms_left > 0) {
        sleep.usleep(TIMESLICE_MILLIS*1000);

senderista

worker_processes 1;
error_log logs/error.log;
events {
    worker_connections 1024;
}
http {
    lua_shared_dict token_bucket 10M;
    lua_shared_dict stats 5M;

    init_by_lua '

senderista

#!/usr/bin/env python

import sys
import boto

bucket_name = sys.argv[1]
s3 = boto.connect_s3()
bucket = s3.get_bucket(bucket_name)

for v in bucket.list_versions():

senderista

for s in `cat servers.txt`; do
  echo $s
  ssh username@$s <<-EOF
  sudo -u nobody bash -c "crontab -l | { echo -n '#'; cat; } | crontab -"
  EOF
done