tomdottom

Basic task processing with distributed rate limiting

Proof of concept of distributed rate limiting multiple workers processing speed.

Rate limiting follows a leaky bucket algorithim. The bucket is implemented using a speical token-bucket queue. Max size of the bucket is enforced by using the max length of a token queue. The bucket is refilled by a single worker, which also is responsible for the refill rate.

Workers must get a token before fetching and processing any tasks.

tomdottom

"""demo.py

Distributed rate limited task queues.
Uses a leaky-bucket token-queue to ensure multiple workers don't exceed global rate limit.

# Setup & Dependencies
docker run -d --hostname my-rabbit --name some-rabbit -p 5672:5672 rabbitmq:3
pip install kombu

# Term 1 & 2

tomdottom

from types import SimpleNamespace
from io import StringIO

from flask.sessions import SecureCookieSessionInterface

session_cookie = "... get me from your browser ..."
secret_key = "... get me from your app settings ..."

app = SimpleNamespace(secret_key=secret)
session = SecureCookieSessionInterface()

tomdottom

# -*- coding: utf-8 -*-
# Author: Douglas Creager <[email protected]>
# This file is placed into the public domain.

# Calculates the current version number.  If possible, this is the
# output of “git describe”, modified to conform to the versioning
# scheme that setuptools uses.  If “git describe” returns an error
# (most likely because we're in an unpacked copy of a release tarball,
# rather than in a git working copy), then we fall back on reading the
# contents of the RELEASE-VERSION file.

tomdottom

# As root prepare build user
apk update
apk add sudo
adduser -D build
addgroup build abuild
addgroup abuild root
echo 'build ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers

# Switch to build user
su build

tomdottom

import mock
import sys

# Create a mock `uwsgi` module so that
# import uwsgi does not throw an ImportError

uwsgi = mock.MagicMock()
sys.modules.setdefault('uwsgi', uwsgi)

uwsgidecorators = mock.MagicMock()

tomdottom

#!/usr/bin/env python

# Python version of https://gist.github.com/tonycoco/661615

import csv
import sys

import untangle

usage = '''

tomdottom

[tox]
envlist = py27

[testenv]
skipdist=True
skip_install=True
deps=
  nose
  wheel
commands=

tomdottom

#!/bin/bash

while read oldrev newrev refname; do
  echo "$oldrev $newrev $refname"
  # your code
done

tomdottom

wget https://launchpad.net/openssh/main/6.6p1/+download/openssh-6.6p1.tar.gz

tar -xvf openssh-6.6p1.tar.gz

cd openssh-6.6p1

./configure --prefix=/usr --sysconfdir=/etc/ssh --with-privsep-path=/var/run/sshd --with-default-path=/usr/local/bin:/usr/bin:/bin --with-superuser-path=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin --includedir=${prefix}/include --mandir=${prefix}/share/man --infodir=${prefix}/share/info

make install