elprup

# master config
port=1456
max_msg_per_second=2000000
max_queue_size=10000000
check_interval=3

<store>
category=heartbeat
type=file
file_path=/mnt/heartbeat

elprup

import boto.iam
import time
import urllib2
import logging

# connetc to iam
# default boto config file location: /etc/boto.cfg
conn = boto.connect_iam()

#setup logging

elprup

sudo apt-get install gfortran g++ libreadline6-dev libxt-dev
make
sudo make install

elprup

#!/bin/bash
wget --no-check-certificate https://ftp-master.debian.org/keys/archive-key-6.0.asc && apt-key add archive-key-6.0.asc

sudo gpg --keyserver hkp://pgp.mit.edu --recv-keys 16126D3A3E5C1192
sudo gpg --export --armor 16126D3A3E5C1192 | sudo apt-key add -

sudo apt-get install libboost-all-dev libboost-test-dev libboost-program-options-dev libevent-dev automake libtool flex bison pkg-config g++ libssl-dev git-core make

wget http://archive.apache.org/dist/thrift/0.9.0/thrift-0.9.0.tar.gz

elprup

'''
EMR tools
Easy emr python console
author: yaowei
create: 2013-06-24
'''
import copy
import os.path

import boto.emr

elprup

蓄水池抽样 Reservoir sampling

elprup

#!/usr/bin/python
import subprocess as spb
import os.path
import sys

msgfmt_path = '/tmp/msgfmt.py'

def run(cmd):
    print 'func[cmd]:' + cmd
    sock = spb.Popen(cmd, shell=True, stdout=spb.PIPE)

elprup

#!/usr/bin/env python
'''
Scribe checker with heartbeat

desc: check scribe alive status, alert if fail
'''

import os
import sys
import time

elprup

class BaseHandler(tornado.web.RequestHandler):
    def get_current_user(self):
        user_json = self.get_secure_cookie("authdemo_user")
        if not user_json: return None
        return tornado.escape.json_decode(user_json)

class GoogleOAuth2LoginHandler(tornado.web.RequestHandler,
                               tornado.auth.GoogleOAuth2Mixin):
    @tornado.gen.coroutine
    def get(self):

elprup

Migrate from Facebook scribe to Apache Flume (Part I)

Reason

We use scribe as our logging server for a long time. At first, everything works fine. Easy to config, easy to manage. As data grows everyday, single scribe server can’t handle that. We have to migrate some category to second log server and attach a big disk. As data is keep growing, we want a big data storage for that instead of local disk. So we decide to use Scribe with HDFS plugin. It is as tough as we first compile scribe from source. Finally we complied the scribed with hdfs support. But after a short period usage, we find a bug that haven’t solved by facebook. (the project is deprecated several years ago). The bug cause scribe can’t write to hdfs if it accidently killed by SIG 9. So we start to test flume and find out ways to migrate.

Configure Flume

Flume is easy to deploy because it is written in Java. Install java and download jar package, we’ve done all the jobs