emmettbutler

{
    "_shards": {
        "failed": 0,
        "successful": 9,
        "total": 9
    },
    "hits": {
        "hits": [
            {
                "_id": "brightcovePlayer2|test.com|2016.01.07.00.00",

emmettbutler

#!/bin/bash

startkafka() {
    tmux new-window
    tmux send-keys /opt/kafka-$VERSION/bin/zookeeper-server-start.sh Space /opt/kafka-$VERSION/config/zookeeper.properties C-m

    sleep 3

    tmux split-window -h
    tmux send-keys /opt/kafka-$VERSION/bin/kafka-server-start.sh Space /opt/kafka-$VERSION/config/server.properties C-m

emmettbutler

import logging as log
import sys
import time

from pykafka import KafkaClient

log.basicConfig(level=log.DEBUG)


class ConsumerRunner():

emmettbutler

GLOB sdist-make: /home/emmett/git/parsely/pykafka/setup.py
py35 inst-nodeps: /home/emmett/git/parsely/pykafka/.tox/dist/pykafka-2.0.1.dev0.zip
py35 installed: coverage==4.0.1,kazoo==2.2.1,py==1.4.30,pykafka==2.0.1.dev0,pytest==2.8.2,pytest-cov==2.2.0,six==1.10.0,tabulate==0.7.5,wheel==0.24.0
py35 runtests: PYTHONHASHSEED='1010638969'
py35 runtests: commands[0] | pip install -r test-requirements.txt
Requirement already satisfied (use --upgrade to upgrade): pytest in ./.tox/py35/lib/python3.5/site-packages (from -r test-requirements.txt (line 1))
Requirement already satisfied (use --upgrade to upgrade): pytest-cov in ./.tox/py35/lib/python3.5/site-packages (from -r test-requirements.txt (line 2))
Collecting python-snappy (from -r test-requirements.txt (line 3))
  Using cached python-snappy-0.5.tar.gz
Collecting mock (from -r test-requirements.txt (line 4))

emmettbutler

import logging as log
import time
import threading

from pykafka import KafkaClient
from pykafka.common import OffsetType

log.basicConfig(level=log.ERROR)

client = KafkaClient(hosts="127.0.0.1:9092")

emmettbutler

[emmett:~/git/parsely/pykafka] [ct-pykafka] sontek_port_to_python34(+1/-1)+ 2s ± tox -e py34
GLOB sdist-make: /home/emmett/git/parsely/pykafka/setup.py
ERROR: invocation failed (exit code 1), logfile: /home/emmett/git/parsely/pykafka/.tox/log/tox-0.log
ERROR: actionid: tox
msg: packaging
cmdargs: ['/home/emmett/virtualenvs/ct-pykafka/bin/python', local('/home/emmett/git/parsely/pykafka/setup.py'), 'sdist', '--formats=zip', '--dist-dir', local('/home/emmett/git/parsely/pykafka/.tox/dist')]
env: None

error in pykafka setup command: 'extras_require' must be a dictionary whose values are strings or lists of strings containing valid project/version requirement specifiers.

emmettbutler

snappymodule.cc:28:20: fatal error: Python.h: No such file or directory
   #include "Python.h"
                      ^
  compilation terminated.
  error: command 'x86_64-linux-gnu-gcc' failed with exit status 1
  
  ----------------------------------------
  Failed building wheel for python-snappy
Failed to build python-snappy
Installing collected packages: python-snappy, pymongo, redis, testinstances

emmettbutler

from pykafka import KafkaClient
from pykafka.common import CompressionType
import time

client = KafkaClient()
topic = client.topics['emmett.test.async']


def test(sync=False, linger_ms=5000, min_queued_messages=5000, max_queued_messages=10000):
    print "Running with sync={}, linger_ms={}, min_queued_messages={}, max_queued_messages={}".format(

emmettbutler

(playground)cogtree@ue1a-hack:~$ python kafka_usage.py 
Running with sync=False, linger_ms=10000, min_queued_messages=100000, max_queued_messages=100000
62937.5129683 messages per second
30732.5692257 messages per second
42709.8328461 messages per second
19977.7277768 messages per second
Running with sync=False, linger_ms=5000, min_queued_messages=100000, max_queued_messages=100000
64113.5755426 messages per second
29659.0868596 messages per second
40632.7092676 messages per second