seeker815

{
  "id": "/production.docker-registry-v2",
  "cmd": null,
  "cpus": 1,
  "mem": 2048,
  "disk": 2048,
  "instances": 1,
  "constraints": [
    [
      "lifecycle",

seeker815

import requests
import json
import boto
import socket

HOSTNAME = socket.gethostbyaddr(socket.gethostname())[0]
JOB_TRACKER_URL = "http://" + HOSTNAME + ":8088/ws/v1/cluster/metrics/"

config = json.load(open("config.json"))
METRIC_NAMESPACE = config['namespace']

seeker815

<clusterMetrics>
<appsSubmitted>183</appsSubmitted>
<appsCompleted>181</appsCompleted>
<appsPending>0</appsPending>
<appsRunning>0</appsRunning>
<appsFailed>0</appsFailed>
<appsKilled>2</appsKilled>
<reservedMB>0</reservedMB>
<availableMB>0</availableMB>
<allocatedMB>0</allocatedMB>

seeker815

#!/bin/bash
# trap
trap suicide SIGINT SIGHUP SIGQUIT
suicide (){
  echo $?
  echo "You hit Cntrl+c, exiting"
  exit
}

seeker815

#!/bin/bash
# trap1
trap suicide EXIT SIGINT SIGQUIT
suicide (){
  if [ $? -ne 0 ]; then
   echo "You hit Cntrl+c, exiting"
  fi
  exit
}
count=0

seeker815

Producer

Setup
bin/kafka-topics.sh --zookeeper localhost:2181/kafka-local --create --topic test-rep-one --partitions 6 --replication-factor 1
bin/kafka-topics.sh --zookeeper localhost:2181/kafka-local --create --topic test-rep-two --partitions 6 --replication-factor 3

Single thread, no replication
bin/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance --print-metrics --topic test-rep-one --num-records 6000000 --throughput 100000 --record-size 100 --producer-props bootstrap.servers=kafka_host:9092 buffer.memory=67108864 batch.size=8196

Single-thread, async 3x replication

seeker815

#!/bin/bash
#
# Author: Ambud Sharma
#
# Purpose: To run kafka Kafka Producer / Consumer benchmark and create results
#
export NUM_RECORDS=10000000
export TOPIC_NAME_PREFIX="perf"
export CLUSTER_SIZE=3
export DRIVE_COUNT=1

seeker815

#!/bin/bash

# Purpose: To run kafka Kafka Producer / Consumer benchmark for Indix cluster
#

export TOPIC_NAME="benchmark_2_400"
export CLUSTER_SIZE=3
export DRIVE_COUNT=1
export BROKER_ADDRESS="10.181.18.63:9092"
export ZOOKEEPER_ADDRESS="10.181.18.63:2181"

seeker815

import boto3
tagname = "Name"
tagvalue = "Value"
ec2 = boto3.resource('ec2')
ec2.instances.filter(Filters=[
    {'Name': 'tag': [tagname], 'Values': [tagvalue]},
    {'Name': 'instance-state-name', 'Values': ['running']}
]).start()

ec2.instances.filter(Filters=[

seeker815

$ docker-compose up -d         # start containers in background
$ docker-compose kill          # stop containers
$ docker-compose up -d --build # force rebuild of Dockerfiles
$ docker-compose rm            # remove stopped containers
$ docker ps                    # see list of running containers
$ docker exec -ti [NAME] bash # ssh to the container

# list all images
docker images