dallasmarlow

require 'tsd_client'
require 'date'

tsd = TSD::Client.new host: 'fibr'
tsd.query(metric: 'killercroc.convert', start: Date.today - 1).sort_by do |entry|
  entry[:time].hour
end.reduce(Hash.new(0)) do |croc_stats, entry|
  croc_stats[:total] += entry[:value]
  croc_stats[entry[:time].hour] += entry[:value]

dallasmarlow

ElephantKeeper::JanitorPolicyGroup.define :pixelbug do |janitor|
  janitor.clusters  = [:peanutbutter]
  janitor.threshold = 0.8 # 80% hdfs usage
  janitor.add_policy(
    '/services',
    recurse: true,
    filter_mode: :all,
    filters: [
      Proc.new {|entry| entry.is_directory?},
      Proc.new {|entry| entry.name =~ /pixelbug_(post|url)\/\d+\/\d+$/},

dallasmarlow

--- sb_logger.c.bak	2014-04-18 22:49:41.000000000 -0400
+++ sb_logger.c	2014-04-18 23:45:55.000000000 -0400
@@ -78,6 +78,10 @@
 static int oper_handler_process(log_msg_t *msg);
 static int oper_handler_done(void);

+/* array of latencies of response times */
+/* (in ms) <=.1,.2,.3,.4,.5,.6,.7,.8,.9, 1, 5, 10, 50, 100, 500, 1000, 5000, 10000, >10,000 (10 sec) */
+static unsigned int latencies[22];
+

dallasmarlow

require 'thread_puddle'

small_puddle = ThreadPuddle.new 2
large_puddle = ThreadPuddle.new 24
application_request_queue = Queue.new

# lets pretend that `application_request_queue` is being written to 
# by another set of threads accepting queries from clients for this application

# normal run logic to process queue

dallasmarlow

rocksdb: A persistent key-value store for flash storage
Authors: * The Facebook Database Engineering Team
         * Build on earlier work on leveldb by Sanjay Ghemawat
           ([email protected]) and Jeff Dean ([email protected])

This code is a library that forms the core building block for a fast
key value server, especially suited for storing data on flash drives.
It has an Log-Structured-Merge-Database (LSM) design with flexible tradeoffs
between Write-Amplification-Factor(WAF), Read-Amplification-Factor (RAF)
and Space-Amplification-Factor(SAF). It has multi-threaded compactions,

dallasmarlow

Bulk Load of keys in Sequential Order

./db_bench --benchmarks=fillseq --disable_seek_compaction=1 \
    --mmap_read=0 --statistics=0 --histogram=0 --threads=1 --value_size=800 \
    --block_size=65536 --cache_size=1048576 --bloom_bits=10 --cache_numshardbits=4 \
    --open_files=500000 --verify_checksum=1 --db=bench_db --sync=0 \
    --disable_wal=1 --disable_data_sync=0 \
    --write_buffer_size=134217728 --target_file_size_base=67108864 --max_write_buffer_number=3 \
    --max_background_compactions=20 --level0_file_num_compaction_trigger=4 \
    --level0_slowdown_writes_trigger=8 --level0_stop_writes_trigger=12 --num_levels=6 \

dallasmarlow

require File.expand_path '~/collins.rb'
require 'timeout'

assets = collins.find(pool: :peanutbutter, secondary_role: :datanode, size: 1000, status: :allocated)
assets.map do |asset|
  Thread.new do
    begin
      Timeout::timeout(120) do
        lldp_info = ssh! asset, 'lldpctl'

dallasmarlow

2014-06-10 08:47:52,319 ERROR org.apache.hadoop.hbase.regionserver.HRegionServer: Failed openScanner
java.lang.IllegalStateException: Schema metrics requested before table/CF name initialization: {"tableName":"null","cfName":"null"}
  at org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured.getSchemaMetrics(SchemaConfigured.java:180)
  at org.apache.hadoop.hbase.io.hfile.LruBlockCache.updateSizeMetrics(LruBlockCache.java:337)
  at org.apache.hadoop.hbase.io.hfile.LruBlockCache.cacheBlock(LruBlockCache.java:292)
  at org.apache.hadoop.hbase.io.hfile.LruBlockCache.cacheBlock(LruBlockCache.java:320)
  at org.apache.hadoop.hbase.io.hfile.DoubleBlockCache.getBlock(DoubleBlockCache.java:102)
  at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:303)
  at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:254)
  at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:480)

dallasmarlow

require 'tsd_client'
tsd = TSD::Client.new host: 'fibr'
quantile = 0.95
query    = metric: 'cache_error_errs',
           start:  Time.parse('2014/06/11-12:00:00'),
           rate:   true,
           tags:   {host: '*'}

results  = tsd.query(query).sort_by {|entry| entry[:value]}
boundary = results.fetch((quantile * results.size).ceil - 1)

dallasmarlow

[READ], Operations, 1000000
[READ], AverageLatency(us), 35280.495734
[READ], MinLatency(us), 211
[READ], MaxLatency(us), 1900142
[READ], 95thPercentileLatency(ms), 121
[READ], 99thPercentileLatency(ms), 238
[READ], Return=0, 1000000
[OVERALL], RunTime(ms), 893002.0
[OVERALL], Throughput(ops/sec), 1119.8183206756537