gistfile1.txt

2015-07-29 20:11:00 =SUPERVISOR REPORT====
     Supervisor: {local,riak_core_handoff_sender_sup}
     Context:    child_terminated
     Reason:     vnode_died
     Offender:   [{pid,<0.25521.9>},{name,riak_core_handoff_sender},{mfargs,{riak_core_handoff_sender,start_link,undefined}},{restart_type,temporary},{shutdown,brutal_kill},{child_type,worker}]

2015-07-29 20:11:00 =ERROR REPORT====
** Generic server <0.25378.9> terminating 
** Last message in was {'$gen_cast',{work,{fold,#Fun<riak_kv_multi_backend.13.132002736>,#Fun<riak_kv_vnode.37.91169427>},{server,undefined,{<0.25521.9>,#Ref<0.0.25.251807>}},<0.25375.9>}}
** When Server state == {state,riak_kv_worker,{state,687933387884272662670093837274656655736483872768}}
** Reason for termination == 
** {badarg,[{eleveldb,async_iterator_close,[#Ref<0.0.25.260649>,<<>>],[]},{eleveldb,iterator_close,1,[{file,"src/eleveldb.erl"},{line,222}]},{eleveldb,do_fold,4,[{file,"src/eleveldb.erl"},{line,353}]},{riak_kv_eleveldb_backend,'-fold_objects/4-fun-0-',4,[{file,"src/riak_kv_eleveldb_backend.erl"},{line,490}]},{lists,foldl,3,[{file,"lists.erl"},{line,1248}]},{riak_kv_worker,handle_work,3,[{file,"src/riak_kv_worker.erl"},{line,46}]},{riak_core_vnode_worker,handle_cast,2,[{file,"src/riak_core_vnode_worker.erl"},{line,70}]},{gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,604}]}]}
2015-07-29 20:11:00 =CRASH REPORT====
  crasher:
    initial call: riak_core_vnode_worker:init/1
    pid: <0.25378.9>
    registered_name: []
    exception exit: {{badarg,[{eleveldb,async_iterator_close,[#Ref<0.0.25.260649>,<<>>],[]},{eleveldb,iterator_close,1,[{file,"src/eleveldb.erl"},{line,222}]},{eleveldb,do_fold,4,[{file,"src/eleveldb.erl"},{line,353}]},{riak_kv_eleveldb_backend,'-fold_objects/4-fun-0-',4,[{file,"src/riak_kv_eleveldb_backend.erl"},{line,490}]},{lists,foldl,3,[{file,"lists.erl"},{line,1248}]},{riak_kv_worker,handle_work,3,[{file,"src/riak_kv_worker.erl"},{line,46}]},{riak_core_vnode_worker,handle_cast,2,[{file,"src/riak_core_vnode_worker.erl"},{line,70}]},{gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,604}]}]},[{gen_server,terminate,6,[{file,"gen_server.erl"},{line,744}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,239}]}]}
    ancestors: [<0.25376.9>,<0.25377.9>,<0.25375.9>,<0.25371.9>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]
    messages: []
    links: [<0.25376.9>,<0.25377.9>,<0.93.0>]
    dictionary: [{bitcask_time_fudge,no_testing},{is_visit_item_timer_set,true}]
    trap_exit: false
    status: running
    heap_size: 75113
    stack_size: 27
    reductions: 10565886
  neighbours:
2015-07-29 20:11:00 =SUPERVISOR REPORT====
     Supervisor: {<0.25376.9>,poolboy_sup}
     Context:    child_terminated
     Reason:     {badarg,[{eleveldb,async_iterator_close,[#Ref<0.0.25.260649>,<<>>],[]},{eleveldb,iterator_close,1,[{file,"src/eleveldb.erl"},{line,222}]},{eleveldb,do_fold,4,[{file,"src/eleveldb.erl"},{line,353}]},{riak_kv_eleveldb_backend,'-fold_objects/4-fun-0-',4,[{file,"src/riak_kv_eleveldb_backend.erl"},{line,490}]},{lists,foldl,3,[{file,"lists.erl"},{line,1248}]},{riak_kv_worker,handle_work,3,[{file,"src/riak_kv_worker.erl"},{line,46}]},{riak_core_vnode_worker,handle_cast,2,[{file,"src/riak_core_vnode_worker.erl"},{line,70}]},{gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,604}]}]}
     Offender:   [{pid,<0.25378.9>},{name,riak_core_vnode_worker},{mfargs,{riak_core_vnode_worker,start_link,undefined}},{restart_type,temporary},{shutdown,5000},{child_type,worker}]

2015-07-29 20:11:00 =CRASH REPORT====
  crasher:
    initial call: poolboy:init/1
    pid: <0.25377.9>
    registered_name: []
    exception exit: {{{badmatch,{reply,{riak_kv_w1c_put_reply_v1,ok,primary},{state,687933387884272662670093837274656655736483872768,riak_kv_multi_backend,false,{state,[{<<"leveldb_mult">>,riak_kv_eleveldb_backend,{state,<<>>,"/var/lib/riak/leveldb/687933387884272662670093837274656655736483872768",[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{limited_developer_mem,false},{sst_block_size,4096},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{write_buffer_size,26065841}],[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{data_root,"/var/lib/riak/leveldb"},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{included_applications,[]},{limited_developer_mem,false},{sst_block_size,4096},{sync,false},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{verify_checksums,true},{verify_compaction,true},{write_buffer_size,26065841},{write_buffer_size_max,31457280},{write_buffer_size_min,15728640}],[{verify_checksums,true}],[{sync,false}],[{verify_checksums,true},{fill_cache,false}],true,false}},{<<"bitcask_mult">>,riak_kv_bitcask_backend,{state,#Ref<0.0.25.250705>,"687933387884272662670093837274656655736483872768",[{io_mode,erlang},{expiry_grace_time,0},{small_file_threshold,10485760},{dead_bytes_threshold,134217728},{frag_threshold,40},{dead_bytes_merge_trigger,536870912},{frag_merge_trigger,60},{max_file_size,2147483648},{open_timeout,4},{data_root,"/var/lib/riak/bitcask"},{sync_strategy,none},{merge_window,always},{max_fold_age,-1},{max_fold_puts,0},{expiry_secs,-1},{require_hint_crc,true},{key_transform,#Fun<riak_kv_bitcask_backend.1.79788753>},{read_write,true}],687933387884272662670093837274656655736483872768,"/var/lib/riak/bitcask",1}}],<<"bitcask_mult">>},{dict,0,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},<<"m\r;³ð¹ÚN">>,3000,1000,100,100,true,true,{687933387884272662670093837274656655736483872768,'[email protected]'},0,undefined,<0.25374.9>,undefined,0,{counter_state,true,190001,200000,10000,false},<0.25372.9>}}},[{riak_kv_vnode,handle_handoff_command,3,[{file,"src/riak_kv_vnode.erl"},{line,1044}]},{riak_core_vnode,vnode_handoff_command,4,[{file,"src/riak_core_vnode.erl"},{line,408}]},{gen_fsm,handle_msg,7,[{file,"gen_fsm.erl"},{line,505}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,239}]}]},[{gen_fsm,terminate,7,[{file,"gen_fsm.erl"},{line,622}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,239}]}]}
    ancestors: [<0.25375.9>,<0.25371.9>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]
    messages: [{'DOWN',#Ref<0.0.25.251811>,process,<0.25375.9>,{{badmatch,{reply,{riak_kv_w1c_put_reply_v1,ok,primary},{state,687933387884272662670093837274656655736483872768,riak_kv_multi_backend,false,{state,[{<<"leveldb_mult">>,riak_kv_eleveldb_backend,{state,<<>>,"/var/lib/riak/leveldb/687933387884272662670093837274656655736483872768",[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{limited_developer_mem,false},{sst_block_size,4096},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{write_buffer_size,26065841}],[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{data_root,"/var/lib/riak/leveldb"},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{included_applications,[]},{limited_developer_mem,false},{sst_block_size,4096},{sync,false},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{verify_checksums,true},{verify_compaction,true},{write_buffer_size,26065841},{write_buffer_size_max,31457280},{write_buffer_size_min,15728640}],[{verify_checksums,true}],[{sync,false}],[{verify_checksums,true},{fill_cache,false}],true,false}},{<<"bitcask_mult">>,riak_kv_bitcask_backend,{state,#Ref<0.0.25.250705>,"687933387884272662670093837274656655736483872768",[{io_mode,erlang},{expiry_grace_time,0},{small_file_threshold,10485760},{dead_bytes_threshold,134217728},{frag_threshold,40},{dead_bytes_merge_trigger,536870912},{frag_merge_trigger,60},{max_file_size,2147483648},{open_timeout,4},{data_root,"/var/lib/riak/bitcask"},{sync_strategy,none},{merge_window,always},{max_fold_age,-1},{max_fold_puts,0},{expiry_secs,-1},{require_hint_crc,true},{key_transform,#Fun<riak_kv_bitcask_backend.1.79788753>},{read_write,true}],687933387884272662670093837274656655736483872768,"/var/lib/riak/bitcask",1}}],<<"bitcask_mult">>},{dict,0,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},<<"m\r;³ð¹ÚN">>,3000,1000,100,100,true,true,{687933387884272662670093837274656655736483872768,'[email protected]'},0,undefined,<0.25374.9>,undefined,0,{counter_state,true,190001,200000,10000,false},<0.25372.9>}}},[{riak_kv_vnode,handle_handoff_command,3,[{file,"src/riak_kv_vnode.erl"},{line,1044}]},{riak_core_vnode,vnode_handoff_command,4,[{file,"src/riak_core_vnode.erl"},{line,408}]},{gen_fsm,handle_msg,7,[{file,"gen_fsm.erl"},{line,505}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,239}]}]}}]
    links: [<0.25383.9>,<0.25385.9>,<0.25386.9>,<0.25387.9>,<0.25384.9>,<0.25379.9>,<0.25381.9>,<0.25382.9>,<0.25380.9>,<0.25376.9>,<0.25378.9>]
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 2586
    stack_size: 27
    reductions: 520
  neighbours:
    neighbour: [{pid,<0.25380.9>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.25376.9>,<0.25377.9>,<0.25375.9>,<0.25371.9>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.25376.9>,<0.25377.9>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
    neighbour: [{pid,<0.25382.9>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.25376.9>,<0.25377.9>,<0.25375.9>,<0.25371.9>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.25376.9>,<0.25377.9>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
    neighbour: [{pid,<0.25381.9>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.25376.9>,<0.25377.9>,<0.25375.9>,<0.25371.9>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.25376.9>,<0.25377.9>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
    neighbour: [{pid,<0.25379.9>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.25376.9>,<0.25377.9>,<0.25375.9>,<0.25371.9>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.25376.9>,<0.25377.9>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
    neighbour: [{pid,<0.25384.9>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.25376.9>,<0.25377.9>,<0.25375.9>,<0.25371.9>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.25376.9>,<0.25377.9>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
    neighbour: [{pid,<0.25387.9>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.25376.9>,<0.25377.9>,<0.25375.9>,<0.25371.9>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.25376.9>,<0.25377.9>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
    neighbour: [{pid,<0.25386.9>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.25376.9>,<0.25377.9>,<0.25375.9>,<0.25371.9>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.25376.9>,<0.25377.9>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
    neighbour: [{pid,<0.25385.9>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.25376.9>,<0.25377.9>,<0.25375.9>,<0.25371.9>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.25376.9>,<0.25377.9>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
    neighbour: [{pid,<0.25383.9>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.25376.9>,<0.25377.9>,<0.25375.9>,<0.25371.9>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.25376.9>,<0.25377.9>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
2015-07-29 20:11:00 =SUPERVISOR REPORT====
     Supervisor: {<0.25376.9>,poolboy_sup}
     Context:    shutdown_error
     Reason:     {{badmatch,{reply,{riak_kv_w1c_put_reply_v1,ok,primary},{state,687933387884272662670093837274656655736483872768,riak_kv_multi_backend,false,{state,[{<<"leveldb_mult">>,riak_kv_eleveldb_backend,{state,<<>>,"/var/lib/riak/leveldb/687933387884272662670093837274656655736483872768",[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{limited_developer_mem,false},{sst_block_size,4096},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{write_buffer_size,26065841}],[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{data_root,"/var/lib/riak/leveldb"},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{included_applications,[]},{limited_developer_mem,false},{sst_block_size,4096},{sync,false},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{verify_checksums,true},{verify_compaction,true},{write_buffer_size,26065841},{write_buffer_size_max,31457280},{write_buffer_size_min,15728640}],[{verify_checksums,true}],[{sync,false}],[{verify_checksums,true},{fill_cache,false}],true,false}},{<<"bitcask_mult">>,riak_kv_bitcask_backend,{state,#Ref<0.0.25.250705>,"687933387884272662670093837274656655736483872768",[{io_mode,erlang},{expiry_grace_time,0},{small_file_threshold,10485760},{dead_bytes_threshold,134217728},{frag_threshold,40},{dead_bytes_merge_trigger,536870912},{frag_merge_trigger,60},{max_file_size,2147483648},{open_timeout,4},{data_root,"/var/lib/riak/bitcask"},{sync_strategy,none},{merge_window,always},{max_fold_age,-1},{max_fold_puts,0},{expiry_secs,-1},{require_hint_crc,true},{key_transform,#Fun<riak_kv_bitcask_backend.1.79788753>},{read_write,true}],687933387884272662670093837274656655736483872768,"/var/lib/riak/bitcask",1}}],<<"bitcask_mult">>},{dict,0,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},<<"m\r;³ð¹ÚN">>,3000,1000,100,100,true,true,{687933387884272662670093837274656655736483872768,'[email protected]'},0,undefined,<0.25374.9>,undefined,0,{counter_state,true,190001,200000,10000,false},<0.25372.9>}}},[{riak_kv_vnode,handle_handoff_command,3,[{file,"src/riak_kv_vnode.erl"},{line,1044}]},{riak_core_vnode,vnode_handoff_command,4,[{file,"src/riak_core_vnode.erl"},{line,408}]},{gen_fsm,handle_msg,7,[{file,"gen_fsm.erl"},{line,505}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,239}]}]}
     Offender:   [{nb_children,9},{name,riak_core_vnode_worker},{mfargs,{riak_core_vnode_worker,start_link,[[{worker_module,riak_core_vnode_worker},{worker_args,[687933387884272662670093837274656655736483872768,[],worker_props,<0.25375.9>]},{worker_callback_mod,riak_kv_worker},{size,10},{max_overflow,0}]]}},{restart_type,temporary},{shutdown,5000},{child_type,worker}]

2015-07-29 20:11:00 =ERROR REPORT====
** Generic server <0.25376.9> terminating 
** Last message in was {'EXIT',<0.25377.9>,{{badmatch,{reply,{riak_kv_w1c_put_reply_v1,ok,primary},{state,687933387884272662670093837274656655736483872768,riak_kv_multi_backend,false,{state,[{<<"leveldb_mult">>,riak_kv_eleveldb_backend,{state,<<>>,"/var/lib/riak/leveldb/687933387884272662670093837274656655736483872768",[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{limited_developer_mem,false},{sst_block_size,4096},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{write_buffer_size,26065841}],[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{data_root,"/var/lib/riak/leveldb"},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{included_applications,[]},{limited_developer_mem,false},{sst_block_size,4096},{sync,false},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{verify_checksums,true},{verify_compaction,true},{write_buffer_size,26065841},{write_buffer_size_max,31457280},{write_buffer_size_min,15728640}],[{verify_checksums,true}],[{sync,false}],[{verify_checksums,true},{fill_cache,false}],true,false}},{<<"bitcask_mult">>,riak_kv_bitcask_backend,{state,#Ref<0.0.25.250705>,"687933387884272662670093837274656655736483872768",[{io_mode,erlang},{expiry_grace_time,0},{small_file_threshold,10485760},{dead_bytes_threshold,134217728},{frag_threshold,40},{dead_bytes_merge_trigger,536870912},{frag_merge_trigger,60},{max_file_size,2147483648},{open_timeout,4},{data_root,"/var/lib/riak/bitcask"},{sync_strategy,none},{merge_window,always},{max_fold_age,-1},{max_fold_puts,0},{expiry_secs,-1},{require_hint_crc,true},{key_transform,#Fun<riak_kv_bitcask_backend.1.79788753>},{read_write,true}],687933387884272662670093837274656655736483872768,"/var/lib/riak/bitcask",1}}],<<"bitcask_mult">>},{dict,0,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},<<"m\r;³ð¹ÚN">>,3000,1000,100,100,true,true,{687933387884272662670093837274656655736483872768,'[email protected]'},0,undefined,<0.25374.9>,undefined,0,{counter_state,true,190001,200000,10000,false},<0.25372.9>}}},[{riak_kv_vnode,handle_handoff_command,3,[{file,"src/riak_kv_vnode.erl"},{line,1044}]},{riak_core_vnode,vnode_handoff_command,4,[{file,"src/riak_core_vnode.erl"},{line,408}]},{gen_fsm,handle_msg,7,[{file,"gen_fsm.erl"},{line,505}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,239}]}]}}
** When Server state == {state,{<0.25376.9>,poolboy_sup},simple_one_for_one,[{child,undefined,riak_core_vnode_worker,{riak_core_vnode_worker,start_link,[[{worker_module,riak_core_vnode_worker},{worker_args,[687933387884272662670093837274656655736483872768,[],worker_props,<0.25375.9>]},{worker_callback_mod,riak_kv_worker},{size,10},{max_overflow,0}]]},temporary,5000,worker,[riak_core_vnode_worker]}],{set,9,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[<0.25381.9>],[<0.25382.9>],[<0.25383.9>],[<0.25384.9>],[<0.25385.9>],[<0.25386.9>],[<0.25387.9>],[],[],[],[],[],[],[],[<0.25379.9>],[<0.25380.9>]}}},0,1,[],poolboy_sup,{riak_core_vnode_worker,[{worker_module,riak_core_vnode_worker},{worker_args,[687933387884272662670093837274656655736483872768,[],worker_props,<0.25375.9>]},{worker_callback_mod,riak_kv_worker},{size,10},{max_overflow,0}]}}
** Reason for termination == 
** {{badmatch,{reply,{riak_kv_w1c_put_reply_v1,ok,primary},{state,687933387884272662670093837274656655736483872768,riak_kv_multi_backend,false,{state,[{<<"leveldb_mult">>,riak_kv_eleveldb_backend,{state,<<>>,"/var/lib/riak/leveldb/687933387884272662670093837274656655736483872768",[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{limited_developer_mem,false},{sst_block_size,4096},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{write_buffer_size,26065841}],[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{data_root,"/var/lib/riak/leveldb"},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{included_applications,[]},{limited_developer_mem,false},{sst_block_size,4096},{sync,false},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{verify_checksums,true},{verify_compaction,true},{write_buffer_size,26065841},{write_buffer_size_max,31457280},{write_buffer_size_min,15728640}],[{verify_checksums,true}],[{sync,false}],[{verify_checksums,true},{fill_cache,false}],true,false}},{<<"bitcask_mult">>,riak_kv_bitcask_backend,{state,#Ref<0.0.25.250705>,"687933387884272662670093837274656655736483872768",[{io_mode,erlang},{expiry_grace_time,0},{small_file_threshold,10485760},{dead_bytes_threshold,134217728},{frag_threshold,40},{dead_bytes_merge_trigger,536870912},{frag_merge_trigger,60},{max_file_size,2147483648},{open_timeout,4},{data_root,"/var/lib/riak/bitcask"},{sync_strategy,none},{merge_window,always},{max_fold_age,-1},{max_fold_puts,0},{expiry_secs,-1},{require_hint_crc,true},{key_transform,#Fun<riak_kv_bitcask_backend.1.79788753>},{read_write,true}],687933387884272662670093837274656655736483872768,"/var/lib/riak/bitcask",1}}],<<"bitcask_mult">>},{dict,0,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},<<"m\r;³ð¹ÚN">>,3000,1000,100,100,true,true,{687933387884272662670093837274656655736483872768,'[email protected]'},0,undefined,<0.25374.9>,undefined,0,{counter_state,true,190001,200000,10000,false},<0.25372.9>}}},[{riak_kv_vnode,handle_handoff_command,3,[{file,"src/riak_kv_vnode.erl"},{line,1044}]},{riak_core_vnode,vnode_handoff_command,4,[{file,"src/riak_core_vnode.erl"},{line,408}]},{gen_fsm,handle_msg,7,[{file,"gen_fsm.erl"},{line,505}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,239}]}]}
2015-07-29 20:11:00 =CRASH REPORT====
  crasher:
    initial call: supervisor:poolboy_sup/1
    pid: <0.25376.9>
    registered_name: []
    exception exit: {{{badmatch,{reply,{riak_kv_w1c_put_reply_v1,ok,primary},{state,687933387884272662670093837274656655736483872768,riak_kv_multi_backend,false,{state,[{<<"leveldb_mult">>,riak_kv_eleveldb_backend,{state,<<>>,"/var/lib/riak/leveldb/687933387884272662670093837274656655736483872768",[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{limited_developer_mem,false},{sst_block_size,4096},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{write_buffer_size,26065841}],[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{data_root,"/var/lib/riak/leveldb"},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{included_applications,[]},{limited_developer_mem,false},{sst_block_size,4096},{sync,false},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{verify_checksums,true},{verify_compaction,true},{write_buffer_size,26065841},{write_buffer_size_max,31457280},{write_buffer_size_min,15728640}],[{verify_checksums,true}],[{sync,false}],[{verify_checksums,true},{fill_cache,false}],true,false}},{<<"bitcask_mult">>,riak_kv_bitcask_backend,{state,#Ref<0.0.25.250705>,"687933387884272662670093837274656655736483872768",[{io_mode,erlang},{expiry_grace_time,0},{small_file_threshold,10485760},{dead_bytes_threshold,134217728},{frag_threshold,40},{dead_bytes_merge_trigger,536870912},{frag_merge_trigger,60},{max_file_size,2147483648},{open_timeout,4},{data_root,"/var/lib/riak/bitcask"},{sync_strategy,none},{merge_window,always},{max_fold_age,-1},{max_fold_puts,0},{expiry_secs,-1},{require_hint_crc,true},{key_transform,#Fun<riak_kv_bitcask_backend.1.79788753>},{read_write,true}],687933387884272662670093837274656655736483872768,"/var/lib/riak/bitcask",1}}],<<"bitcask_mult">>},{dict,0,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},<<"m\r;³ð¹ÚN">>,3000,1000,100,100,true,true,{687933387884272662670093837274656655736483872768,'[email protected]'},0,undefined,<0.25374.9>,undefined,0,{counter_state,true,190001,200000,10000,false},<0.25372.9>}}},[{riak_kv_vnode,handle_handoff_command,3,[{file,"src/riak_kv_vnode.erl"},{line,1044}]},{riak_core_vnode,vnode_handoff_command,4,[{file,"src/riak_core_vnode.erl"},{line,408}]},{gen_fsm,handle_msg,7,[{file,"gen_fsm.erl"},{line,505}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,239}]}]},[{gen_server,terminate,6,[{file,"gen_server.erl"},{line,744}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,239}]}]}
    ancestors: [<0.25377.9>,<0.25375.9>,<0.25371.9>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]
    messages: []
    links: []
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 17731
    stack_size: 27
    reductions: 3754
  neighbours: