riak crash report

gistfile1.txt

2015-07-29 19:12:51 =CRASH REPORT====
  crasher:
    initial call: riak_core_vnode:init/1
    pid: <0.28158.0>
    registered_name: []
    exception exit: {{{badmatch,{reply,{riak_kv_w1c_put_reply_v1,ok,primary},{state,598016783243797190163421821199338462144370835456,riak_kv_multi_backend,false,{state,[{<<"leveldb_mult">>,riak_kv_eleveldb_backend,{state,<<>>,"/var/lib/riak/leveldb/598016783243797190163421821199338462144370835456",[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{limited_developer_mem,false},{sst_block_size,4096},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{write_buffer_size,22216967}],[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{data_root,"/var/lib/riak/leveldb"},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{included_applications,[]},{limited_developer_mem,false},{sst_block_size,4096},{sync,false},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{verify_checksums,true},{verify_compaction,true},{write_buffer_size,22216967},{write_buffer_size_max,31457280},{write_buffer_size_min,15728640}],[{verify_checksums,true}],[{sync,false}],[{verify_checksums,true},{fill_cache,false}],true,false}},{<<"bitcask_mult">>,riak_kv_bitcask_backend,{state,#Ref<0.0.2.213408>,"598016783243797190163421821199338462144370835456",[{io_mode,erlang},{expiry_grace_time,0},{small_file_threshold,10485760},{dead_bytes_threshold,134217728},{frag_threshold,40},{dead_bytes_merge_trigger,536870912},{frag_merge_trigger,60},{max_file_size,2147483648},{open_timeout,4},{data_root,"/var/lib/riak/bitcask"},{sync_strategy,none},{merge_window,always},{max_fold_age,-1},{max_fold_puts,0},{expiry_secs,-1},{require_hint_crc,true},{key_transform,#Fun<riak_kv_bitcask_backend.1.79788753>},{read_write,true}],598016783243797190163421821199338462144370835456,"/var/lib/riak/bitcask",1}}],<<"bitcask_mult">>},{dict,0,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},<<214,6,153,82,182,189,39,36>>,3000,1000,100,100,true,true,{59801678324379719016342182119933
^Croot@riak9:/var/log/riak# ^C
root@riak9:/var/log/riak# cat crash.log | tail -n 50
    registered_name: []
    exception exit: {{{badmatch,{reply,{riak_kv_w1c_put_reply_v1,ok,primary},{state,675088158649919023740569263549611199509039153152,riak_kv_multi_backend,false,{state,[{<<"leveldb_mult">>,riak_kv_eleveldb_backend,{state,<<>>,"/var/lib/riak/leveldb/675088158649919023740569263549611199509039153152",[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{limited_developer_mem,false},{sst_block_size,4096},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{write_buffer_size,22280886}],[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{data_root,"/var/lib/riak/leveldb"},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{included_applications,[]},{limited_developer_mem,false},{sst_block_size,4096},{sync,false},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{verify_checksums,true},{verify_compaction,true},{write_buffer_size,22280886},{write_buffer_size_max,31457280},{write_buffer_size_min,15728640}],[{verify_checksums,true}],[{sync,false}],[{verify_checksums,true},{fill_cache,false}],true,false}},{<<"bitcask_mult">>,riak_kv_bitcask_backend,{state,#Ref<0.0.2.143246>,"675088158649919023740569263549611199509039153152",[{io_mode,erlang},{expiry_grace_time,0},{small_file_threshold,10485760},{dead_bytes_threshold,134217728},{frag_threshold,40},{dead_bytes_merge_trigger,536870912},{frag_merge_trigger,60},{max_file_size,2147483648},{open_timeout,4},{data_root,"/var/lib/riak/bitcask"},{sync_strategy,none},{merge_window,always},{max_fold_age,-1},{max_fold_puts,0},{expiry_secs,-1},{require_hint_crc,true},{key_transform,#Fun<riak_kv_bitcask_backend.1.79788753>},{read_write,true}],675088158649919023740569263549611199509039153152,"/var/lib/riak/bitcask",1}}],<<"bitcask_mult">>},{dict,0,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},<<214,6,153,82,182,202,172,168>>,3000,1000,100,100,true,true,{675088158649919023740569263549611199509039153152,'[email protected]'},0,undefined,<0.8155.0>,undefined,0,{counter_state,true,400001,410000,10000,false},<0.8153.0>}}},[{riak_kv_vnode,handle_handoff_command,3,[{file,"src/riak_kv_vnode.erl"},{line,1044}]},{riak_core_vnode,vnode_handoff_command,4,[{file,"src/riak_core_vnode.erl"},{line,408}]},{gen_fsm,handle_msg,7,[{file,"gen_fsm.erl"},{line,505}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,239}]}]},[{gen_fsm,terminate,7,[{file,"gen_fsm.erl"},{line,622}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,239}]}]}
    ancestors: [<0.8156.0>,<0.8152.0>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]
    messages: []
    links: [<0.8164.0>,<0.8166.0>,<0.8168.0>,<0.9067.0>,<0.8167.0>,<0.8165.0>,<0.8160.0>,<0.8162.0>,<0.8163.0>,<0.8161.0>,<0.8158.0>]
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 2586
    stack_size: 27
    reductions: 544
  neighbours:
    neighbour: [{pid,<0.8161.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.8158.0>,<0.8157.0>,<0.8156.0>,<0.8152.0>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.8158.0>,<0.8157.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
    neighbour: [{pid,<0.8163.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.8158.0>,<0.8157.0>,<0.8156.0>,<0.8152.0>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.8158.0>,<0.8157.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
    neighbour: [{pid,<0.8162.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.8158.0>,<0.8157.0>,<0.8156.0>,<0.8152.0>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.8158.0>,<0.8157.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
    neighbour: [{pid,<0.8160.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.8158.0>,<0.8157.0>,<0.8156.0>,<0.8152.0>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.8158.0>,<0.8157.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
    neighbour: [{pid,<0.8165.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.8158.0>,<0.8157.0>,<0.8156.0>,<0.8152.0>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.8158.0>,<0.8157.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
    neighbour: [{pid,<0.8167.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.8158.0>,<0.8157.0>,<0.8156.0>,<0.8152.0>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.8158.0>,<0.8157.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
    neighbour: [{pid,<0.9067.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.8158.0>,<0.8157.0>,<0.8156.0>,<0.8152.0>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.8158.0>,<0.8157.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
    neighbour: [{pid,<0.8168.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.8158.0>,<0.8157.0>,<0.8156.0>,<0.8152.0>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.8158.0>,<0.8157.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
    neighbour: [{pid,<0.8166.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.8158.0>,<0.8157.0>,<0.8156.0>,<0.8152.0>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.8158.0>,<0.8157.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
    neighbour: [{pid,<0.8164.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.8158.0>,<0.8157.0>,<0.8156.0>,<0.8152.0>,riak_core_vnode_sup,riak_core_sup,<0.161.0>]},{messages,[]},{links,[<0.8158.0>,<0.8157.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,27}]
2015-07-29 20:03:28 =SUPERVISOR REPORT====
     Supervisor: {<0.8158.0>,poolboy_sup}
     Context:    shutdown_error
     Reason:     {{badmatch,{reply,{riak_kv_w1c_put_reply_v1,ok,primary},{state,675088158649919023740569263549611199509039153152,riak_kv_multi_backend,false,{state,[{<<"leveldb_mult">>,riak_kv_eleveldb_backend,{state,<<>>,"/var/lib/riak/leveldb/675088158649919023740569263549611199509039153152",[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{limited_developer_mem,false},{sst_block_size,4096},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{write_buffer_size,22280886}],[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{data_root,"/var/lib/riak/leveldb"},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{included_applications,[]},{limited_developer_mem,false},{sst_block_size,4096},{sync,false},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{verify_checksums,true},{verify_compaction,true},{write_buffer_size,22280886},{write_buffer_size_max,31457280},{write_buffer_size_min,15728640}],[{verify_checksums,true}],[{sync,false}],[{verify_checksums,true},{fill_cache,false}],true,false}},{<<"bitcask_mult">>,riak_kv_bitcask_backend,{state,#Ref<0.0.2.143246>,"675088158649919023740569263549611199509039153152",[{io_mode,erlang},{expiry_grace_time,0},{small_file_threshold,10485760},{dead_bytes_threshold,134217728},{frag_threshold,40},{dead_bytes_merge_trigger,536870912},{frag_merge_trigger,60},{max_file_size,2147483648},{open_timeout,4},{data_root,"/var/lib/riak/bitcask"},{sync_strategy,none},{merge_window,always},{max_fold_age,-1},{max_fold_puts,0},{expiry_secs,-1},{require_hint_crc,true},{key_transform,#Fun<riak_kv_bitcask_backend.1.79788753>},{read_write,true}],675088158649919023740569263549611199509039153152,"/var/lib/riak/bitcask",1}}],<<"bitcask_mult">>},{dict,0,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},<<214,6,153,82,182,202,172,168>>,3000,1000,100,100,true,true,{675088158649919023740569263549611199509039153152,'[email protected]'},0,undefined,<0.8155.0>,undefined,0,{counter_state,true,400001,410000,10000,false},<0.8153.0>}}},[{riak_kv_vnode,handle_handoff_command,3,[{file,"src/riak_kv_vnode.erl"},{line,1044}]},{riak_core_vnode,vnode_handoff_command,4,[{file,"src/riak_core_vnode.erl"},{line,408}]},{gen_fsm,handle_msg,7,[{file,"gen_fsm.erl"},{line,505}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,239}]}]}
     Offender:   [{nb_children,6},{name,riak_core_vnode_worker},{mfargs,{riak_core_vnode_worker,start_link,[[{worker_module,riak_core_vnode_worker},{worker_args,[675088158649919023740569263549611199509039153152,[],worker_props,<0.8156.0>]},{worker_callback_mod,riak_kv_worker},{size,10},{max_overflow,0}]]}},{restart_type,temporary},{shutdown,5000},{child_type,worker}]

2015-07-29 20:03:28 =ERROR REPORT====
** Generic server <0.8158.0> terminating 
** Last message in was {'EXIT',<0.8157.0>,{{badmatch,{reply,{riak_kv_w1c_put_reply_v1,ok,primary},{state,675088158649919023740569263549611199509039153152,riak_kv_multi_backend,false,{state,[{<<"leveldb_mult">>,riak_kv_eleveldb_backend,{state,<<>>,"/var/lib/riak/leveldb/675088158649919023740569263549611199509039153152",[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{limited_developer_mem,false},{sst_block_size,4096},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{write_buffer_size,22280886}],[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{data_root,"/var/lib/riak/leveldb"},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{included_applications,[]},{limited_developer_mem,false},{sst_block_size,4096},{sync,false},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{verify_checksums,true},{verify_compaction,true},{write_buffer_size,22280886},{write_buffer_size_max,31457280},{write_buffer_size_min,15728640}],[{verify_checksums,true}],[{sync,false}],[{verify_checksums,true},{fill_cache,false}],true,false}},{<<"bitcask_mult">>,riak_kv_bitcask_backend,{state,#Ref<0.0.2.143246>,"675088158649919023740569263549611199509039153152",[{io_mode,erlang},{expiry_grace_time,0},{small_file_threshold,10485760},{dead_bytes_threshold,134217728},{frag_threshold,40},{dead_bytes_merge_trigger,536870912},{frag_merge_trigger,60},{max_file_size,2147483648},{open_timeout,4},{data_root,"/var/lib/riak/bitcask"},{sync_strategy,none},{merge_window,always},{max_fold_age,-1},{max_fold_puts,0},{expiry_secs,-1},{require_hint_crc,true},{key_transform,#Fun<riak_kv_bitcask_backend.1.79788753>},{read_write,true}],675088158649919023740569263549611199509039153152,"/var/lib/riak/bitcask",1}}],<<"bitcask_mult">>},{dict,0,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},<<214,6,153,82,182,202,172,168>>,3000,1000,100,100,true,true,{675088158649919023740569263549611199509039153152,'[email protected]'},0,undefined,<0.8155.0>,undefined,0,{counter_state,true,400001,410000,10000,false},<0.8153.0>}}},[{riak_kv_vnode,handle_handoff_command,3,[{file,"src/riak_kv_vnode.erl"},{line,1044}]},{riak_core_vnode,vnode_handoff_command,4,[{file,"src/riak_core_vnode.erl"},{line,408}]},{gen_fsm,handle_msg,7,[{file,"gen_fsm.erl"},{line,505}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,239}]}]}}
** When Server state == {state,{<0.8158.0>,poolboy_sup},simple_one_for_one,[{child,undefined,riak_core_vnode_worker,{riak_core_vnode_worker,start_link,[[{worker_module,riak_core_vnode_worker},{worker_args,[675088158649919023740569263549611199509039153152,[],worker_props,<0.8156.0>]},{worker_callback_mod,riak_kv_worker},{size,10},{max_overflow,0}]]},temporary,5000,worker,[riak_core_vnode_worker]}],{set,10,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[],[],[],[],[<0.9067.0>],[<0.8160.0>],[<0.8161.0>],[<0.8162.0>],[<0.8163.0>],[<0.8164.0>],[<0.8165.0>],[<0.8166.0>],[<0.8167.0>],[<0.8168.0>]}}},0,1,[],poolboy_sup,{riak_core_vnode_worker,[{worker_module,riak_core_vnode_worker},{worker_args,[675088158649919023740569263549611199509039153152,[],worker_props,<0.8156.0>]},{worker_callback_mod,riak_kv_worker},{size,10},{max_overflow,0}]}}
** Reason for termination == 
** {{badmatch,{reply,{riak_kv_w1c_put_reply_v1,ok,primary},{state,675088158649919023740569263549611199509039153152,riak_kv_multi_backend,false,{state,[{<<"leveldb_mult">>,riak_kv_eleveldb_backend,{state,<<>>,"/var/lib/riak/leveldb/675088158649919023740569263549611199509039153152",[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{limited_developer_mem,false},{sst_block_size,4096},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{write_buffer_size,22280886}],[{block_cache_threshold,33554432},{block_restart_interval,16},{block_size_steps,16},{compression,true},{create_if_missing,true},{data_root,"/var/lib/riak/leveldb"},{delete_threshold,1000},{eleveldb_threads,71},{fadvise_willneed,false},{included_applications,[]},{limited_developer_mem,false},{sst_block_size,4096},{sync,false},{tiered_slow_level,0},{total_leveldb_mem_percent,35},{use_bloomfilter,true},{verify_checksums,true},{verify_compaction,true},{write_buffer_size,22280886},{write_buffer_size_max,31457280},{write_buffer_size_min,15728640}],[{verify_checksums,true}],[{sync,false}],[{verify_checksums,true},{fill_cache,false}],true,false}},{<<"bitcask_mult">>,riak_kv_bitcask_backend,{state,#Ref<0.0.2.143246>,"675088158649919023740569263549611199509039153152",[{io_mode,erlang},{expiry_grace_time,0},{small_file_threshold,10485760},{dead_bytes_threshold,134217728},{frag_threshold,40},{dead_bytes_merge_trigger,536870912},{frag_merge_trigger,60},{max_file_size,2147483648},{open_timeout,4},{data_root,"/var/lib/riak/bitcask"},{sync_strategy,none},{merge_window,always},{max_fold_age,-1},{max_fold_puts,0},{expiry_secs,-1},{require_hint_crc,true},{key_transform,#Fun<riak_kv_bitcask_backend.1.79788753>},{read_write,true}],675088158649919023740569263549611199509039153152,"/var/lib/riak/bitcask",1}}],<<"bitcask_mult">>},{dict,0,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},<<214,6,153,82,182,202,172,168>>,3000,1000,100,100,true,true,{675088158649919023740569263549611199509039153152,'[email protected]'},0,undefined,<0.8155.0>,undefined,0,{counter_state,true,400001,410000,10000,false},<0.8153.0>}}},[{riak_kv_vnode,handle_handoff_command,3,[{file,"src/riak_kv_vnode.erl"},{line,1044}]},{riak_core_vnode,vnode_handoff_command,4,[{file,"src/riak_core_vnode.erl"},{line,408}]},{gen_fsm,handle_msg,7,[{file,"gen_fsm.erl"},{line,505}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,239}]}]}