Created
February 20, 2013 16:54
-
-
Save polyfractal/4997040 to your computer and use it in GitHub Desktop.
Reroute API breaks Cluster/Node/Stats API. More details
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ##Listed in order of operations and the resulting output: | |
| ## reroute the shards - executed on C1 client node | |
| $curl -XPOST 'localhost:9200/_cluster/reroute' -d '{ | |
| "commands":[ | |
| { | |
| "move":{ | |
| "index":"test", | |
| "shard":2, | |
| "from_node":"J47gdOIwQMq2GTmzzmzJBA", | |
| "to_node":"kH152vsLTL-y20mcLFs9GQ" | |
| } | |
| } | |
| ] | |
| }' | |
| { | |
| "ok":true, | |
| "state":{ | |
| "master_node":"J47gdOIwQMq2GTmzzmzJBA", | |
| "blocks":{ | |
| }, | |
| "nodes":{ | |
| "ez-rzcnfSESrVYP6zTWksA":{ | |
| "name":"S1", | |
| "transport_address":"inet[/144.76.3.102:9300]", | |
| "attributes":{ | |
| } | |
| }, | |
| "-VNWEiiaSG2IRlwM6kzcxg":{ | |
| "name":"C1", | |
| "transport_address":"inet[/144.76.8.228:9300]", | |
| "attributes":{ | |
| "data":"false" | |
| } | |
| }, | |
| "kH152vsLTL-y20mcLFs9GQ":{ | |
| "name":"S3", | |
| "transport_address":"inet[/144.76.2.205:9300]", | |
| "attributes":{ | |
| } | |
| }, | |
| "J47gdOIwQMq2GTmzzmzJBA":{ | |
| "name":"S2", | |
| "transport_address":"inet[/144.76.3.103:9300]", | |
| "attributes":{ | |
| } | |
| } | |
| }, | |
| "routing_table":{ | |
| "indices":{ | |
| "test":{ | |
| "shards":{ | |
| "0":[ | |
| { | |
| "state":"STARTED", | |
| "primary":true, | |
| "node":"kH152vsLTL-y20mcLFs9GQ", | |
| "relocating_node":null, | |
| "shard":0, | |
| "index":"test" | |
| }, | |
| { | |
| "state":"STARTED", | |
| "primary":false, | |
| "node":"J47gdOIwQMq2GTmzzmzJBA", | |
| "relocating_node":null, | |
| "shard":0, | |
| "index":"test" | |
| } | |
| ], | |
| "1":[ | |
| { | |
| "state":"STARTED", | |
| "primary":true, | |
| "node":"ez-rzcnfSESrVYP6zTWksA", | |
| "relocating_node":null, | |
| "shard":1, | |
| "index":"test" | |
| }, | |
| { | |
| "state":"STARTED", | |
| "primary":false, | |
| "node":"J47gdOIwQMq2GTmzzmzJBA", | |
| "relocating_node":null, | |
| "shard":1, | |
| "index":"test" | |
| } | |
| ], | |
| "2":[ | |
| { | |
| "state":"STARTED", | |
| "primary":false, | |
| "node":"ez-rzcnfSESrVYP6zTWksA", | |
| "relocating_node":null, | |
| "shard":2, | |
| "index":"test" | |
| }, | |
| { | |
| "state":"RELOCATING", | |
| "primary":true, | |
| "node":"J47gdOIwQMq2GTmzzmzJBA", | |
| "relocating_node":"kH152vsLTL-y20mcLFs9GQ", | |
| "shard":2, | |
| "index":"test" | |
| } | |
| ] | |
| } | |
| }, | |
| "test123":{ | |
| "shards":{ | |
| "0":[ | |
| { | |
| "state":"STARTED", | |
| "primary":false, | |
| "node":"ez-rzcnfSESrVYP6zTWksA", | |
| "relocating_node":null, | |
| "shard":0, | |
| "index":"test123" | |
| }, | |
| { | |
| "state":"STARTED", | |
| "primary":true, | |
| "node":"kH152vsLTL-y20mcLFs9GQ", | |
| "relocating_node":null, | |
| "shard":0, | |
| "index":"test123" | |
| } | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| "routing_nodes":{ | |
| "unassigned":[ | |
| ], | |
| "nodes":{ | |
| "ez-rzcnfSESrVYP6zTWksA":[ | |
| { | |
| "state":"STARTED", | |
| "primary":true, | |
| "node":"ez-rzcnfSESrVYP6zTWksA", | |
| "relocating_node":null, | |
| "shard":1, | |
| "index":"test" | |
| }, | |
| { | |
| "state":"STARTED", | |
| "primary":false, | |
| "node":"ez-rzcnfSESrVYP6zTWksA", | |
| "relocating_node":null, | |
| "shard":2, | |
| "index":"test" | |
| }, | |
| { | |
| "state":"STARTED", | |
| "primary":false, | |
| "node":"ez-rzcnfSESrVYP6zTWksA", | |
| "relocating_node":null, | |
| "shard":0, | |
| "index":"test123" | |
| } | |
| ], | |
| "kH152vsLTL-y20mcLFs9GQ":[ | |
| { | |
| "state":"STARTED", | |
| "primary":true, | |
| "node":"kH152vsLTL-y20mcLFs9GQ", | |
| "relocating_node":null, | |
| "shard":0, | |
| "index":"test" | |
| }, | |
| { | |
| "state":"INITIALIZING", | |
| "primary":true, | |
| "node":"kH152vsLTL-y20mcLFs9GQ", | |
| "relocating_node":"J47gdOIwQMq2GTmzzmzJBA", | |
| "shard":2, | |
| "index":"test" | |
| }, | |
| { | |
| "state":"STARTED", | |
| "primary":true, | |
| "node":"kH152vsLTL-y20mcLFs9GQ", | |
| "relocating_node":null, | |
| "shard":0, | |
| "index":"test123" | |
| } | |
| ], | |
| "J47gdOIwQMq2GTmzzmzJBA":[ | |
| { | |
| "state":"STARTED", | |
| "primary":false, | |
| "node":"J47gdOIwQMq2GTmzzmzJBA", | |
| "relocating_node":null, | |
| "shard":0, | |
| "index":"test" | |
| }, | |
| { | |
| "state":"STARTED", | |
| "primary":false, | |
| "node":"J47gdOIwQMq2GTmzzmzJBA", | |
| "relocating_node":null, | |
| "shard":1, | |
| "index":"test" | |
| }, | |
| { | |
| "state":"RELOCATING", | |
| "primary":true, | |
| "node":"J47gdOIwQMq2GTmzzmzJBA", | |
| "relocating_node":"kH152vsLTL-y20mcLFs9GQ", | |
| "shard":2, | |
| "index":"test" | |
| } | |
| ] | |
| } | |
| }, | |
| "allocations":[ | |
| ] | |
| } | |
| } | |
| ## Check API stats - executed quickly after reroute API | |
| ## Executed on C1 and S1 just to be sure | |
| $ curl -XGET localhost:9200/_cluster/nodes/stats?all=true | |
| { | |
| "error": "ArithmeticException[Value cannot fit in an int: -2562047788015]", | |
| "status": 500 | |
| } | |
| ## Running curl -s -XGET 'http://localhost:9200/_nodes/_local/stats?all=true' on all nodes | |
| ## Ran while the shard was relocating and the error message was appearing. | |
| ## Performed in parallel with Ansible, hence the ansible formatting | |
| polyfractal@Debian-60-squeeze-64-minimal:~/ansible$ ansible es_cluster -u root -m raw -a "curl -s -XGET 'http://localhost:9200/_nodes/_local/stats?all=true&pretty'" | |
| 144.76.3.103 | success | rc=0 >> | |
| { | |
| "cluster_name" : "elasticsearch_benchmark", | |
| "nodes" : { | |
| "J47gdOIwQMq2GTmzzmzJBA" : { | |
| "timestamp" : 1361378828241, | |
| "name" : "S2", | |
| "transport_address" : "inet[/144.76.3.103:9300]", | |
| "hostname" : "Debian-60-squeeze-64-minimal", | |
| "indices" : { | |
| "store" : { | |
| "size" : "40.7gb", | |
| "size_in_bytes" : 43807647857, | |
| "throttle_time" : "0s", | |
| "throttle_time_in_millis" : 0 | |
| }, | |
| "docs" : { | |
| "count" : 9999900, | |
| "deleted" : 0 | |
| }, | |
| "indexing" : { | |
| "index_total" : 113768216, | |
| "index_time" : "140.1d", | |
| "index_time_in_millis" : 12107123757, | |
| "index_current" : 2003, | |
| "delete_total" : 0, | |
| "delete_time" : "0s", | |
| "delete_time_in_millis" : 0, | |
| "delete_current" : 0 | |
| }, | |
| "get" : { | |
| "total" : 1, | |
| "time" : "2ms", | |
| "time_in_millis" : 2, | |
| "exists_total" : 1, | |
| "exists_time" : "2ms", | |
| "exists_time_in_millis" : 2, | |
| "missing_total" : 0, | |
| "missing_time" : "0s", | |
| "missing_time_in_millis" : 0, | |
| "current" : 0 | |
| }, | |
| "search" : { | |
| "query_total" : 10008324, | |
| "query_time" : "7.3d", | |
| "query_time_in_millis" : 631980281, | |
| "query_current" : 0, | |
| "fetch_total" : 3330672, | |
| "fetch_time" : "48.2m", | |
| "fetch_time_in_millis" : 2897513, | |
| "fetch_current" : 0 | |
| }, | |
| "cache" : { | |
| "field_evictions" : 0, | |
| "field_size" : "0b", | |
| "field_size_in_bytes" : 0, | |
| "filter_count" : 0, | |
| "filter_evictions" : 0, | |
| "filter_size" : "-418936b", | |
| "filter_size_in_bytes" : -418936, | |
| "bloom_size" : "11.8mb", | |
| "bloom_size_in_bytes" : 12458464, | |
| "id_cache_size" : "0b", | |
| "id_cache_size_in_bytes" : 0 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 6942, | |
| "total_time" : "3.3h", | |
| "total_time_in_millis" : 11942103, | |
| "total_docs" : 176159645, | |
| "total_size" : "491.1gb", | |
| "total_size_in_bytes" : 527413889942 | |
| }, | |
| "refresh" : { | |
| "total" : 49917, | |
| "total_time" : "5.5h", | |
| "total_time_in_millis" : 19889282 | |
| }, | |
| "flush" : { | |
| "total" : 12197, | |
| "total_time" : "9.8h", | |
| "total_time_in_millis" : 35416826 | |
| } | |
| }, | |
| "os" : { | |
| "timestamp" : 1361378828241, | |
| "uptime" : "643 hours, 2 minutes and 19 seconds", | |
| "uptime_in_millis" : 2314939000, | |
| "load_average" : [ 0.06, 0.17, 2.17 ], | |
| "cpu" : { | |
| "sys" : 2, | |
| "user" : 15, | |
| "idle" : 78 | |
| }, | |
| "mem" : { | |
| "free" : "255.9mb", | |
| "free_in_bytes" : 268345344, | |
| "used" : "30.9gb", | |
| "used_in_bytes" : 33225302016, | |
| "free_percent" : 49, | |
| "used_percent" : 50, | |
| "actual_free" : "15.3gb", | |
| "actual_free_in_bytes" : 16495669248, | |
| "actual_used" : "15.8gb", | |
| "actual_used_in_bytes" : 16997978112 | |
| }, | |
| "swap" : { | |
| "used" : "160.7mb", | |
| "used_in_bytes" : 168534016, | |
| "free" : "3.8gb", | |
| "free_in_bytes" : 4125364224 | |
| } | |
| }, | |
| "process" : { | |
| "timestamp" : 1361378828241, | |
| "open_file_descriptors" : 809, | |
| "cpu" : { | |
| "percent" : 155, | |
| "sys" : "3 hours, 26 minutes, 8 seconds and 400 milliseconds", | |
| "sys_in_millis" : 12368400, | |
| "user" : "53 hours, 1 minute, 56 seconds and 480 milliseconds", | |
| "user_in_millis" : 190916480, | |
| "total" : "56 hours, 28 minutes, 4 seconds and 880 milliseconds", | |
| "total_in_millis" : 203284880 | |
| }, | |
| "mem" : { | |
| "resident" : "15gb", | |
| "resident_in_bytes" : 16106811392, | |
| "share" : "5.6mb", | |
| "share_in_bytes" : 5898240, | |
| "total_virtual" : "18.1gb", | |
| "total_virtual_in_bytes" : 19476443136 | |
| } | |
| }, | |
| "jvm" : { | |
| "timestamp" : 1361378828242, | |
| "uptime" : "595 hours, 1 minute, 17 seconds and 759 milliseconds", | |
| "uptime_in_millis" : 2142077759, | |
| "mem" : { | |
| "heap_used" : "11.2gb", | |
| "heap_used_in_bytes" : 12122468248, | |
| "heap_committed" : "15.9gb", | |
| "heap_committed_in_bytes" : 17110138880, | |
| "non_heap_used" : "62.2mb", | |
| "non_heap_used_in_bytes" : 65318680, | |
| "non_heap_committed" : "90mb", | |
| "non_heap_committed_in_bytes" : 94392320, | |
| "pools" : { | |
| "Code Cache" : { | |
| "used" : "20.6mb", | |
| "used_in_bytes" : 21648064, | |
| "max" : "48mb", | |
| "max_in_bytes" : 50331648, | |
| "peak_used" : "20.6mb", | |
| "peak_used_in_bytes" : 21688512, | |
| "peak_max" : "48mb", | |
| "peak_max_in_bytes" : 50331648 | |
| }, | |
| "Par Eden Space" : { | |
| "used" : "396.4mb", | |
| "used_in_bytes" : 415742080, | |
| "max" : "532.5mb", | |
| "max_in_bytes" : 558432256, | |
| "peak_used" : "532.5mb", | |
| "peak_used_in_bytes" : 558432256, | |
| "peak_max" : "532.5mb", | |
| "peak_max_in_bytes" : 558432256 | |
| }, | |
| "Par Survivor Space" : { | |
| "used" : "3.2mb", | |
| "used_in_bytes" : 3434896, | |
| "max" : "66.5mb", | |
| "max_in_bytes" : 69730304, | |
| "peak_used" : "66.5mb", | |
| "peak_used_in_bytes" : 69730304, | |
| "peak_max" : "66.5mb", | |
| "peak_max_in_bytes" : 69730304 | |
| }, | |
| "CMS Old Gen" : { | |
| "used" : "10.8gb", | |
| "used_in_bytes" : 11703291272, | |
| "max" : "15.3gb", | |
| "max_in_bytes" : 16481976320, | |
| "peak_used" : "11.5gb", | |
| "peak_used_in_bytes" : 12368076416, | |
| "peak_max" : "15.3gb", | |
| "peak_max_in_bytes" : 16481976320 | |
| }, | |
| "CMS Perm Gen" : { | |
| "used" : "41.6mb", | |
| "used_in_bytes" : 43670616, | |
| "max" : "82mb", | |
| "max_in_bytes" : 85983232, | |
| "peak_used" : "41.6mb", | |
| "peak_used_in_bytes" : 43670616, | |
| "peak_max" : "82mb", | |
| "peak_max_in_bytes" : 85983232 | |
| } | |
| } | |
| }, | |
| "threads" : { | |
| "count" : 81, | |
| "peak_count" : 1884 | |
| }, | |
| "gc" : { | |
| "collection_count" : 19102, | |
| "collection_time" : "12 minutes, 54 seconds and 59 milliseconds", | |
| "collection_time_in_millis" : 774059, | |
| "collectors" : { | |
| "ParNew" : { | |
| "collection_count" : 19098, | |
| "collection_time" : "12 minutes, 53 seconds and 246 milliseconds", | |
| "collection_time_in_millis" : 773246 | |
| }, | |
| "ConcurrentMarkSweep" : { | |
| "collection_count" : 4, | |
| "collection_time" : "813 milliseconds", | |
| "collection_time_in_millis" : 813 | |
| } | |
| } | |
| }, | |
| "buffer_pools" : { | |
| "direct" : { | |
| "count" : 2379, | |
| "used" : "141.9mb", | |
| "used_in_bytes" : 148831994, | |
| "total_capacity" : "141.9mb", | |
| "total_capacity_in_bytes" : 148831994 | |
| }, | |
| "mapped" : { | |
| "count" : 0, | |
| "used" : "0b", | |
| "used_in_bytes" : 0, | |
| "total_capacity" : "0b", | |
| "total_capacity_in_bytes" : 0 | |
| } | |
| } | |
| }, | |
| "thread_pool" : { | |
| "generic" : { | |
| "threads" : 2, | |
| "queue" : 0, | |
| "active" : 1, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 871104 | |
| }, | |
| "index" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 113342179 | |
| }, | |
| "flush" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "largest" : -1, | |
| "completed" : 12210 | |
| }, | |
| "cache" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "largest" : -1, | |
| "completed" : 117372 | |
| }, | |
| "get" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 1 | |
| }, | |
| "search" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 13235485 | |
| }, | |
| "snapshot" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "largest" : -1, | |
| "completed" : 13174 | |
| }, | |
| "merge" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "largest" : -1, | |
| "completed" : 54310 | |
| }, | |
| "percolate" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 0 | |
| }, | |
| "refresh" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "largest" : -1, | |
| "completed" : 49795 | |
| }, | |
| "management" : { | |
| "threads" : 5, | |
| "queue" : 0, | |
| "active" : 1, | |
| "largest" : -1, | |
| "completed" : 4907581 | |
| }, | |
| "bulk" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 8358 | |
| } | |
| }, | |
| "network" : { | |
| "tcp" : { | |
| "active_opens" : 925, | |
| "passive_opens" : 60644, | |
| "curr_estab" : 79, | |
| "in_segs" : 922723258, | |
| "out_segs" : 1051507108, | |
| "retrans_segs" : 153113, | |
| "estab_resets" : 186, | |
| "attempt_fails" : 438, | |
| "in_errs" : 202, | |
| "out_rsts" : 22280 | |
| } | |
| }, | |
| "fs" : { | |
| "timestamp" : 1361378828242, | |
| "data" : [ { | |
| "path" : "/var/lib/elasticsearch/elasticsearch_benchmark/nodes/0", | |
| "mount" : "/", | |
| "dev" : "/dev/md2", | |
| "total" : "1015.9gb", | |
| "total_in_bytes" : 1090848337920, | |
| "free" : "969.2gb", | |
| "free_in_bytes" : 1040711577600, | |
| "available" : "918gb", | |
| "available_in_bytes" : 985736056832, | |
| "disk_reads" : 6709955, | |
| "disk_writes" : 685277272, | |
| "disk_read_size" : "247.9gb", | |
| "disk_read_size_in_bytes" : 266215659520, | |
| "disk_write_size" : "2613.6gb", | |
| "disk_write_size_in_bytes" : 2806365339648, | |
| "disk_queue" : "0", | |
| "disk_service_time" : "0" | |
| } ] | |
| }, | |
| "transport" : { | |
| "server_open" : 36, | |
| "rx_count" : 182926385, | |
| "rx_size" : "138.1gb", | |
| "rx_size_in_bytes" : 148337613933, | |
| "tx_count" : 157961709, | |
| "tx_size" : "162gb", | |
| "tx_size_in_bytes" : 173972714666 | |
| }, | |
| "http" : { | |
| "current_open" : 3, | |
| "total_opened" : 4378 | |
| } | |
| } | |
| } | |
| } | |
| 144.76.3.102 | success | rc=0 >> | |
| { | |
| "cluster_name" : "elasticsearch_benchmark", | |
| "nodes" : { | |
| "ez-rzcnfSESrVYP6zTWksA" : { | |
| "timestamp" : 1361378828251, | |
| "name" : "S1", | |
| "transport_address" : "inet[/144.76.3.102:9300]", | |
| "hostname" : "Debian-60-squeeze-64-minimal", | |
| "indices" : { | |
| "store" : { | |
| "size" : "27.2gb", | |
| "size_in_bytes" : 29207177550, | |
| "throttle_time" : "0s", | |
| "throttle_time_in_millis" : 0 | |
| }, | |
| "docs" : { | |
| "count" : 6667075, | |
| "deleted" : 0 | |
| }, | |
| "indexing" : { | |
| "index_total" : 156881055, | |
| "index_time" : "125.4d", | |
| "index_time_in_millis" : 10841797463, | |
| "index_current" : 3168, | |
| "delete_total" : 0, | |
| "delete_time" : "0s", | |
| "delete_time_in_millis" : 0, | |
| "delete_current" : 0 | |
| }, | |
| "get" : { | |
| "total" : 0, | |
| "time" : "0s", | |
| "time_in_millis" : 0, | |
| "exists_total" : 0, | |
| "exists_time" : "0s", | |
| "exists_time_in_millis" : 0, | |
| "missing_total" : 0, | |
| "missing_time" : "0s", | |
| "missing_time_in_millis" : 0, | |
| "current" : 0 | |
| }, | |
| "search" : { | |
| "query_total" : 144519, | |
| "query_time" : "1.5h", | |
| "query_time_in_millis" : 5654687, | |
| "query_current" : 0, | |
| "fetch_total" : 112114, | |
| "fetch_time" : "4.5s", | |
| "fetch_time_in_millis" : 4590, | |
| "fetch_current" : 0 | |
| }, | |
| "cache" : { | |
| "field_evictions" : 0, | |
| "field_size" : "0b", | |
| "field_size_in_bytes" : 0, | |
| "filter_count" : -1, | |
| "filter_evictions" : 0, | |
| "filter_size" : "-834128b", | |
| "filter_size_in_bytes" : -834128, | |
| "bloom_size" : "5.9mb", | |
| "bloom_size_in_bytes" : 6250768, | |
| "id_cache_size" : "0b", | |
| "id_cache_size_in_bytes" : 0 | |
| }, | |
| "merges" : { | |
| "current" : 7, | |
| "current_docs" : 17917691, | |
| "current_size" : "19.8gb", | |
| "current_size_in_bytes" : 21282790288, | |
| "total" : 4652, | |
| "total_time" : "3.4h", | |
| "total_time_in_millis" : 12246838, | |
| "total_docs" : 312656615, | |
| "total_size" : "611.6gb", | |
| "total_size_in_bytes" : 656802646552 | |
| }, | |
| "refresh" : { | |
| "total" : 34371, | |
| "total_time" : "3.3h", | |
| "total_time_in_millis" : 11935634 | |
| }, | |
| "flush" : { | |
| "total" : 5790, | |
| "total_time" : "7.5h", | |
| "total_time_in_millis" : 27212611 | |
| } | |
| }, | |
| "os" : { | |
| "timestamp" : 1361378828251, | |
| "uptime" : "641 hours, 20 minutes and 34 seconds", | |
| "uptime_in_millis" : 2308834000, | |
| "load_average" : [ 0.02, 0.04, 0.16 ], | |
| "cpu" : { | |
| "sys" : 0, | |
| "user" : 0, | |
| "idle" : 97 | |
| }, | |
| "mem" : { | |
| "free" : "194.1mb", | |
| "free_in_bytes" : 203608064, | |
| "used" : "31gb", | |
| "used_in_bytes" : 33290039296, | |
| "free_percent" : 43, | |
| "used_percent" : 56, | |
| "actual_free" : "13.5gb", | |
| "actual_free_in_bytes" : 14563196928, | |
| "actual_used" : "17.6gb", | |
| "actual_used_in_bytes" : 18930450432 | |
| }, | |
| "swap" : { | |
| "used" : "178.3mb", | |
| "used_in_bytes" : 187031552, | |
| "free" : "3.8gb", | |
| "free_in_bytes" : 4106866688 | |
| } | |
| }, | |
| "process" : { | |
| "timestamp" : 1361378828252, | |
| "open_file_descriptors" : 621, | |
| "cpu" : { | |
| "percent" : 0, | |
| "sys" : "3 hours, 34 minutes, 34 seconds and 920 milliseconds", | |
| "sys_in_millis" : 12874920, | |
| "user" : "30 hours, 35 minutes, 53 seconds and 940 milliseconds", | |
| "user_in_millis" : 110153940, | |
| "total" : "34 hours, 10 minutes, 28 seconds and 860 milliseconds", | |
| "total_in_millis" : 123028860 | |
| }, | |
| "mem" : { | |
| "resident" : "16.9gb", | |
| "resident_in_bytes" : 18203938816, | |
| "share" : "5.3mb", | |
| "share_in_bytes" : 5623808, | |
| "total_virtual" : "17.3gb", | |
| "total_virtual_in_bytes" : 18643701760 | |
| } | |
| }, | |
| "jvm" : { | |
| "timestamp" : 1361378828252, | |
| "uptime" : "160 hours, 37 minutes, 33 seconds and 722 milliseconds", | |
| "uptime_in_millis" : 578253722, | |
| "mem" : { | |
| "heap_used" : "8.1gb", | |
| "heap_used_in_bytes" : 8735667352, | |
| "heap_committed" : "15.9gb", | |
| "heap_committed_in_bytes" : 17110138880, | |
| "non_heap_used" : "59.5mb", | |
| "non_heap_used_in_bytes" : 62454432, | |
| "non_heap_committed" : "86.8mb", | |
| "non_heap_committed_in_bytes" : 91029504, | |
| "pools" : { | |
| "Code Cache" : { | |
| "used" : "18.7mb", | |
| "used_in_bytes" : 19685824, | |
| "max" : "48mb", | |
| "max_in_bytes" : 50331648, | |
| "peak_used" : "18.8mb", | |
| "peak_used_in_bytes" : 19777408, | |
| "peak_max" : "48mb", | |
| "peak_max_in_bytes" : 50331648 | |
| }, | |
| "Par Eden Space" : { | |
| "used" : "361.3mb", | |
| "used_in_bytes" : 378866504, | |
| "max" : "532.5mb", | |
| "max_in_bytes" : 558432256, | |
| "peak_used" : "532.5mb", | |
| "peak_used_in_bytes" : 558432256, | |
| "peak_max" : "532.5mb", | |
| "peak_max_in_bytes" : 558432256 | |
| }, | |
| "Par Survivor Space" : { | |
| "used" : "3.1mb", | |
| "used_in_bytes" : 3267968, | |
| "max" : "66.5mb", | |
| "max_in_bytes" : 69730304, | |
| "peak_used" : "66.5mb", | |
| "peak_used_in_bytes" : 69730304, | |
| "peak_max" : "66.5mb", | |
| "peak_max_in_bytes" : 69730304 | |
| }, | |
| "CMS Old Gen" : { | |
| "used" : "7.7gb", | |
| "used_in_bytes" : 8353532880, | |
| "max" : "15.3gb", | |
| "max_in_bytes" : 16481976320, | |
| "peak_used" : "15.3gb", | |
| "peak_used_in_bytes" : 16459831872, | |
| "peak_max" : "15.3gb", | |
| "peak_max_in_bytes" : 16481976320 | |
| }, | |
| "CMS Perm Gen" : { | |
| "used" : "40.7mb", | |
| "used_in_bytes" : 42768608, | |
| "max" : "82mb", | |
| "max_in_bytes" : 85983232, | |
| "peak_used" : "40.7mb", | |
| "peak_used_in_bytes" : 42768608, | |
| "peak_max" : "82mb", | |
| "peak_max_in_bytes" : 85983232 | |
| } | |
| } | |
| }, | |
| "threads" : { | |
| "count" : 77, | |
| "peak_count" : 1884 | |
| }, | |
| "gc" : { | |
| "collection_count" : 15317, | |
| "collection_time" : "13 minutes, 11 seconds and 740 milliseconds", | |
| "collection_time_in_millis" : 791740, | |
| "collectors" : { | |
| "ParNew" : { | |
| "collection_count" : 15203, | |
| "collection_time" : "12 minutes, 37 seconds and 24 milliseconds", | |
| "collection_time_in_millis" : 757024 | |
| }, | |
| "ConcurrentMarkSweep" : { | |
| "collection_count" : 114, | |
| "collection_time" : "34 seconds and 716 milliseconds", | |
| "collection_time_in_millis" : 34716 | |
| } | |
| } | |
| }, | |
| "buffer_pools" : { | |
| "direct" : { | |
| "count" : 2010, | |
| "used" : "134mb", | |
| "used_in_bytes" : 140594624, | |
| "total_capacity" : "134mb", | |
| "total_capacity_in_bytes" : 140594624 | |
| }, | |
| "mapped" : { | |
| "count" : 0, | |
| "used" : "0b", | |
| "used_in_bytes" : 0, | |
| "total_capacity" : "0b", | |
| "total_capacity_in_bytes" : 0 | |
| } | |
| } | |
| }, | |
| "thread_pool" : { | |
| "generic" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 256268 | |
| }, | |
| "index" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 156850395 | |
| }, | |
| "flush" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "largest" : -1, | |
| "completed" : 5861 | |
| }, | |
| "cache" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "largest" : -1, | |
| "completed" : 60041 | |
| }, | |
| "get" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 0 | |
| }, | |
| "search" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 195127 | |
| }, | |
| "snapshot" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "largest" : -1, | |
| "completed" : 8411 | |
| }, | |
| "merge" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "largest" : -1, | |
| "completed" : 37295 | |
| }, | |
| "percolate" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 0 | |
| }, | |
| "refresh" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "largest" : -1, | |
| "completed" : 34264 | |
| }, | |
| "management" : { | |
| "threads" : 5, | |
| "queue" : 0, | |
| "active" : 1, | |
| "largest" : -1, | |
| "completed" : 1568233 | |
| }, | |
| "bulk" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 4364 | |
| } | |
| }, | |
| "network" : { | |
| "tcp" : { | |
| "active_opens" : 873, | |
| "passive_opens" : 193538, | |
| "curr_estab" : 86, | |
| "in_segs" : 2026668822, | |
| "out_segs" : 2090425545, | |
| "retrans_segs" : 173838, | |
| "estab_resets" : 5088, | |
| "attempt_fails" : 450, | |
| "in_errs" : 242, | |
| "out_rsts" : 18050 | |
| } | |
| }, | |
| "fs" : { | |
| "timestamp" : 1361378828253, | |
| "data" : [ { | |
| "path" : "/var/lib/elasticsearch/elasticsearch_benchmark/nodes/0", | |
| "mount" : "/", | |
| "dev" : "/dev/md2", | |
| "total" : "1015.9gb", | |
| "total_in_bytes" : 1090848337920, | |
| "free" : "984gb", | |
| "free_in_bytes" : 1056649801728, | |
| "available" : "932.8gb", | |
| "available_in_bytes" : 1001674280960, | |
| "disk_reads" : 5733548, | |
| "disk_writes" : 737387651, | |
| "disk_read_size" : "194.6gb", | |
| "disk_read_size_in_bytes" : 209036359680, | |
| "disk_write_size" : "2812.4gb", | |
| "disk_write_size_in_bytes" : 3019828723712, | |
| "disk_queue" : "0", | |
| "disk_service_time" : "0" | |
| } ] | |
| }, | |
| 144.76.2.205 | success | rc=0 >> | |
| { | |
| "cluster_name" : "elasticsearch_benchmark", | |
| "nodes" : { | |
| "kH152vsLTL-y20mcLFs9GQ" : { | |
| "timestamp" : 1361378828252, | |
| "name" : "S3", | |
| "transport_address" : "inet[/144.76.2.205:9300]", | |
| "hostname" : "Debian-60-squeeze-64-minimal", | |
| "indices" : { | |
| "store" : { | |
| "size" : "16.5gb", | |
| "size_in_bytes" : 17750418910, | |
| "throttle_time" : "0s", | |
| "throttle_time_in_millis" : 0 | |
| }, | |
| "docs" : { | |
| "count" : 3332825, | |
| "deleted" : 0 | |
| }, | |
| "indexing" : { | |
| "index_total" : 109079109, | |
| "index_time" : "74.3d", | |
| "index_time_in_millis" : 6425954679, | |
| "index_current" : 52535, | |
| "delete_total" : 0, | |
| "delete_time" : "0s", | |
| "delete_time_in_millis" : 0, | |
| "delete_current" : 0 | |
| }, | |
| "get" : { | |
| "total" : 2, | |
| "time" : "2ms", | |
| "time_in_millis" : 2, | |
| "exists_total" : 1, | |
| "exists_time" : "2ms", | |
| "exists_time_in_millis" : 2, | |
| "missing_total" : 1, | |
| "missing_time" : "0s", | |
| "missing_time_in_millis" : 0, | |
| "current" : 0 | |
| }, | |
| "search" : { | |
| "query_total" : 9837062, | |
| "query_time" : "3.1d", | |
| "query_time_in_millis" : 274939193, | |
| "query_current" : 0, | |
| "fetch_total" : 3442003, | |
| "fetch_time" : "25.1m", | |
| "fetch_time_in_millis" : 1511150, | |
| "fetch_current" : 0 | |
| }, | |
| "cache" : { | |
| "field_evictions" : 0, | |
| "field_size" : "0b", | |
| "field_size_in_bytes" : 0, | |
| "filter_count" : 0, | |
| "filter_evictions" : 0, | |
| "filter_size" : "-834192b", | |
| "filter_size_in_bytes" : -834192, | |
| "bloom_size" : "0b", | |
| "bloom_size_in_bytes" : 0, | |
| "id_cache_size" : "0b", | |
| "id_cache_size_in_bytes" : 0 | |
| }, | |
| "merges" : { | |
| "current" : 1, | |
| "current_docs" : 19179, | |
| "current_size" : "87.5mb", | |
| "current_size_in_bytes" : 91763722, | |
| "total" : 4723, | |
| "total_time" : "1.9h", | |
| "total_time_in_millis" : 7156548, | |
| "total_docs" : 188705565, | |
| "total_size" : "328.1gb", | |
| "total_size_in_bytes" : 352332073622 | |
| }, | |
| "refresh" : { | |
| "total" : 35642, | |
| "total_time" : "4h", | |
| "total_time_in_millis" : 14487592 | |
| }, | |
| "flush" : { | |
| "total" : 11468, | |
| "total_time" : "6.5h", | |
| "total_time_in_millis" : 23574420 | |
| } | |
| }, | |
| "os" : { | |
| "timestamp" : 1361378828253, | |
| "uptime" : "641 hours, 12 minutes and 22 seconds", | |
| "uptime_in_millis" : 2308342000, | |
| "load_average" : [ 0.42, 0.16, 0.5 ], | |
| "cpu" : { | |
| "sys" : 3, | |
| "user" : 9, | |
| "idle" : 82 | |
| }, | |
| "mem" : { | |
| "free" : "5.6gb", | |
| "free_in_bytes" : 6099591168, | |
| "used" : "25.5gb", | |
| "used_in_bytes" : 27394056192, | |
| "free_percent" : 54, | |
| "used_percent" : 45, | |
| "actual_free" : "16.9gb", | |
| "actual_free_in_bytes" : 18164064256, | |
| "actual_used" : "14.2gb", | |
| "actual_used_in_bytes" : 15329583104 | |
| }, | |
| "swap" : { | |
| "used" : "13.5mb", | |
| "used_in_bytes" : 14209024, | |
| "free" : "3.9gb", | |
| "free_in_bytes" : 4279689216 | |
| } | |
| }, | |
| "process" : { | |
| "timestamp" : 1361378828253, | |
| "open_file_descriptors" : 439, | |
| "cpu" : { | |
| "percent" : 111, | |
| "sys" : "2 hours, 41 minutes, 32 seconds and 310 milliseconds", | |
| "sys_in_millis" : 9692310, | |
| "user" : "36 hours, 9 minutes, 59 seconds and 820 milliseconds", | |
| "user_in_millis" : 130199820, | |
| "total" : "38 hours, 51 minutes, 32 seconds and 130 milliseconds", | |
| "total_in_millis" : 139892130 | |
| }, | |
| "mem" : { | |
| "resident" : "13.5gb", | |
| "resident_in_bytes" : 14519554048, | |
| "share" : "6mb", | |
| "share_in_bytes" : 6332416, | |
| "total_virtual" : "17.7gb", | |
| "total_virtual_in_bytes" : 19038695424 | |
| } | |
| }, | |
| "jvm" : { | |
| "timestamp" : 1361378828253, | |
| "uptime" : "594 hours, 33 minutes, 44 seconds and 331 milliseconds", | |
| "uptime_in_millis" : 2140424331, | |
| "mem" : { | |
| "heap_used" : "7.8gb", | |
| "heap_used_in_bytes" : 8469861056, | |
| "heap_committed" : "15.9gb", | |
| "heap_committed_in_bytes" : 17110138880, | |
| "non_heap_used" : "60.1mb", | |
| "non_heap_used_in_bytes" : 63109280, | |
| "non_heap_committed" : "86.9mb", | |
| "non_heap_committed_in_bytes" : 91156480, | |
| "pools" : { | |
| "Code Cache" : { | |
| "used" : "19.4mb", | |
| "used_in_bytes" : 20395264, | |
| "max" : "48mb", | |
| "max_in_bytes" : 50331648, | |
| "peak_used" : "19.4mb", | |
| "peak_used_in_bytes" : 20406016, | |
| "peak_max" : "48mb", | |
| "peak_max_in_bytes" : 50331648 | |
| }, | |
| "Par Eden Space" : { | |
| "used" : "425mb", | |
| "used_in_bytes" : 445647592, | |
| "max" : "532.5mb", | |
| "max_in_bytes" : 558432256, | |
| "peak_used" : "532.5mb", | |
| "peak_used_in_bytes" : 558432256, | |
| "peak_max" : "532.5mb", | |
| "peak_max_in_bytes" : 558432256 | |
| }, | |
| "Par Survivor Space" : { | |
| "used" : "1.4mb", | |
| "used_in_bytes" : 1492456, | |
| "max" : "66.5mb", | |
| "max_in_bytes" : 69730304, | |
| "peak_used" : "66.5mb", | |
| "peak_used_in_bytes" : 69730304, | |
| "peak_max" : "66.5mb", | |
| "peak_max_in_bytes" : 69730304 | |
| }, | |
| "CMS Old Gen" : { | |
| "used" : "7.4gb", | |
| "used_in_bytes" : 8022723080, | |
| "max" : "15.3gb", | |
| "max_in_bytes" : 16481976320, | |
| "peak_used" : "11.5gb", | |
| "peak_used_in_bytes" : 12365203168, | |
| "peak_max" : "15.3gb", | |
| "peak_max_in_bytes" : 16481976320 | |
| }, | |
| "CMS Perm Gen" : { | |
| "used" : "40.7mb", | |
| "used_in_bytes" : 42714016, | |
| "max" : "82mb", | |
| "max_in_bytes" : 85983232, | |
| "peak_used" : "40.7mb", | |
| "peak_used_in_bytes" : 42714016, | |
| "peak_max" : "82mb", | |
| "peak_max_in_bytes" : 85983232 | |
| } | |
| } | |
| }, | |
| "threads" : { | |
| "count" : 80, | |
| "peak_count" : 1883 | |
| }, | |
| "gc" : { | |
| "collection_count" : 11614, | |
| "collection_time" : "8 minutes, 18 seconds and 107 milliseconds", | |
| "collection_time_in_millis" : 498107, | |
| "collectors" : { | |
| "ParNew" : { | |
| "collection_count" : 11612, | |
| "collection_time" : "8 minutes, 17 seconds and 412 milliseconds", | |
| "collection_time_in_millis" : 497412 | |
| }, | |
| "ConcurrentMarkSweep" : { | |
| "collection_count" : 2, | |
| "collection_time" : "695 milliseconds", | |
| "collection_time_in_millis" : 695 | |
| } | |
| } | |
| }, | |
| "buffer_pools" : { | |
| "direct" : { | |
| "count" : 2768, | |
| "used" : "169.9mb", | |
| "used_in_bytes" : 178231097, | |
| "total_capacity" : "169.9mb", | |
| "total_capacity_in_bytes" : 178231097 | |
| }, | |
| "mapped" : { | |
| "count" : 0, | |
| "used" : "0b", | |
| "used_in_bytes" : 0, | |
| "total_capacity" : "0b", | |
| "total_capacity_in_bytes" : 0 | |
| } | |
| } | |
| }, | |
| "thread_pool" : { | |
| "generic" : { | |
| "threads" : 4, | |
| "queue" : 0, | |
| "active" : 2, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 454456 | |
| }, | |
| "index" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 109099469 | |
| }, | |
| "flush" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "largest" : -1, | |
| "completed" : 11480 | |
| }, | |
| "cache" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "largest" : -1, | |
| "completed" : 78112 | |
| }, | |
| "get" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 5 | |
| }, | |
| "search" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 13308545 | |
| }, | |
| "snapshot" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "largest" : -1, | |
| "completed" : 10045 | |
| }, | |
| "merge" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "largest" : -1, | |
| "completed" : 42342 | |
| }, | |
| "percolate" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 0 | |
| }, | |
| "refresh" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "largest" : -1, | |
| "completed" : 35451 | |
| }, | |
| "management" : { | |
| "threads" : 5, | |
| "queue" : 0, | |
| "active" : 1, | |
| "largest" : -1, | |
| "completed" : 4943344 | |
| }, | |
| "bulk" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 4364 | |
| } | |
| }, | |
| "network" : { | |
| "tcp" : { | |
| "active_opens" : 21642, | |
| "passive_opens" : 87901, | |
| "curr_estab" : 79, | |
| "in_segs" : 394139093, | |
| "out_segs" : 417270877, | |
| "retrans_segs" : 28062, | |
| "estab_resets" : 177, | |
| "attempt_fails" : 370, | |
| "in_errs" : 75, | |
| "out_rsts" : 6892 | |
| } | |
| }, | |
| "fs" : { | |
| "timestamp" : 1361378828254, | |
| "data" : [ { | |
| "path" : "/var/lib/elasticsearch/elasticsearch_benchmark/nodes/0", | |
| "mount" : "/", | |
| "dev" : "/dev/md2", | |
| "total" : "1015.9gb", | |
| "total_in_bytes" : 1090848337920, | |
| "free" : "994.7gb", | |
| "free_in_bytes" : 1068083478528, | |
| "available" : "943.5gb", | |
| "available_in_bytes" : 1013107957760, | |
| "disk_reads" : 380878, | |
| "disk_writes" : 331184225, | |
| "disk_read_size" : "20.4gb", | |
| "disk_read_ "transport" : { | |
| "server_open" : 36, | |
| "rx_count" : 151218350, | |
| "rx_size" : "144.3gb", | |
| "rx_size_in_bytes" : 155045029541, | |
| size_in_bytes" : 21938227200, | |
| "tx_count" : 125459795, | |
| "disk_write_size" : "1263gb", | |
| "tx_size" : "93.5gb", | |
| "disk_write_size_in_bytes" : 1356241309696, | |
| "tx_size_in_bytes" : 100429683098 | |
| "disk_queue" : "0", | |
| }, | |
| "disk_service_time" : "0" | |
| "http" : { | |
| } ] | |
| "current_open" : 9, | |
| }, | |
| "total_opened" : 24474 | |
| "transport" : { | |
| } | |
| "server_open" : 36, | |
| } | |
| "rx_count" : 167290341, | |
| } | |
| "rx_size" : "155.9gb", | |
| } | |
| "rx_size_in_bytes" : 167495293626, | |
| "tx_count" : 142811276, | |
| "tx_size" : "70.6gb", | |
| "tx_size_in_bytes" : 75859638076 | |
| }, | |
| "http" : { | |
| "current_open" : 3, | |
| "total_opened" : 6 | |
| } | |
| } | |
| } | |
| } | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment