Skip to content

Instantly share code, notes, and snippets.

@rsmoorthy

rsmoorthy/es541_cluster_allocation_explain

Last active October 12, 2018 19:33
Show Gist options
  • Save rsmoorthy/061114e063d9a2ad02f06a128cf68240 to your computer and use it in GitHub Desktop.
Save rsmoorthy/061114e063d9a2ad02f06a128cf68240 to your computer and use it in GitHub Desktop.
Download ZIP
ElasticSearch issues
Raw
es541_cluster_allocation_explain
# curl -XGET localhost:9200/_cluster/allocation/explain?pretty
{
"index" : "filebeat-2017.09.08",
"shard" : 0,
"primary" : true,
"current_state" : "unassigned",
"unassigned_info" : {
"reason" : "CLUSTER_RECOVERED",
"at" : "2018-10-12T19:09:07.740Z",
"last_allocation_status" : "no_valid_shard_copy"
},
"can_allocate" : "no_valid_shard_copy",
"allocate_explanation" : "cannot allocate because all found copies of the shard are either stale or corrupt",
"node_allocation_decisions" : [
{
"node_id" : "5awoLm1yQ0yOkBdLy3FTzQ",
"node_name" : "5awoLm1",
"transport_address" : "172.18.0.2:9300",
"node_decision" : "no",
"store" : {
"in_sync" : true,
"allocation_id" : "l9SRskjDRDeq0PcYbXrKvA",
"store_exception" : {
"type" : "index_format_too_new_exception",
"reason" : "Format version is not supported (resource BufferedChecksumIndexInput(SimpleFSIndexInput(path=\"/var/lib/elasticsearch/nodes/0/indices/xy3ktob4SN-CJoy05XkxWQ/0/index/segments_30\"))): 9 (needs to be between 4 and 6)"
}
}
}
]
}
Raw
es541_ingest_pipelines
# curl 'localhost:9200/_ingest/pipeline?pretty=1'
{
"xpack_monitoring_2" : {
"description" : "This pipeline upgrades documents from the older version of the Monitoring API to the newer version (6) by fixing breaking changes in those older documents before they are indexed from the older version (2).",
"version" : 6040099,
"processors" : [
{
"script" : {
"source" : "boolean legacyIndex = ctx._index == '.monitoring-data-2';if (legacyIndex || ctx._index.startsWith('.monitoring-es-2')) {if (ctx._type == 'cluster_info') {ctx._type = 'cluster_stats';ctx._id = null;} else if (legacyIndex || ctx._type == 'cluster_stats' || ctx._type == 'node') {String index = ctx._index;Object clusterUuid = ctx.cluster_uuid;Object timestamp = ctx.timestamp;ctx.clear();ctx._id = 'xpack_monitoring_2_drop_bucket';ctx._index = index;ctx._type = 'legacy_data';ctx.timestamp = timestamp;ctx.cluster_uuid = clusterUuid;}if (legacyIndex) {ctx._index = '<.monitoring-es-6-{now}>';}}"
}
},
{
"rename" : {
"field" : "_type",
"target_field" : "type"
}
},
{
"set" : {
"field" : "_type",
"value" : "doc"
}
},
{
"gsub" : {
"field" : "_index",
"pattern" : "(.monitoring-\\w+-)2(-.+)",
"replacement" : "$16$2"
}
}
]
},
"xpack_monitoring_6" : {
"description" : "This is a placeholder pipeline for Monitoring API version 6 so that future versions may fix breaking changes.",
"version" : 6040099,
"processors" : [ ]
}
}
Raw
es541_startup.health
# curl localhost:9200/_cluster/health?pretty
{
"cluster_name" : "elasticsearch",
"status" : "red",
"timed_out" : false,
"number_of_nodes" : 1,
"number_of_data_nodes" : 1,
"active_primary_shards" : 0,
"active_shards" : 0,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 1113,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 0.0
}
Raw
es_541_startup.error
[2018-10-12T18:11:33,207][INFO ][o.e.n.Node ] [] initializing ...
[2018-10-12T18:11:33,261][INFO ][o.e.e.NodeEnvironment ] [5awoLm1] using [1] data paths, mounts [[/var/lib/elasticsearch (/dev/nvme1n1)]], net usable_space [229.1gb], net total_space [1.4tb], spins? [possibly], types [ext4]
[2018-10-12T18:11:33,262][INFO ][o.e.e.NodeEnvironment ] [5awoLm1] heap size [21.9gb], compressed ordinary object pointers [true]
[2018-10-12T18:11:34,498][INFO ][o.e.n.Node ] node name [5awoLm1] derived from node ID [5awoLm1yQ0yOkBdLy3FTzQ]; set [node.name] to override
[2018-10-12T18:11:34,499][INFO ][o.e.n.Node ] version[5.4.1], pid[411], build[2cfe0df/2017-05-29T16:05:51.443Z], OS[Linux/3.10.0-862.11.6.el7.x86_64/amd64], JVM[Oracle Corporation/OpenJDK 64-Bit Server VM/1.8.0_131/25.131-b11]
[2018-10-12T18:11:34,499][INFO ][o.e.n.Node ] JVM arguments [-Xms22g, -Xmx22g, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75, -XX:+UseCMSInitiatingOccupancyOnly, -XX:+DisableExplicitGC, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -Djdk.io.permissionsUseCanonicalPath=true, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Dlog4j.skipJansi=true, -XX:+HeapDumpOnOutOfMemoryError, -Dio.netty.recycler.maxCapacityPerThread=0, -Des.path.home=/opt/elasticsearch]
[2018-10-12T18:11:35,239][INFO ][o.e.p.PluginsService ] [5awoLm1] loaded module [aggs-matrix-stats]
[2018-10-12T18:11:35,239][INFO ][o.e.p.PluginsService ] [5awoLm1] loaded module [ingest-common]
[2018-10-12T18:11:35,239][INFO ][o.e.p.PluginsService ] [5awoLm1] loaded module [lang-expression]
[2018-10-12T18:11:35,239][INFO ][o.e.p.PluginsService ] [5awoLm1] loaded module [lang-groovy]
[2018-10-12T18:11:35,239][INFO ][o.e.p.PluginsService ] [5awoLm1] loaded module [lang-mustache]
[2018-10-12T18:11:35,239][INFO ][o.e.p.PluginsService ] [5awoLm1] loaded module [lang-painless]
[2018-10-12T18:11:35,239][INFO ][o.e.p.PluginsService ] [5awoLm1] loaded module [percolator]
[2018-10-12T18:11:35,239][INFO ][o.e.p.PluginsService ] [5awoLm1] loaded module [reindex]
[2018-10-12T18:11:35,239][INFO ][o.e.p.PluginsService ] [5awoLm1] loaded module [transport-netty3]
[2018-10-12T18:11:35,240][INFO ][o.e.p.PluginsService ] [5awoLm1] loaded module [transport-netty4]
[2018-10-12T18:11:35,240][INFO ][o.e.p.PluginsService ] [5awoLm1] no plugins loaded
[2018-10-12T18:11:36,442][INFO ][o.e.d.DiscoveryModule ] [5awoLm1] using discovery type [zen]
[2018-10-12T18:11:40,530][INFO ][o.e.n.Node ] initialized
[2018-10-12T18:11:40,530][INFO ][o.e.n.Node ] [5awoLm1] starting ...
[2018-10-12T18:11:40,647][INFO ][o.e.t.TransportService ] [5awoLm1] publish_address {172.18.0.2:9300}, bound_addresses {[::]:9300}
[2018-10-12T18:11:40,651][INFO ][o.e.b.BootstrapChecks ] [5awoLm1] bound or publishing to a non-loopback or non-link-local address, enforcing bootstrap checks
[2018-10-12T18:11:43,691][INFO ][o.e.c.s.ClusterService ] [5awoLm1] new_master {5awoLm1}{5awoLm1yQ0yOkBdLy3FTzQ}{ycEDt_K0SOG9xEQrhNpN5Q}{172.18.0.2}{172.18.0.2:9300}, reason: zen-disco-elected-as-master ([0] nodes joined)
[2018-10-12T18:11:43,703][INFO ][o.e.h.n.Netty4HttpServerTransport] [5awoLm1] publish_address {172.18.0.2:9200}, bound_addresses {[::]:9200}
[2018-10-12T18:11:43,704][INFO ][o.e.n.Node ] [5awoLm1] started
[2018-10-12T18:11:46,554][INFO ][o.e.m.j.JvmGcMonitorService] [5awoLm1] [gc][6] overhead, spent [257ms] collecting in the last [1s]
[2018-10-12T18:11:47,555][INFO ][o.e.m.j.JvmGcMonitorService] [5awoLm1] [gc][7] overhead, spent [300ms] collecting in the last [1s]
[2018-10-12T18:11:49,556][INFO ][o.e.m.j.JvmGcMonitorService] [5awoLm1] [gc][9] overhead, spent [290ms] collecting in the last [1s]
[2018-10-12T18:11:50,720][WARN ][o.e.c.s.ClusterService ] [5awoLm1] failed to notify ClusterStateApplier
org.elasticsearch.ElasticsearchParseException: Need [file], [id], or [inline] parameter to refer to scripts
at org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException(ConfigurationUtils.java:233) ~[elasticsearch-5.4.1.jar:5.4.1]
at org.elasticsearch.ingest.common.ScriptProcessor$Factory.create(ScriptProcessor.java:107) ~[?:?]
at org.elasticsearch.ingest.common.ScriptProcessor$Factory.create(ScriptProcessor.java:87) ~[?:?]
at org.elasticsearch.ingest.ConfigurationUtils.readProcessor(ConfigurationUtils.java:307) ~[elasticsearch-5.4.1.jar:5.4.1]
at org.elasticsearch.ingest.ConfigurationUtils.readProcessorConfigs(ConfigurationUtils.java:253) ~[elasticsearch-5.4.1.jar:5.4.1]
at org.elasticsearch.ingest.Pipeline$Factory.create(Pipeline.java:122) ~[elasticsearch-5.4.1.jar:5.4.1]
at org.elasticsearch.ingest.PipelineStore.innerUpdatePipelines(PipelineStore.java:81) ~[elasticsearch-5.4.1.jar:5.4.1]
at org.elasticsearch.ingest.PipelineStore.applyClusterState(PipelineStore.java:68) ~[elasticsearch-5.4.1.jar:5.4.1]
at org.elasticsearch.cluster.service.ClusterService.callClusterStateAppliers(ClusterService.java:815) [elasticsearch-5.4.1.jar:5.4.1]
at org.elasticsearch.cluster.service.ClusterService.publishAndApplyChanges(ClusterService.java:769) [elasticsearch-5.4.1.jar:5.4.1]
at org.elasticsearch.cluster.service.ClusterService.runTasks(ClusterService.java:586) [elasticsearch-5.4.1.jar:5.4.1]
at org.elasticsearch.cluster.service.ClusterService$ClusterServiceTaskBatcher.run(ClusterService.java:262) [elasticsearch-5.4.1.jar:5.4.1]
at org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150) [elasticsearch-5.4.1.jar:5.4.1]
at org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188) [elasticsearch-5.4.1.jar:5.4.1]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:569) [elasticsearch-5.4.1.jar:5.4.1]
at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:247) [elasticsearch-5.4.1.jar:5.4.1]
at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:210) [elasticsearch-5.4.1.jar:5.4.1]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_131]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_131]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_131]
[2018-10-12T18:11:55,569][INFO ][o.e.g.GatewayService ] [5awoLm1] recovered [1113] indices into cluster_state
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment