Created
August 17, 2020 20:18
-
-
Save jerrac/451abb45fd88810288af63cba8237d7a to your computer and use it in GitHub Desktop.
elasticsearch troubleshooting summer 2020
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [2020-08-17T12:57:58,747][INFO ][o.e.n.Node ] [esnode3.example.org] version[7.8.0], pid[19625], build[default/deb/757314695644ea9a1dc2fecd26d1a43856725e65/2020-06-14T19:35:50.234439Z], OS[Linux/4.15.0-112-generic/amd64], JVM[AdoptOpenJDK/OpenJDK 64-Bit Server VM/14.0.1/14.0.1+7] | |
| [2020-08-17T12:57:58,751][INFO ][o.e.n.Node ] [esnode3.example.org] JVM home [/usr/share/elasticsearch/jdk] | |
| [2020-08-17T12:57:58,751][INFO ][o.e.n.Node ] [esnode3.example.org] JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, -Xms6g, -Xmx6g, -XX:+UseG1GC, -XX:G1ReservePercent=25, -XX:InitiatingHeapOccupancyPercent=30, -Djava.io.tmpdir=/tmp/elasticsearch-17118603538019497405, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=/var/lib/elasticsearch, -XX:ErrorFile=/mnt/elasticsearch_iscsi/testelk03/logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=/mnt/elasticsearch_iscsi/testelk03/logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -XX:MaxDirectMemorySize=3221225472, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/etc/elasticsearch, -Des.distribution.flavor=default, -Des.distribution.type=deb, -Des.bundled_jdk=true] | |
| [2020-08-17T12:58:04,848][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [aggs-matrix-stats] | |
| [2020-08-17T12:58:04,848][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [analysis-common] | |
| [2020-08-17T12:58:04,849][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [constant-keyword] | |
| [2020-08-17T12:58:04,849][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [flattened] | |
| [2020-08-17T12:58:04,850][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [frozen-indices] | |
| [2020-08-17T12:58:04,850][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [ingest-common] | |
| [2020-08-17T12:58:04,850][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [ingest-geoip] | |
| [2020-08-17T12:58:04,851][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [ingest-user-agent] | |
| [2020-08-17T12:58:04,851][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [kibana] | |
| [2020-08-17T12:58:04,851][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [lang-expression] | |
| [2020-08-17T12:58:04,851][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [lang-mustache] | |
| [2020-08-17T12:58:04,852][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [lang-painless] | |
| [2020-08-17T12:58:04,852][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [mapper-extras] | |
| [2020-08-17T12:58:04,852][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [parent-join] | |
| [2020-08-17T12:58:04,852][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [percolator] | |
| [2020-08-17T12:58:04,853][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [rank-eval] | |
| [2020-08-17T12:58:04,853][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [reindex] | |
| [2020-08-17T12:58:04,853][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [repository-url] | |
| [2020-08-17T12:58:04,854][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [search-business-rules] | |
| [2020-08-17T12:58:04,854][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [searchable-snapshots] | |
| [2020-08-17T12:58:04,854][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [spatial] | |
| [2020-08-17T12:58:04,854][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [systemd] | |
| [2020-08-17T12:58:04,855][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [tasks] | |
| [2020-08-17T12:58:04,855][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [transform] | |
| [2020-08-17T12:58:04,855][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [transport-netty4] | |
| [2020-08-17T12:58:04,855][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [vectors] | |
| [2020-08-17T12:58:04,856][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-analytics] | |
| [2020-08-17T12:58:04,856][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-async-search] | |
| [2020-08-17T12:58:04,856][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-autoscaling] | |
| [2020-08-17T12:58:04,856][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-ccr] | |
| [2020-08-17T12:58:04,857][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-core] | |
| [2020-08-17T12:58:04,857][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-deprecation] | |
| [2020-08-17T12:58:04,857][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-enrich] | |
| [2020-08-17T12:58:04,857][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-eql] | |
| [2020-08-17T12:58:04,858][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-graph] | |
| [2020-08-17T12:58:04,858][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-identity-provider] | |
| [2020-08-17T12:58:04,858][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-ilm] | |
| [2020-08-17T12:58:04,858][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-logstash] | |
| [2020-08-17T12:58:04,859][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-ml] | |
| [2020-08-17T12:58:04,859][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-monitoring] | |
| [2020-08-17T12:58:04,859][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-ql] | |
| [2020-08-17T12:58:04,859][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-rollup] | |
| [2020-08-17T12:58:04,860][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-security] | |
| [2020-08-17T12:58:04,860][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-sql] | |
| [2020-08-17T12:58:04,860][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-voting-only-node] | |
| [2020-08-17T12:58:04,860][INFO ][o.e.p.PluginsService ] [esnode3.example.org] loaded module [x-pack-watcher] | |
| [2020-08-17T12:58:04,861][INFO ][o.e.p.PluginsService ] [esnode3.example.org] no plugins loaded | |
| [2020-08-17T12:58:04,931][INFO ][o.e.e.NodeEnvironment ] [esnode3.example.org] using [1] data paths, mounts [[/mnt/elasticsearch_iscsi (/dev/mapper/20024f40068050027)]], net usable_space [196.2gb], net total_space [199.9gb], types [xfs] | |
| [2020-08-17T12:58:04,931][INFO ][o.e.e.NodeEnvironment ] [esnode3.example.org] heap size [6gb], compressed ordinary object pointers [true] | |
| [2020-08-17T12:58:05,464][INFO ][o.e.n.Node ] [esnode3.example.org] node name [esnode3.example.org], node ID [qSe6p9lcR3mafSXjT_SI-A], cluster name [exampletestelk] | |
| [2020-08-17T12:58:07,335][ERROR][o.e.b.Bootstrap ] [esnode3.example.org] Exception | |
| java.lang.IllegalArgumentException: error loading whitelist(s) [java.lang.txt]:[288] | |
| at org.elasticsearch.painless.lookup.PainlessLookupBuilder.buildFromWhitelists(PainlessLookupBuilder.java:180) ~[?:?] | |
| at org.elasticsearch.painless.PainlessScriptEngine.<init>(PainlessScriptEngine.java:101) ~[?:?] | |
| at org.elasticsearch.painless.PainlessPlugin.getScriptEngine(PainlessPlugin.java:101) ~[?:?] | |
| at org.elasticsearch.script.ScriptModule.<init>(ScriptModule.java:84) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.node.Node.<init>(Node.java:358) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.node.Node.<init>(Node.java:266) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:227) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:227) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:393) [elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:170) [elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:161) [elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:86) [elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:127) [elasticsearch-cli-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.cli.Command.main(Command.java:90) [elasticsearch-cli-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:126) [elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:92) [?:7.8.0] | |
| Caused by: java.lang.IllegalArgumentException: reflection object not found for field [[java.lang.Character.UnicodeBlock], [CUNEIFORM_NUMBERS_AND_PUNCTUATION] | |
| at org.elasticsearch.painless.lookup.PainlessLookupBuilder.addPainlessField(PainlessLookupBuilder.java:650) ~[?:?] | |
| at org.elasticsearch.painless.lookup.PainlessLookupBuilder.addPainlessField(PainlessLookupBuilder.java:613) ~[?:?] | |
| at org.elasticsearch.painless.lookup.PainlessLookupBuilder.buildFromWhitelists(PainlessLookupBuilder.java:150) ~[?:?] | |
| ... 15 more | |
| Caused by: java.lang.NoSuchFieldException: CUNEIFORM_NUMBERS_AND_PUNCTUATION | |
| at java.lang.Class.getField(Class.java:2013) ~[?:?] | |
| at org.elasticsearch.painless.lookup.PainlessLookupBuilder.addPainlessField(PainlessLookupBuilder.java:648) ~[?:?] | |
| at org.elasticsearch.painless.lookup.PainlessLookupBuilder.addPainlessField(PainlessLookupBuilder.java:613) ~[?:?] | |
| at org.elasticsearch.painless.lookup.PainlessLookupBuilder.buildFromWhitelists(PainlessLookupBuilder.java:150) ~[?:?] | |
| ... 15 more | |
| [2020-08-17T12:58:07,344][ERROR][o.e.b.ElasticsearchUncaughtExceptionHandler] [esnode3.example.org] uncaught exception in thread [main] | |
| org.elasticsearch.bootstrap.StartupException: java.lang.IllegalArgumentException: error loading whitelist(s) [java.lang.txt]:[288] | |
| at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:174) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:161) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:86) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:127) ~[elasticsearch-cli-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.cli.Command.main(Command.java:90) ~[elasticsearch-cli-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:126) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:92) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| Caused by: java.lang.IllegalArgumentException: error loading whitelist(s) [java.lang.txt]:[288] | |
| at org.elasticsearch.painless.lookup.PainlessLookupBuilder.buildFromWhitelists(PainlessLookupBuilder.java:180) ~[?:?] | |
| at org.elasticsearch.painless.PainlessScriptEngine.<init>(PainlessScriptEngine.java:101) ~[?:?] | |
| at org.elasticsearch.painless.PainlessPlugin.getScriptEngine(PainlessPlugin.java:101) ~[?:?] | |
| at org.elasticsearch.script.ScriptModule.<init>(ScriptModule.java:84) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.node.Node.<init>(Node.java:358) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.node.Node.<init>(Node.java:266) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:227) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:227) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:393) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:170) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| ... 6 more | |
| Caused by: java.lang.IllegalArgumentException: reflection object not found for field [[java.lang.Character.UnicodeBlock], [CUNEIFORM_NUMBERS_AND_PUNCTUATION] | |
| at org.elasticsearch.painless.lookup.PainlessLookupBuilder.addPainlessField(PainlessLookupBuilder.java:650) ~[?:?] | |
| at org.elasticsearch.painless.lookup.PainlessLookupBuilder.addPainlessField(PainlessLookupBuilder.java:613) ~[?:?] | |
| at org.elasticsearch.painless.lookup.PainlessLookupBuilder.buildFromWhitelists(PainlessLookupBuilder.java:150) ~[?:?] | |
| at org.elasticsearch.painless.PainlessScriptEngine.<init>(PainlessScriptEngine.java:101) ~[?:?] | |
| at org.elasticsearch.painless.PainlessPlugin.getScriptEngine(PainlessPlugin.java:101) ~[?:?] | |
| at org.elasticsearch.script.ScriptModule.<init>(ScriptModule.java:84) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.node.Node.<init>(Node.java:358) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.node.Node.<init>(Node.java:266) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:227) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:227) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:393) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:170) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| ... 6 more | |
| Caused by: java.lang.NoSuchFieldException: CUNEIFORM_NUMBERS_AND_PUNCTUATION | |
| at java.lang.Class.getField(Class.java:2013) ~[?:?] | |
| at org.elasticsearch.painless.lookup.PainlessLookupBuilder.addPainlessField(PainlessLookupBuilder.java:648) ~[?:?] | |
| at org.elasticsearch.painless.lookup.PainlessLookupBuilder.addPainlessField(PainlessLookupBuilder.java:613) ~[?:?] | |
| at org.elasticsearch.painless.lookup.PainlessLookupBuilder.buildFromWhitelists(PainlessLookupBuilder.java:150) ~[?:?] | |
| at org.elasticsearch.painless.PainlessScriptEngine.<init>(PainlessScriptEngine.java:101) ~[?:?] | |
| at org.elasticsearch.painless.PainlessPlugin.getScriptEngine(PainlessPlugin.java:101) ~[?:?] | |
| at org.elasticsearch.script.ScriptModule.<init>(ScriptModule.java:84) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.node.Node.<init>(Node.java:358) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.node.Node.<init>(Node.java:266) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:227) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:227) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:393) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:170) ~[elasticsearch-7.8.0.jar:7.8.0] | |
| ... 6 more |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment