Created
September 29, 2023 20:48
-
-
Save dtjm/f1f9d2e52f46d961fe6091c54082500a to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ::: {mbWgjAZ}{mbWgjAZxSNaAW_l9X8fNlg}{p2C53LZTT4qTYWW1XQIunw}{x.x.x.x}{x.x.x.x:9300}{zone=us-west-2a, distributed_snapshot_deletion_enabled=true} | |
| Hot threads at 2023-09-29T20:45:22.684, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 50.5% (252.5ms out of 500ms) cpu usage by thread 'elasticsearch[mbWgjAZ][[2023092902][9]: Lucene Merge Thread #444]' | |
| 2/10 snapshots sharing following 14 elements | |
| app//org.apache.lucene.index.MultiTermsEnum.postings(MultiTermsEnum.java:361) | |
| app//org.apache.lucene.index.MappedMultiFields$MappedMultiTermsEnum.postings(MappedMultiFields.java:127) | |
| app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:122) | |
| app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:864) | |
| app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:343) | |
| app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105) | |
| app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:164) | |
| app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:231) | |
| app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:116) | |
| app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446) | |
| app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4068) | |
| app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625) | |
| app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:99) | |
| app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662) | |
| 2/10 snapshots sharing following 11 elements | |
| app//org.apache.lucene.index.FilterLeafReader$FilterTermsEnum.next(FilterLeafReader.java:189) | |
| app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:335) | |
| app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105) | |
| app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:164) | |
| app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:231) | |
| app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:116) | |
| app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446) | |
| app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4068) | |
| app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625) | |
| app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:99) | |
| app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662) | |
| 2/10 snapshots sharing following 13 elements | |
| app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:602) | |
| app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:905) | |
| app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:869) | |
| app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:343) | |
| app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105) | |
| app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:164) | |
| app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:231) | |
| app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:116) | |
| app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446) | |
| app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4068) | |
| app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625) | |
| app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:99) | |
| app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662) | |
| 3/10 snapshots sharing following 13 elements | |
| app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:624) | |
| app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:905) | |
| app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:869) | |
| app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:343) | |
| app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105) | |
| app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:164) | |
| app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:231) | |
| app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:116) | |
| app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446) | |
| app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4068) | |
| app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625) | |
| app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:99) | |
| app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662) | |
| unique snapshot | |
| app//org.apache.lucene.util.BytesRefBuilder.append(BytesRefBuilder.java:78) | |
| app//org.apache.lucene.util.BytesRefBuilder.append(BytesRefBuilder.java:87) | |
| app//org.apache.lucene.util.BytesRefBuilder.copyBytes(BytesRefBuilder.java:119) | |
| app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:919) | |
| app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:869) | |
| app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:343) | |
| app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105) | |
| app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:164) | |
| app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:231) | |
| app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:116) | |
| app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446) | |
| app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4068) | |
| app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625) | |
| app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:99) | |
| app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662) | |
| 47.9% (239.5ms out of 500ms) cpu usage by thread 'elasticsearch[mbWgjAZ][write][T#4]' | |
| 3/10 snapshots sharing following 28 elements | |
| app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:774) | |
| app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:743) | |
| app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:714) | |
| app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:423) | |
| app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:391) | |
| app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:378) | |
| app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:79) | |
| app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:568) | |
| app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:528) | |
| app//org.elasticsearch.index.shard.IndexShard$3.onResponse(IndexShard.java:2435) | |
| app//org.elasticsearch.index.shard.IndexShard$3.onResponse(IndexShard.java:2413) | |
| app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:271) | |
| app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:238) | |
| app//org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationPermit(IndexShard.java:2412) | |
| app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:638) | |
| app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| app//org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:511) | |
| app//org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:490) | |
| [AMAZON INTERNAL] | |
| com.floragunn.searchguard.ssl.transport.SearchGuardSSLRequestHandler.messageReceivedDecorate(SearchGuardSSLRequestHandler.java:170) | |
| com.floragunn.searchguard.ssl.transport.SearchGuardSSLRequestHandler.messageReceived(SearchGuardSSLRequestHandler.java:142) | |
| app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| app//org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1350) | |
| app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) | |
| app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
| [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
| [email protected]/java.lang.Thread.run(Thread.java:829) | |
| 7/10 snapshots sharing following 10 elements | |
| [email protected]/jdk.internal.misc.Unsafe.park(Native Method) | |
| [email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) | |
| [email protected]/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:743) | |
| [email protected]/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684) | |
| [email protected]/java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1366) | |
| app//org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) | |
| [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) | |
| [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
| [email protected]/java.lang.Thread.run(Thread.java:829) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment