Created
July 7, 2022 17:46
-
-
Save ycyr/80a9a4179dbadcc708b1f96863677db6 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ::: {opensearch-node2}{NWq2Cpn5TBCXO1Lo4GNV6w}{NOUocwuXTX6MAwECgMFnAQ}{172.18.0.9}{172.18.0.9:9300}{dimr}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-07-07T17:41:31.918Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 100.2% (501.1ms out of 500ms) cpu usage by thread 'opensearch[opensearch-node2][flush][T#1]' | |
| 2/10 snapshots sharing following 23 elements | |
| app//org.apache.lucene.util.MSBRadixSorter.reorder(MSBRadixSorter.java:292) | |
| app//org.apache.lucene.util.MSBRadixSorter.radixSort(MSBRadixSorter.java:172) | |
| app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:136) | |
| app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:129) | |
| app//org.apache.lucene.util.BytesRefHash.sort(BytesRefHash.java:168) | |
| app//org.apache.lucene.index.TermsHashPerField.sortTerms(TermsHashPerField.java:112) | |
| app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:95) | |
| app//org.apache.lucene.index.IndexingChain.flush(IndexingChain.java:300) | |
| app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:391) | |
| app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:493) | |
| app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:672) | |
| app//org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3428) | |
| app//org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3863) | |
| app//org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3825) | |
| app//org.opensearch.index.engine.InternalEngine.commitIndexWriter(InternalEngine.java:2637) | |
| app//org.opensearch.index.engine.InternalEngine.flush(InternalEngine.java:1970) | |
| app//org.opensearch.index.shard.IndexShard.flush(IndexShard.java:1260) | |
| app//org.opensearch.index.shard.IndexShard$3.doRun(IndexShard.java:2100) | |
| app//org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:798) | |
| app//org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52) | |
| [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) | |
| [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) | |
| [email protected]/java.lang.Thread.run(Thread.java:833) | |
| 3/10 snapshots sharing following 26 elements | |
| app//org.apache.lucene.util.MSBRadixSorter.buildHistogram(MSBRadixSorter.java:262) | |
| app//org.apache.lucene.util.MSBRadixSorter.computeCommonPrefixLengthAndBuildHistogram(MSBRadixSorter.java:248) | |
| app//org.apache.lucene.util.MSBRadixSorter.radixSort(MSBRadixSorter.java:157) | |
| app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:136) | |
| app//org.apache.lucene.util.MSBRadixSorter.radixSort(MSBRadixSorter.java:182) | |
| app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:136) | |
| app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:129) | |
| app//org.apache.lucene.util.BytesRefHash.sort(BytesRefHash.java:168) | |
| app//org.apache.lucene.index.TermsHashPerField.sortTerms(TermsHashPerField.java:112) | |
| app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:95) | |
| app//org.apache.lucene.index.IndexingChain.flush(IndexingChain.java:300) | |
| app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:391) | |
| app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:493) | |
| app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:672) | |
| app//org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3428) | |
| app//org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3863) | |
| app//org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3825) | |
| app//org.opensearch.index.engine.InternalEngine.commitIndexWriter(InternalEngine.java:2637) | |
| app//org.opensearch.index.engine.InternalEngine.flush(InternalEngine.java:1970) | |
| app//org.opensearch.index.shard.IndexShard.flush(IndexShard.java:1260) | |
| app//org.opensearch.index.shard.IndexShard$3.doRun(IndexShard.java:2100) | |
| app//org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:798) | |
| app//org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52) | |
| [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) | |
| [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) | |
| [email protected]/java.lang.Thread.run(Thread.java:833) | |
| 4/10 snapshots sharing following 25 elements | |
| app//org.apache.lucene.util.MSBRadixSorter.reorder(MSBRadixSorter.java:292) | |
| app//org.apache.lucene.util.MSBRadixSorter.radixSort(MSBRadixSorter.java:172) | |
| app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:136) | |
| app//org.apache.lucene.util.MSBRadixSorter.radixSort(MSBRadixSorter.java:182) | |
| app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:136) | |
| app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:129) | |
| app//org.apache.lucene.util.BytesRefHash.sort(BytesRefHash.java:168) | |
| app//org.apache.lucene.index.TermsHashPerField.sortTerms(TermsHashPerField.java:112) | |
| app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:95) | |
| app//org.apache.lucene.index.IndexingChain.flush(IndexingChain.java:300) | |
| app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:391) | |
| app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:493) | |
| app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:672) | |
| app//org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3428) | |
| app//org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3863) | |
| app//org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3825) | |
| app//org.opensearch.index.engine.InternalEngine.commitIndexWriter(InternalEngine.java:2637) | |
| app//org.opensearch.index.engine.InternalEngine.flush(InternalEngine.java:1970) | |
| app//org.opensearch.index.shard.IndexShard.flush(IndexShard.java:1260) | |
| app//org.opensearch.index.shard.IndexShard$3.doRun(IndexShard.java:2100) | |
| app//org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:798) | |
| app//org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52) | |
| [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) | |
| [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) | |
| [email protected]/java.lang.Thread.run(Thread.java:833) | |
| unique snapshot | |
| app//org.apache.lucene.util.MSBRadixSorter.reorder(MSBRadixSorter.java:292) | |
| app//org.apache.lucene.util.MSBRadixSorter.radixSort(MSBRadixSorter.java:172) | |
| app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:136) | |
| app//org.apache.lucene.util.MSBRadixSorter.radixSort(MSBRadixSorter.java:182) | |
| app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:136) | |
| app//org.apache.lucene.util.MSBRadixSorter.radixSort(MSBRadixSorter.java:182) | |
| app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:136) | |
| app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:129) | |
| app//org.apache.lucene.util.BytesRefHash.sort(BytesRefHash.java:168) | |
| app//org.apache.lucene.index.TermsHashPerField.sortTerms(TermsHashPerField.java:112) | |
| app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:95) | |
| app//org.apache.lucene.index.IndexingChain.flush(IndexingChain.java:300) | |
| app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:391) | |
| app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:493) | |
| app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:672) | |
| app//org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3428) | |
| app//org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3863) | |
| app//org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3825) | |
| app//org.opensearch.index.engine.InternalEngine.commitIndexWriter(InternalEngine.java:2637) | |
| app//org.opensearch.index.engine.InternalEngine.flush(InternalEngine.java:1970) | |
| app//org.opensearch.index.shard.IndexShard.flush(IndexShard.java:1260) | |
| app//org.opensearch.index.shard.IndexShard$3.doRun(IndexShard.java:2100) | |
| app//org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:798) | |
| app//org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52) | |
| [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) | |
| [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) | |
| [email protected]/java.lang.Thread.run(Thread.java:833) | |
| ::: {opensearch-node1}{Sa-IjmPeQcynu2ra8_DQqg}{574_SXKeRki69hovxeNVDg}{172.18.0.3}{172.18.0.3:9300}{dimr}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-07-07T17:41:31.918Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 92.9% (464.5ms out of 500ms) cpu usage by thread 'opensearch[opensearch-node1][snapshot][T#3]' | |
| 10/10 snapshots sharing following 18 elements | |
| [email protected]/javax.crypto.CipherInputStream.ensureCapacity(CipherInputStream.java:113) | |
| [email protected]/javax.crypto.CipherInputStream.getMoreData(CipherInputStream.java:156) | |
| [email protected]/javax.crypto.CipherInputStream.read(CipherInputStream.java:261) | |
| app//org.opensearch.index.snapshots.blobstore.SlicedInputStream.read(SlicedInputStream.java:107) | |
| [email protected]/java.io.FilterInputStream.read(FilterInputStream.java:132) | |
| app//org.opensearch.index.snapshots.blobstore.RateLimitingInputStream.read(RateLimitingInputStream.java:93) | |
| [email protected]/java.io.FilterInputStream.read(FilterInputStream.java:132) | |
| app//org.opensearch.index.snapshots.blobstore.RateLimitingInputStream.read(RateLimitingInputStream.java:93) | |
| [email protected]/java.io.FilterInputStream.read(FilterInputStream.java:106) | |
| app//org.opensearch.repositories.blobstore.BlobStoreRepository$9.restoreFile(BlobStoreRepository.java:2672) | |
| app//org.opensearch.repositories.blobstore.BlobStoreRepository$9.lambda$executeOneFileRestore$1(BlobStoreRepository.java:2638) | |
| app//org.opensearch.repositories.blobstore.BlobStoreRepository$9$$Lambda$4992/0x0000000801c2f278.accept(Unknown Source) | |
| app//org.opensearch.action.ActionRunnable$2.doRun(ActionRunnable.java:88) | |
| app//org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:798) | |
| app//org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52) | |
| [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) | |
| [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) | |
| [email protected]/java.lang.Thread.run(Thread.java:833) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment