Elasticsearch OOM Log

es13.iad

[2014-04-30 03:11:13,900][WARN ][action.deletebyquery     ] [es13.global.search.iad.wordpress.com] Failed to perform deleteByQuery/shard on replica [global-test-0-40m-50m][205]
org.elasticsearch.transport.RemoteTransportException: [es13.global.search.sat.wordpress.com][inet[/207.198.101.96:9300]][deleteByQuery/shard/replica]
Caused by: org.elasticsearch.index.engine.RefreshFailedEngineException: [global-test-0-40m-50m][205] Refresh failed
    at org.elasticsearch.index.engine.internal.InternalEngine.refresh(InternalEngine.java:736)
    at org.elasticsearch.index.engine.internal.InternalEngine.refreshVersioningTable(InternalEngine.java:906)
    at org.elasticsearch.index.engine.internal.InternalEngine.delete(InternalEngine.java:648)
    at org.elasticsearch.index.shard.service.InternalIndexShard.deleteByQuery(InternalIndexShard.java:451)
    at org.elasticsearch.action.deletebyquery.TransportShardDeleteByQueryAction.shardOperationOnReplica(TransportShardDeleteByQueryAction.java:148)
    at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$ReplicaOperationTransportHandler.messageReceived(TransportShardReplicationOperationAction.java:245)
    at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$ReplicaOperationTransportHandler.messageReceived(TransportShardReplicationOperationAction.java:225)
    at org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:270)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)
Caused by: java.io.IOException: Map failed
    at sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:849)
    at org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:283)
    at org.apache.lucene.store.MMapDirectory$MMapIndexInput.<init>(MMapDirectory.java:228)
    at org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:195)
    at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:80)
    at org.elasticsearch.index.store.Store$StoreDirectory.openInput(Store.java:473)
    at org.apache.lucene.store.Directory.copy(Directory.java:185)
    at org.elasticsearch.index.store.Store$StoreDirectory.copy(Store.java:348)
    at org.apache.lucene.store.TrackingDirectoryWrapper.copy(TrackingDirectoryWrapper.java:50)
    at org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:4671)
    at org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:535)
    at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:502)
    at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:508)
    at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:618)
    at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:370)
    at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:289)
    at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:264)
    at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:254)
    at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:170)
    at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:118)
    at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
    at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
    at org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
    at org.elasticsearch.index.engine.internal.InternalEngine.refresh(InternalEngine.java:717)
    ... 10 more
Caused by: java.lang.OutOfMemoryError: Map failed
    at sun.nio.ch.FileChannelImpl.map0(Native Method)
    at sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:846)
    ... 33 more
[2014-04-30 03:11:13,902][WARN ][cluster.action.shard     ] [es13.global.search.iad.wordpress.com] [global-test-0-40m-50m][205] sending failed shard for [global-test-0-40m-50m][205], node[Ck98fj15T1mZ2_YeLowp0g], [R], s[STARTED], indexUUID [85l1FF2ySFCzcg9lnslIYw], reason [Failed to perform [deleteByQuery/shard] on replica, message [RemoteTransportException[[es13.global.search.sat.wordpress.com][inet[/207.198.101.96:9300]][deleteByQuery/shard/replica]]; nested: RefreshFailedEngineException[[global-test-0-40m-50m][205] Refresh failed]; nested: IOException[Map failed]; nested: OutOfMemoryError[Map failed]; ]]
[2014-04-30 03:18:39,308][INFO ][cluster.service          ] [es13.global.search.iad.wordpress.com] removed {[es13.global.search.sat.wordpress.com][Ck98fj15T1mZ2_YeLowp0g][es13.global.search.sat.wordpress.com][inet[/207.198.101.96:9300]]{dc=sat, parity=1, master=false},}, reason: zen-disco-receive(from master [[esm2.global.search.dfw.wordpress.com][fKAA9MGtQQSkpHD5B4oS-A][esm2.global.search.dfw.wordpress.com][inet[/192.0.81.195:9300]]{dc=dfw, parity=0, data=false, master=true}])

es13.sat

[2014-04-30 03:07:50,254][WARN ][monitor.jvm              ] [es13.global.search.sat.wordpress.com] [gc][old][690921][528] duration [2.1m], collections [3]/[2.1m], total [2.1m]/[4.7m], memory [17.8gb]->[14gb]/[29.8gb], all_pools {[young] [744.2mb]->[46.6mb]/[1.1gb]}{[survivor] [149.7mb]->[0b]/[149.7mb]}{[old] [16.9gb]->[14gb]/[28.5gb]}
[2014-04-30 03:11:14,113][WARN ][monitor.jvm              ] [es13.global.search.sat.wordpress.com] [gc][old][690922][533] duration [3.3m], collections [5]/[3.3m], total [3.3m]/[8.1m], memory [14gb]->[14.3gb]/[29.8gb], all_pools {[young] [46.6mb]->[233.4mb]/[1.1gb]}{[survivor] [0b]->[149.7mb]/[149.7mb]}{[old] [14gb]->[13.9gb]/[28.5gb]}
[2014-04-30 03:11:55,994][WARN ][monitor.jvm              ] [es13.global.search.sat.wordpress.com] [gc][old][690923][534] duration [41.8s], collections [1]/[41.8s], total [41.8s]/[8.8m], memory [14.3gb]->[14gb]/[29.8gb], all_pools {[young] [233.4mb]->[243.2kb]/[1.1gb]}{[survivor] [149.7mb]->[0b]/[149.7mb]}{[old] [13.9gb]->[14gb]/[28.5gb]}
[2014-04-30 03:12:37,752][WARN ][monitor.jvm              ] [es13.global.search.sat.wordpress.com] [gc][old][690924][535] duration [41.6s], collections [1]/[41.7s], total [41.6s]/[9.5m], memory [14gb]->[14.1gb]/[29.8gb], all_pools {[young] [243.2kb]->[1.7mb]/[1.1gb]}{[survivor] [0b]->[101.8mb]/[149.7mb]}{[old] [14gb]->[14gb]/[28.5gb]}
[2014-04-30 03:13:19,423][WARN ][monitor.jvm              ] [es13.global.search.sat.wordpress.com] [gc][old][690925][536] duration [41.6s], collections [1]/[41.6s], total [41.6s]/[10.2m], memory [14.1gb]->[14.1gb]/[29.8gb], all_pools {[young] [1.7mb]->[145mb]/[1.1gb]}{[survivor] [101.8mb]->[0b]/[149.7mb]}{[old] [14gb]->[14gb]/[28.5gb]}
[2014-04-30 03:14:42,539][DEBUG][action.deletebyquery     ] [es13.global.search.sat.wordpress.com] [global-test-0-40m-50m][10], node[Ck98fj15T1mZ2_YeLowp0g], [P], s[STARTED]: Failed to execute [delete_by_query {[global-test-0-40m-50m][blog], query [{"query":{"filtered":{"query":{"match_all":[]},"filter":{"and":[{"term":{"blog_id":40187780}},{"bool":{"must":[{"term":{"blog_id":40187780}}],"must_not":[{"type":{"value":"blog"}}]}}]}}}}]}]
org.elasticsearch.index.engine.RefreshFailedEngineException: [global-test-0-40m-50m][10] Refresh failed
    at org.elasticsearch.index.engine.internal.InternalEngine.refresh(InternalEngine.java:736)
    at org.elasticsearch.index.engine.internal.InternalEngine.refreshVersioningTable(InternalEngine.java:906)
    at org.elasticsearch.index.engine.internal.InternalEngine.delete(InternalEngine.java:648)
    at org.elasticsearch.index.shard.service.InternalIndexShard.deleteByQuery(InternalIndexShard.java:451)
    at org.elasticsearch.action.deletebyquery.TransportShardDeleteByQueryAction.shardOperationOnPrimary(TransportShardDeleteByQueryAction.java:125)
    at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:556)
    at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$1.run(TransportShardReplicationOperationAction.java:426)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)
Caused by: java.io.IOException: Map failed
    at sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:849)
    at org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:283)
    at org.apache.lucene.store.MMapDirectory$MMapIndexInput.<init>(MMapDirectory.java:228)
    at org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:195)
    at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:80)
    at org.elasticsearch.index.store.Store$StoreDirectory.openInput(Store.java:473)
    at org.apache.lucene.store.Directory.copy(Directory.java:185)
    at org.elasticsearch.index.store.Store$StoreDirectory.copy(Store.java:348)
    at org.apache.lucene.store.TrackingDirectoryWrapper.copy(TrackingDirectoryWrapper.java:50)
    at org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:4671)
    at org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:535)
    at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:502)
    at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:508)
    at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:618)
    at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:370)
    at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:289)
    at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:264)
    at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:254)
    at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:170)
    at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:118)
    at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
    at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
    at org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
    at org.elasticsearch.index.engine.internal.InternalEngine.refresh(InternalEngine.java:717)
    ... 9 more
Caused by: java.lang.OutOfMemoryError: Map failed
    at sun.nio.ch.FileChannelImpl.map0(Native Method)
    at sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:846)
    ... 32 more

These are the logs for the issue opened below:

elastic/elasticsearch#6025