Created
December 23, 2013 16:22
-
-
Save jpountz/8099938 to your computer and use it in GitHub Desktop.
Patch that gives more detailed ram usage.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Index: lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java | |
| =================================================================== | |
| --- lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java (revision 1553093) | |
| +++ lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java (working copy) | |
| @@ -18,8 +18,10 @@ | |
| */ | |
| import java.io.IOException; | |
| +import java.util.ArrayList; | |
| import java.util.Collections; | |
| import java.util.Iterator; | |
| +import java.util.List; | |
| import java.util.TreeMap; | |
| import org.apache.lucene.codecs.BlockTermState; | |
| @@ -45,6 +47,7 @@ | |
| import org.apache.lucene.util.Bits; | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.DoubleBarrelLRUCache; | |
| +import org.apache.lucene.util.RamUsage; | |
| /** Handles a terms dict, but decouples all details of | |
| * doc/freqs/positions reading to an instance of {@link | |
| @@ -858,9 +861,7 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - long sizeInBytes = (postingsReader!=null) ? postingsReader.ramBytesUsed() : 0; | |
| - sizeInBytes += (indexReader!=null) ? indexReader.ramBytesUsed() : 0; | |
| - return sizeInBytes; | |
| + public RamUsage ramBytesUsed() { | |
| + return new RamUsage(getClass().getName(), postingsReader.ramBytesUsed(), indexReader.ramBytesUsed()); | |
| } | |
| } | |
| Index: lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java | |
| =================================================================== | |
| --- lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java (revision 1553093) | |
| +++ lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java (working copy) | |
| @@ -27,10 +27,14 @@ | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.IOUtils; | |
| import org.apache.lucene.util.PagedBytes; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.packed.MonotonicBlockPackedReader; | |
| +import java.util.ArrayList; | |
| import java.util.HashMap; | |
| import java.util.Comparator; | |
| +import java.util.List; | |
| +import java.util.Map; | |
| import java.io.IOException; | |
| import org.apache.lucene.index.IndexFileNames; | |
| @@ -59,7 +63,7 @@ | |
| // all fields share this single logical byte[] | |
| private final PagedBytes termBytes = new PagedBytes(PAGED_BYTES_BITS); | |
| - private PagedBytes.Reader termBytesReader; | |
| + private final PagedBytes.Reader termBytesReader; | |
| final HashMap<FieldInfo,FieldIndexData> fields = new HashMap<FieldInfo,FieldIndexData>(); | |
| @@ -279,13 +283,13 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - long sizeInBytes = ((termBytes!=null) ? termBytes.ramBytesUsed() : 0) + | |
| - ((termBytesReader!=null)? termBytesReader.ramBytesUsed() : 0); | |
| - | |
| - for(FieldIndexData entry : fields.values()) { | |
| - sizeInBytes += entry.ramBytesUsed(); | |
| + public RamUsage ramBytesUsed() { | |
| + List<RamUsage> details = new ArrayList<RamUsage>(); | |
| + details.add(new RamUsage(termBytes.ramBytesUsed(), "term bytes")); | |
| + details.add(new RamUsage(termBytesReader.ramBytesUsed(), "term bytes reader")); | |
| + for (Map.Entry<FieldInfo, FieldIndexData> entry : fields.entrySet()) { | |
| + details.add(new RamUsage(entry.getValue().ramBytesUsed(), entry.getKey().name)); | |
| } | |
| - return sizeInBytes; | |
| + return new RamUsage(getClass().getName(), details); | |
| } | |
| } | |
| Index: lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/TermsIndexReaderBase.java | |
| =================================================================== | |
| --- lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/TermsIndexReaderBase.java (revision 1553093) | |
| +++ lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/TermsIndexReaderBase.java (working copy) | |
| @@ -19,6 +19,7 @@ | |
| import org.apache.lucene.index.FieldInfo; | |
| import org.apache.lucene.util.BytesRef; | |
| +import org.apache.lucene.util.RamUsage; | |
| import java.io.IOException; | |
| import java.io.Closeable; | |
| @@ -72,5 +73,5 @@ | |
| } | |
| /** Returns approximate RAM bytes used */ | |
| - public abstract long ramBytesUsed(); | |
| + public abstract RamUsage ramBytesUsed(); | |
| } | |
| Index: lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java | |
| =================================================================== | |
| --- lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java (revision 1553093) | |
| +++ lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java (working copy) | |
| @@ -21,7 +21,10 @@ | |
| import java.io.FileOutputStream; // for toDot | |
| import java.io.OutputStreamWriter; // for toDot | |
| import java.io.Writer; // for toDot | |
| +import java.util.ArrayList; | |
| import java.util.HashMap; | |
| +import java.util.List; | |
| +import java.util.Map; | |
| import org.apache.lucene.codecs.CodecUtil; | |
| import org.apache.lucene.index.CorruptIndexException; | |
| @@ -33,6 +36,7 @@ | |
| import org.apache.lucene.store.IndexInput; | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.fst.BytesRefFSTEnum; | |
| import org.apache.lucene.util.fst.FST; | |
| import org.apache.lucene.util.fst.PositiveIntOutputs; | |
| @@ -198,11 +202,11 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - long sizeInBytes = 0; | |
| - for(FieldIndexData entry : fields.values()) { | |
| - sizeInBytes += entry.ramBytesUsed(); | |
| + public RamUsage ramBytesUsed() { | |
| + List<RamUsage> details = new ArrayList<RamUsage>(); | |
| + for (Map.Entry<FieldInfo, FieldIndexData> entry : fields.entrySet()) { | |
| + details.add(new RamUsage(entry.getValue().ramBytesUsed(), entry.getKey().name)); | |
| } | |
| - return sizeInBytes; | |
| + return new RamUsage(getClass().getName(), details); | |
| } | |
| } | |
| Index: lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java | |
| =================================================================== | |
| --- lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (revision 1553093) | |
| +++ lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (working copy) | |
| @@ -45,6 +45,7 @@ | |
| import org.apache.lucene.util.Bits; | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.RamUsageEstimator; | |
| import org.apache.lucene.util.automaton.CompiledAutomaton; | |
| @@ -382,13 +383,15 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - long sizeInBytes = ((delegateFieldsProducer!=null) ? delegateFieldsProducer.ramBytesUsed() : 0); | |
| - for(Map.Entry<String,FuzzySet> entry: bloomsByFieldName.entrySet()) { | |
| - sizeInBytes += entry.getKey().length() * RamUsageEstimator.NUM_BYTES_CHAR; | |
| - sizeInBytes += entry.getValue().ramBytesUsed(); | |
| + public RamUsage ramBytesUsed() { | |
| + List<RamUsage> details = new ArrayList<RamUsage>(); | |
| + if (delegateFieldsProducer != null) { | |
| + details.add(delegateFieldsProducer.ramBytesUsed()); | |
| } | |
| - return sizeInBytes; | |
| + for (Map.Entry<String,FuzzySet> entry : bloomsByFieldName.entrySet()) { | |
| + details.add(new RamUsage(entry.getValue().ramBytesUsed(), "bloom filter for " + entry.getKey())); | |
| + } | |
| + return new RamUsage(getClass().getName(), details); | |
| } | |
| } | |
| Index: lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java | |
| =================================================================== | |
| --- lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java (revision 1553093) | |
| +++ lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java (working copy) | |
| @@ -18,7 +18,9 @@ | |
| */ | |
| import java.io.IOException; | |
| +import java.util.ArrayList; | |
| import java.util.HashMap; | |
| +import java.util.List; | |
| import java.util.Map; | |
| import java.util.concurrent.atomic.AtomicLong; | |
| @@ -37,6 +39,7 @@ | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.FixedBitSet; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.RamUsageEstimator; | |
| /** | |
| @@ -63,7 +66,7 @@ | |
| private final Map<Integer,Bits> docsWithFieldInstances = new HashMap<Integer,Bits>(); | |
| private final int maxDoc; | |
| - private final AtomicLong ramBytesUsed; | |
| + private final List<RamUsage> ramUsage; | |
| static final byte NUMBER = 0; | |
| static final byte BYTES = 1; | |
| @@ -78,7 +81,7 @@ | |
| String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension); | |
| // read in the entries from the metadata file. | |
| IndexInput in = state.directory.openInput(metaName, state.context); | |
| - ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass())); | |
| + ramUsage = new ArrayList<RamUsage>(); | |
| boolean success = false; | |
| final int version; | |
| try { | |
| @@ -180,8 +183,8 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - return ramBytesUsed.get(); | |
| + public synchronized RamUsage ramBytesUsed() { | |
| + return new RamUsage(getClass().getName(), new ArrayList<RamUsage>(ramUsage)); | |
| } | |
| @Override | |
| @@ -189,20 +192,20 @@ | |
| NumericDocValues instance = numericInstances.get(field.number); | |
| if (instance == null) { | |
| // Lazy load | |
| - instance = loadNumeric(numerics.get(field.number)); | |
| + instance = loadNumeric(numerics.get(field.number), field); | |
| numericInstances.put(field.number, instance); | |
| } | |
| return instance; | |
| } | |
| - private NumericDocValues loadNumeric(NumericEntry entry) throws IOException { | |
| + private NumericDocValues loadNumeric(NumericEntry entry, FieldInfo field) throws IOException { | |
| data.seek(entry.offset + entry.missingBytes); | |
| switch (entry.byteWidth) { | |
| case 1: | |
| { | |
| final byte[] values = new byte[entry.count]; | |
| data.readBytes(values, 0, entry.count); | |
| - ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(values)); | |
| + ramUsage.add(new RamUsage(RamUsageEstimator.sizeOf(values), field.name)); | |
| return new NumericDocValues() { | |
| @Override | |
| public long get(int idx) { | |
| @@ -217,7 +220,7 @@ | |
| for(int i=0;i<entry.count;i++) { | |
| values[i] = data.readShort(); | |
| } | |
| - ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(values)); | |
| + ramUsage.add(new RamUsage(RamUsageEstimator.sizeOf(values), field.name)); | |
| return new NumericDocValues() { | |
| @Override | |
| public long get(int idx) { | |
| @@ -232,7 +235,7 @@ | |
| for(int i=0;i<entry.count;i++) { | |
| values[i] = data.readInt(); | |
| } | |
| - ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(values)); | |
| + ramUsage.add(new RamUsage(RamUsageEstimator.sizeOf(values), field.name)); | |
| return new NumericDocValues() { | |
| @Override | |
| public long get(int idx) { | |
| @@ -247,7 +250,7 @@ | |
| for(int i=0;i<entry.count;i++) { | |
| values[i] = data.readLong(); | |
| } | |
| - ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(values)); | |
| + ramUsage.add(new RamUsage(RamUsageEstimator.sizeOf(values), field.name)); | |
| return new NumericDocValues() { | |
| @Override | |
| public long get(int idx) { | |
| @@ -266,13 +269,13 @@ | |
| BinaryDocValues instance = binaryInstances.get(field.number); | |
| if (instance == null) { | |
| // Lazy load | |
| - instance = loadBinary(binaries.get(field.number)); | |
| + instance = loadBinary(binaries.get(field.number), field); | |
| binaryInstances.put(field.number, instance); | |
| } | |
| return instance; | |
| } | |
| - private BinaryDocValues loadBinary(BinaryEntry entry) throws IOException { | |
| + private BinaryDocValues loadBinary(BinaryEntry entry, FieldInfo field) throws IOException { | |
| data.seek(entry.offset); | |
| final byte[] bytes = new byte[entry.numBytes]; | |
| data.readBytes(bytes, 0, entry.numBytes); | |
| @@ -284,7 +287,7 @@ | |
| } | |
| address[entry.count] = data.readInt(); | |
| - ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(bytes) + RamUsageEstimator.sizeOf(address)); | |
| + ramUsage.add(new RamUsage(field.name, new RamUsage(RamUsageEstimator.sizeOf(bytes), "bytes"), new RamUsage(RamUsageEstimator.sizeOf(address), "addresses"))); | |
| return new BinaryDocValues() { | |
| @Override | |
| @@ -309,8 +312,8 @@ | |
| private SortedDocValues loadSorted(FieldInfo field) throws IOException { | |
| final SortedEntry entry = sorteds.get(field.number); | |
| - final NumericDocValues docToOrd = loadNumeric(entry.docToOrd); | |
| - final BinaryDocValues values = loadBinary(entry.values); | |
| + final NumericDocValues docToOrd = loadNumeric(entry.docToOrd, field); | |
| + final BinaryDocValues values = loadBinary(entry.values, field); | |
| return new SortedDocValues() { | |
| @@ -341,7 +344,7 @@ | |
| final SortedSetEntry entry = sortedSets.get(field.number); | |
| if (instance == null) { | |
| // Lazy load | |
| - instance = loadSortedSet(entry); | |
| + instance = loadSortedSet(entry, field); | |
| sortedSetInstances.put(field.number, instance); | |
| } | |
| @@ -385,11 +388,11 @@ | |
| }; | |
| } | |
| - private SortedSetRawValues loadSortedSet(SortedSetEntry entry) throws IOException { | |
| + private SortedSetRawValues loadSortedSet(SortedSetEntry entry, FieldInfo field) throws IOException { | |
| SortedSetRawValues instance = new SortedSetRawValues(); | |
| - instance.docToOrdAddress = loadNumeric(entry.docToOrdAddress); | |
| - instance.ords = loadNumeric(entry.ords); | |
| - instance.values = loadBinary(entry.values); | |
| + instance.docToOrdAddress = loadNumeric(entry.docToOrdAddress, field); | |
| + instance.ords = loadNumeric(entry.ords, field); | |
| + instance.values = loadBinary(entry.values, field); | |
| return instance; | |
| } | |
| Index: lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java | |
| =================================================================== | |
| --- lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (revision 1553093) | |
| +++ lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (working copy) | |
| @@ -18,8 +18,10 @@ | |
| */ | |
| import java.io.IOException; | |
| +import java.util.ArrayList; | |
| import java.util.Collections; | |
| import java.util.Iterator; | |
| +import java.util.List; | |
| import java.util.Map; | |
| import java.util.TreeMap; | |
| @@ -43,6 +45,7 @@ | |
| import org.apache.lucene.util.ArrayUtil; | |
| import org.apache.lucene.util.Bits; | |
| import org.apache.lucene.util.BytesRef; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.RamUsageEstimator; | |
| import org.apache.lucene.util.automaton.CompiledAutomaton; | |
| import org.apache.lucene.util.automaton.RunAutomaton; | |
| @@ -149,13 +152,12 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - long sizeInBytes = 0; | |
| - for(Map.Entry<String,DirectField> entry: fields.entrySet()) { | |
| - sizeInBytes += entry.getKey().length() * RamUsageEstimator.NUM_BYTES_CHAR; | |
| - sizeInBytes += entry.getValue().ramBytesUsed(); | |
| + public RamUsage ramBytesUsed() { | |
| + List<RamUsage> details = new ArrayList<RamUsage>(); | |
| + for (Map.Entry<String,DirectField> entry : fields.entrySet()) { | |
| + details.add(new RamUsage(entry.getValue().ramBytesUsed(), entry.getKey())); | |
| } | |
| - return sizeInBytes; | |
| + return new RamUsage(getClass().getName(), details); | |
| } | |
| } | |
| Index: lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java | |
| =================================================================== | |
| --- lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java (revision 1553093) | |
| +++ lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java (working copy) | |
| @@ -23,6 +23,8 @@ | |
| import java.util.BitSet; | |
| import java.util.Collections; | |
| import java.util.Iterator; | |
| +import java.util.List; | |
| +import java.util.Map; | |
| import java.util.TreeMap; | |
| import org.apache.lucene.index.CorruptIndexException; | |
| @@ -45,6 +47,7 @@ | |
| import org.apache.lucene.util.Bits; | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.RamUsageEstimator; | |
| import org.apache.lucene.util.fst.BytesRefFSTEnum; | |
| import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput; | |
| @@ -806,17 +809,20 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - long ramBytesUsed = 0; | |
| - for (TermsReader r : fields.values()) { | |
| + public RamUsage ramBytesUsed() { | |
| + List<RamUsage> details = new ArrayList<RamUsage>(); | |
| + for (Map.Entry<String,TermsReader> entry : fields.entrySet()) { | |
| + final TermsReader r = entry.getValue(); | |
| if (r.index != null) { | |
| + long ramBytesUsed = 0; | |
| ramBytesUsed += r.index.sizeInBytes(); | |
| ramBytesUsed += RamUsageEstimator.sizeOf(r.metaBytesBlock); | |
| ramBytesUsed += RamUsageEstimator.sizeOf(r.metaLongsBlock); | |
| ramBytesUsed += RamUsageEstimator.sizeOf(r.skipInfo); | |
| ramBytesUsed += RamUsageEstimator.sizeOf(r.statsBlock); | |
| + details.add(new RamUsage(ramBytesUsed, entry.getKey())); | |
| } | |
| } | |
| - return ramBytesUsed; | |
| + return new RamUsage(getClass().getName(), details); | |
| } | |
| } | |
| Index: lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java | |
| =================================================================== | |
| --- lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java (revision 1553093) | |
| +++ lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java (working copy) | |
| @@ -22,6 +22,8 @@ | |
| import java.util.BitSet; | |
| import java.util.Collections; | |
| import java.util.Iterator; | |
| +import java.util.List; | |
| +import java.util.Map; | |
| import java.util.TreeMap; | |
| import org.apache.lucene.index.CorruptIndexException; | |
| @@ -44,6 +46,7 @@ | |
| import org.apache.lucene.util.Bits; | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.RamUsageEstimator; | |
| import org.apache.lucene.util.fst.BytesRefFSTEnum; | |
| import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput; | |
| @@ -722,11 +725,12 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - long ramBytesUsed = 0; | |
| - for (TermsReader r : fields.values()) { | |
| - ramBytesUsed += r.dict == null ? 0 : r.dict.sizeInBytes(); | |
| + public RamUsage ramBytesUsed() { | |
| + List<RamUsage> details = new ArrayList<RamUsage>(); | |
| + for (Map.Entry<String,TermsReader> entry : fields.entrySet()) { | |
| + final TermsReader r = entry.getValue(); | |
| + details.add(new RamUsage(r.dict.sizeInBytes(), entry.getKey())); | |
| } | |
| - return ramBytesUsed; | |
| + return new RamUsage(getClass().getName(), details); | |
| } | |
| } | |
| Index: lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java | |
| =================================================================== | |
| --- lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java (revision 1553093) | |
| +++ lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java (working copy) | |
| @@ -18,7 +18,9 @@ | |
| */ | |
| import java.io.IOException; | |
| +import java.util.ArrayList; | |
| import java.util.HashMap; | |
| +import java.util.List; | |
| import java.util.Map; | |
| import java.util.concurrent.atomic.AtomicLong; | |
| @@ -44,6 +46,7 @@ | |
| import org.apache.lucene.util.IOUtils; | |
| import org.apache.lucene.util.IntsRef; | |
| import org.apache.lucene.util.PagedBytes; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.RamUsageEstimator; | |
| import org.apache.lucene.util.fst.BytesRefFSTEnum; | |
| import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput; | |
| @@ -76,7 +79,7 @@ | |
| private final Map<Integer,Bits> docsWithFieldInstances = new HashMap<Integer,Bits>(); | |
| private final int maxDoc; | |
| - private final AtomicLong ramBytesUsed; | |
| + private final List<RamUsage> ramUsage; | |
| static final byte NUMBER = 0; | |
| static final byte BYTES = 1; | |
| @@ -108,7 +111,7 @@ | |
| binaries = new HashMap<Integer,BinaryEntry>(); | |
| fsts = new HashMap<Integer,FSTEntry>(); | |
| readFields(in, state.fieldInfos); | |
| - ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass())); | |
| + ramUsage = new ArrayList<RamUsage>(); | |
| success = true; | |
| } finally { | |
| if (success) { | |
| @@ -204,8 +207,8 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - return ramBytesUsed.get(); | |
| + public synchronized RamUsage ramBytesUsed() { | |
| + return new RamUsage(getClass().getName(), new ArrayList<RamUsage>(ramUsage)); | |
| } | |
| private NumericDocValues loadNumeric(FieldInfo field) throws IOException { | |
| @@ -224,7 +227,9 @@ | |
| final int formatID = data.readVInt(); | |
| final int bitsPerValue = data.readVInt(); | |
| final PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), entry.packedIntsVersion, maxDoc, bitsPerValue); | |
| - ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed()); | |
| + ramUsage.add(new RamUsage(field.name, | |
| + new RamUsage(RamUsageEstimator.sizeOf(decode), "table"), | |
| + new RamUsage(ordsReader.ramBytesUsed(), "ords "))); | |
| return new NumericDocValues() { | |
| @Override | |
| public long get(int docID) { | |
| @@ -234,12 +239,14 @@ | |
| case DELTA_COMPRESSED: | |
| final int blockSize = data.readVInt(); | |
| final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false); | |
| - ramBytesUsed.addAndGet(reader.ramBytesUsed()); | |
| + ramUsage.add(new RamUsage(field.name, | |
| + new RamUsage(reader.ramBytesUsed(), "values (delta-compressed)"))); | |
| return reader; | |
| case UNCOMPRESSED: | |
| final byte bytes[] = new byte[maxDoc]; | |
| data.readBytes(bytes, 0, bytes.length); | |
| - ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(bytes)); | |
| + ramUsage.add(new RamUsage(field.name, | |
| + new RamUsage(RamUsageEstimator.sizeOf(bytes), "values (uncompressed)"))); | |
| return new NumericDocValues() { | |
| @Override | |
| public long get(int docID) { | |
| @@ -251,7 +258,8 @@ | |
| final long mult = data.readLong(); | |
| final int quotientBlockSize = data.readVInt(); | |
| final BlockPackedReader quotientReader = new BlockPackedReader(data, entry.packedIntsVersion, quotientBlockSize, maxDoc, false); | |
| - ramBytesUsed.addAndGet(quotientReader.ramBytesUsed()); | |
| + ramUsage.add(new RamUsage(field.name, | |
| + new RamUsage(quotientReader.ramBytesUsed(), "quotients (gcd compression)"))); | |
| return new NumericDocValues() { | |
| @Override | |
| public long get(int docID) { | |
| @@ -281,7 +289,9 @@ | |
| final PagedBytes.Reader bytesReader = bytes.freeze(true); | |
| if (entry.minLength == entry.maxLength) { | |
| final int fixedLength = entry.minLength; | |
| - ramBytesUsed.addAndGet(bytes.ramBytesUsed()); | |
| + ramUsage.add(new RamUsage(field.name, | |
| + new RamUsage(bytes.ramBytesUsed(), "term bytes"), | |
| + new RamUsage(bytesReader.ramBytesUsed(), "term bytes reader"))); | |
| return new BinaryDocValues() { | |
| @Override | |
| public void get(int docID, BytesRef result) { | |
| @@ -291,7 +301,10 @@ | |
| } else { | |
| data.seek(data.getFilePointer() + entry.missingBytes); | |
| final MonotonicBlockPackedReader addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false); | |
| - ramBytesUsed.addAndGet(bytes.ramBytesUsed() + addresses.ramBytesUsed()); | |
| + ramUsage.add(new RamUsage(field.name, | |
| + new RamUsage(bytes.ramBytesUsed(), "term bytes"), | |
| + new RamUsage(bytesReader.ramBytesUsed(), "term bytes reader"), | |
| + new RamUsage(addresses.ramBytesUsed(), "addresses"))); | |
| return new BinaryDocValues() { | |
| @Override | |
| public void get(int docID, BytesRef result) { | |
| @@ -315,7 +328,7 @@ | |
| if (instance == null) { | |
| data.seek(entry.offset); | |
| instance = new FST<Long>(data, PositiveIntOutputs.getSingleton()); | |
| - ramBytesUsed.addAndGet(instance.sizeInBytes()); | |
| + ramUsage.add(new RamUsage(field.name, new RamUsage(instance.sizeInBytes(), "terms index (fst)"))); | |
| fstInstances.put(field.number, instance); | |
| } | |
| } | |
| @@ -390,7 +403,7 @@ | |
| if (instance == null) { | |
| data.seek(entry.offset); | |
| instance = new FST<Long>(data, PositiveIntOutputs.getSingleton()); | |
| - ramBytesUsed.addAndGet(instance.sizeInBytes()); | |
| + ramUsage.add(new RamUsage(field.name, new RamUsage(instance.sizeInBytes(), "terms index (fst)"))); | |
| fstInstances.put(field.number, instance); | |
| } | |
| } | |
| Index: lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java | |
| =================================================================== | |
| --- lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (revision 1553093) | |
| +++ lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (working copy) | |
| @@ -19,8 +19,10 @@ | |
| import java.io.Closeable; | |
| import java.io.IOException; | |
| +import java.util.ArrayList; | |
| import java.util.Collections; | |
| import java.util.Iterator; | |
| +import java.util.List; | |
| import java.util.Map; | |
| import java.util.SortedMap; | |
| import java.util.TreeMap; | |
| @@ -51,6 +53,7 @@ | |
| import org.apache.lucene.util.FixedBitSet; | |
| import org.apache.lucene.util.IOUtils; | |
| import org.apache.lucene.util.IntsRef; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.RamUsageEstimator; | |
| import org.apache.lucene.util.fst.Builder; | |
| import org.apache.lucene.util.fst.ByteSequenceOutputs; | |
| @@ -943,8 +946,8 @@ | |
| return field.hasPayloads(); | |
| } | |
| - public long ramBytesUsed() { | |
| - return ((fst!=null) ? fst.sizeInBytes() : 0); | |
| + public RamUsage ramBytesUsed() { | |
| + return new RamUsage(fst.sizeInBytes(), "fst"); | |
| } | |
| } | |
| @@ -994,13 +997,12 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - long sizeInBytes = 0; | |
| + public RamUsage ramBytesUsed() { | |
| + List<RamUsage> details = new ArrayList<RamUsage>(); | |
| for(Map.Entry<String,TermsReader> entry: fields.entrySet()) { | |
| - sizeInBytes += (entry.getKey().length() * RamUsageEstimator.NUM_BYTES_CHAR); | |
| - sizeInBytes += entry.getValue().ramBytesUsed(); | |
| + details.add(new RamUsage(entry.getKey(), entry.getValue().ramBytesUsed())); | |
| } | |
| - return sizeInBytes; | |
| + return new RamUsage(getClass().getName(), details); | |
| } | |
| }; | |
| } | |
| Index: lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java | |
| =================================================================== | |
| --- lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java (revision 1553093) | |
| +++ lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java (working copy) | |
| @@ -42,6 +42,7 @@ | |
| import org.apache.lucene.util.Bits; | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| /** Concrete class that reads the current doc/freq/skip | |
| * postings format | |
| @@ -650,7 +651,7 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - return ((wrappedPostingsReader!=null) ? wrappedPostingsReader.ramBytesUsed(): 0); | |
| + public RamUsage ramBytesUsed() { | |
| + return new RamUsage(getClass().getName(), wrappedPostingsReader.ramBytesUsed()); | |
| } | |
| } | |
| Index: lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java | |
| =================================================================== | |
| --- lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java (revision 1553093) | |
| +++ lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java (working copy) | |
| @@ -39,6 +39,7 @@ | |
| import org.apache.lucene.util.Bits; | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| /** Concrete class that reads the current doc/freq/skip | |
| * postings format. | |
| @@ -703,7 +704,7 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - return 0; | |
| + public RamUsage ramBytesUsed() { | |
| + return RamUsage.noRamUsage(this); | |
| } | |
| } | |
| Index: lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java | |
| =================================================================== | |
| --- lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java (revision 1553093) | |
| +++ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java (working copy) | |
| @@ -50,6 +50,7 @@ | |
| import org.apache.lucene.store.IndexInput; | |
| import org.apache.lucene.util.Bits; | |
| import org.apache.lucene.util.BytesRef; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.StringHelper; | |
| class SimpleTextDocValuesReader extends DocValuesProducer { | |
| @@ -464,7 +465,7 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - return 0; | |
| + public RamUsage ramBytesUsed() { | |
| + return RamUsage.noRamUsage(this); | |
| } | |
| } | |
| Index: lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java | |
| =================================================================== | |
| --- lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (revision 1553093) | |
| +++ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (working copy) | |
| @@ -18,9 +18,11 @@ | |
| */ | |
| import java.io.IOException; | |
| +import java.util.ArrayList; | |
| import java.util.Collections; | |
| import java.util.HashMap; | |
| import java.util.Iterator; | |
| +import java.util.List; | |
| import java.util.Map; | |
| import java.util.TreeMap; | |
| @@ -41,6 +43,7 @@ | |
| import org.apache.lucene.util.IOUtils; | |
| import org.apache.lucene.util.IntsRef; | |
| import org.apache.lucene.util.OpenBitSet; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.StringHelper; | |
| import org.apache.lucene.util.UnicodeUtil; | |
| import org.apache.lucene.util.fst.Builder; | |
| @@ -571,7 +574,7 @@ | |
| /** Returns approximate RAM bytes used */ | |
| public long ramBytesUsed() { | |
| - return (fst!=null) ? fst.sizeInBytes() : 0; | |
| + return fst.sizeInBytes(); | |
| } | |
| @Override | |
| @@ -657,11 +660,11 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - long sizeInBytes = 0; | |
| - for(SimpleTextTerms simpleTextTerms : termsCache.values()) { | |
| - sizeInBytes += (simpleTextTerms!=null) ? simpleTextTerms.ramBytesUsed() : 0; | |
| + public RamUsage ramBytesUsed() { | |
| + List<RamUsage> details = new ArrayList<RamUsage>(); | |
| + for (Map.Entry<String, SimpleTextTerms> entry : termsCache.entrySet()) { | |
| + details.add(new RamUsage(entry.getValue().ramBytesUsed(), entry.getKey())); | |
| } | |
| - return sizeInBytes; | |
| + return new RamUsage(getClass().getName(), details); | |
| } | |
| } | |
| Index: lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java | |
| =================================================================== | |
| --- lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java (revision 1553093) | |
| +++ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java (working copy) | |
| @@ -33,6 +33,7 @@ | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.CharsRef; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.StringHelper; | |
| import org.apache.lucene.util.UnicodeUtil; | |
| @@ -194,7 +195,7 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - return 0; | |
| + public RamUsage ramBytesUsed() { | |
| + return RamUsage.noRamUsage(this); | |
| } | |
| } | |
| Index: lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java | |
| =================================================================== | |
| --- lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (revision 1553093) | |
| +++ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (working copy) | |
| @@ -41,8 +41,10 @@ | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.CharsRef; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.StringHelper; | |
| import org.apache.lucene.util.UnicodeUtil; | |
| + | |
| import static org.apache.lucene.codecs.simpletext.SimpleTextTermVectorsWriter.*; | |
| /** | |
| @@ -534,7 +536,7 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - return 0; | |
| + public RamUsage ramBytesUsed() { | |
| + return RamUsage.noRamUsage(this); | |
| } | |
| } | |
| Index: lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java (working copy) | |
| @@ -21,9 +21,12 @@ | |
| import java.io.IOException; | |
| import java.io.PrintStream; | |
| import java.io.UnsupportedEncodingException; | |
| +import java.util.ArrayList; | |
| import java.util.Collections; | |
| import java.util.Iterator; | |
| +import java.util.List; | |
| import java.util.Locale; | |
| +import java.util.Map; | |
| import java.util.TreeMap; | |
| import org.apache.lucene.index.CorruptIndexException; | |
| @@ -45,6 +48,7 @@ | |
| import org.apache.lucene.util.Bits; | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.RamUsageEstimator; | |
| import org.apache.lucene.util.StringHelper; | |
| import org.apache.lucene.util.automaton.CompiledAutomaton; | |
| @@ -550,8 +554,8 @@ | |
| } | |
| /** Returns approximate RAM bytes used */ | |
| - public long ramBytesUsed() { | |
| - return ((index!=null)? index.sizeInBytes() : 0); | |
| + public RamUsage ramBytesUsed() { | |
| + return new RamUsage(index.sizeInBytes(), "terms index (fst)"); | |
| } | |
| // NOTE: cannot seek! | |
| @@ -2970,11 +2974,12 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - long sizeInByes = ((postingsReader!=null) ? postingsReader.ramBytesUsed() : 0); | |
| - for(FieldReader reader : fields.values()) { | |
| - sizeInByes += reader.ramBytesUsed(); | |
| + public RamUsage ramBytesUsed() { | |
| + List<RamUsage> details = new ArrayList<RamUsage>(); | |
| + details.add(postingsReader.ramBytesUsed()); | |
| + for (Map.Entry<String, FieldReader> entry : fields.entrySet()) { | |
| + details.add(new RamUsage(entry.getKey(), entry.getValue().ramBytesUsed())); | |
| } | |
| - return sizeInByes; | |
| + return new RamUsage(getClass().getName(), details); | |
| } | |
| } | |
| Index: lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java (working copy) | |
| @@ -26,6 +26,7 @@ | |
| import org.apache.lucene.index.SortedDocValues; | |
| import org.apache.lucene.index.SortedSetDocValues; | |
| import org.apache.lucene.util.Bits; | |
| +import org.apache.lucene.util.RamUsage; | |
| /** Abstract API that produces numeric, binary and | |
| * sorted docvalues. | |
| @@ -65,7 +66,7 @@ | |
| public abstract Bits getDocsWithField(FieldInfo field) throws IOException; | |
| /** Returns approximate RAM bytes used */ | |
| - public abstract long ramBytesUsed(); | |
| + public abstract RamUsage ramBytesUsed(); | |
| /** | |
| * A simple implementation of {@link DocValuesProducer#getDocsWithField} that | |
| Index: lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java (working copy) | |
| @@ -21,6 +21,7 @@ | |
| import java.io.IOException; | |
| import org.apache.lucene.index.Fields; | |
| +import org.apache.lucene.util.RamUsage; | |
| /** Abstract API that produces terms, doc, freq, prox, offset and | |
| * payloads postings. | |
| @@ -38,5 +39,5 @@ | |
| public abstract void close() throws IOException; | |
| /** Returns approximate RAM bytes used */ | |
| - public abstract long ramBytesUsed(); | |
| + public abstract RamUsage ramBytesUsed(); | |
| } | |
| Index: lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java (working copy) | |
| @@ -26,6 +26,7 @@ | |
| import org.apache.lucene.store.IndexInput; | |
| import org.apache.lucene.store.DataInput; | |
| import org.apache.lucene.util.Bits; | |
| +import org.apache.lucene.util.RamUsage; | |
| /** The core terms dictionaries (BlockTermsReader, | |
| * BlockTreeTermsReader) interact with a single instance | |
| @@ -70,7 +71,7 @@ | |
| int flags) throws IOException; | |
| /** Returns approximate RAM bytes used */ | |
| - public abstract long ramBytesUsed(); | |
| + public abstract RamUsage ramBytesUsed(); | |
| @Override | |
| public abstract void close() throws IOException; | |
| Index: lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java (working copy) | |
| @@ -20,6 +20,7 @@ | |
| import java.io.IOException; | |
| import org.apache.lucene.index.StoredFieldVisitor; | |
| +import org.apache.lucene.util.RamUsage; | |
| /** | |
| * Codec API for reading stored fields. | |
| @@ -42,5 +43,5 @@ | |
| public abstract StoredFieldsReader clone(); | |
| /** Returns approximate RAM bytes used */ | |
| - public abstract long ramBytesUsed(); | |
| + public abstract RamUsage ramBytesUsed(); | |
| } | |
| Index: lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java (working copy) | |
| @@ -23,6 +23,7 @@ | |
| import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; // javadocs | |
| import org.apache.lucene.index.DocsAndPositionsEnum; // javadocs | |
| import org.apache.lucene.index.Fields; | |
| +import org.apache.lucene.util.RamUsage; | |
| /** | |
| * Codec API for reading term vectors: | |
| @@ -43,7 +44,7 @@ | |
| public abstract Fields get(int doc) throws IOException; | |
| /** Returns approximate RAM bytes used */ | |
| - public abstract long ramBytesUsed(); | |
| + public abstract RamUsage ramBytesUsed(); | |
| /** Create a clone that one caller at a time may use to | |
| * read term vectors. */ | |
| Index: lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java (working copy) | |
| @@ -55,6 +55,7 @@ | |
| import org.apache.lucene.util.ArrayUtil; | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.packed.PackedInts; | |
| /** | |
| @@ -502,8 +503,8 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - return indexReader.ramBytesUsed(); | |
| + public RamUsage ramBytesUsed() { | |
| + return new RamUsage(getClass().getName(), new RamUsage(indexReader.ramBytesUsed(), "doc index")); | |
| } | |
| } | |
| Index: lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java (working copy) | |
| @@ -56,6 +56,7 @@ | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.IOUtils; | |
| import org.apache.lucene.util.LongsRef; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.packed.BlockPackedReaderIterator; | |
| import org.apache.lucene.util.packed.PackedInts; | |
| @@ -1036,8 +1037,8 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - return indexReader.ramBytesUsed(); | |
| + public RamUsage ramBytesUsed() { | |
| + return new RamUsage(getClass().getName(), new RamUsage(indexReader.ramBytesUsed(), "doc index")); | |
| } | |
| } | |
| Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java (working copy) | |
| @@ -40,6 +40,7 @@ | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.IOUtils; | |
| import org.apache.lucene.util.PagedBytes; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.RamUsageEstimator; | |
| import org.apache.lucene.util.packed.PackedInts; | |
| @@ -651,7 +652,7 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - return ramBytesUsed.get(); | |
| + public RamUsage ramBytesUsed() { | |
| + return new RamUsage(ramBytesUsed.get(), getClass().getName()); | |
| } | |
| } | |
| Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java (working copy) | |
| @@ -40,6 +40,7 @@ | |
| import org.apache.lucene.util.Bits; | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| /** | |
| * Concrete class that reads the 4.0 frq/prox | |
| @@ -1164,8 +1165,8 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - return 0; | |
| + public RamUsage ramBytesUsed() { | |
| + return RamUsage.noRamUsage(this); | |
| } | |
| } | |
| Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java (working copy) | |
| @@ -32,6 +32,7 @@ | |
| import org.apache.lucene.store.IOContext; | |
| import org.apache.lucene.store.IndexInput; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| import java.io.Closeable; | |
| @@ -246,7 +247,7 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - return 0; | |
| + public RamUsage ramBytesUsed() { | |
| + return RamUsage.noRamUsage(this); | |
| } | |
| } | |
| Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java (working copy) | |
| @@ -42,6 +42,8 @@ | |
| import org.apache.lucene.util.Bits; | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| +import org.apache.lucene.util.RamUsageEstimator; | |
| /** | |
| * Lucene 4.0 Term Vectors reader. | |
| @@ -757,8 +759,8 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - return 0; | |
| + public RamUsage ramBytesUsed() { | |
| + return RamUsage.noRamUsage(this); | |
| } | |
| } | |
| Index: lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java (working copy) | |
| @@ -44,6 +44,7 @@ | |
| import org.apache.lucene.util.Bits; | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| /** | |
| * Concrete class that reads docId(maybe frq,pos,offset,payloads) list | |
| @@ -1543,8 +1544,8 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - return 0; | |
| + public RamUsage ramBytesUsed() { | |
| + return RamUsage.noRamUsage(this); | |
| } | |
| } | |
| Index: lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java (working copy) | |
| @@ -18,7 +18,9 @@ | |
| */ | |
| import java.io.IOException; | |
| +import java.util.ArrayList; | |
| import java.util.HashMap; | |
| +import java.util.List; | |
| import java.util.Map; | |
| import java.util.concurrent.atomic.AtomicLong; | |
| @@ -43,6 +45,7 @@ | |
| import org.apache.lucene.util.IOUtils; | |
| import org.apache.lucene.util.IntsRef; | |
| import org.apache.lucene.util.PagedBytes; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.RamUsageEstimator; | |
| import org.apache.lucene.util.fst.BytesRefFSTEnum; | |
| import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput; | |
| @@ -74,7 +77,7 @@ | |
| new HashMap<Integer,FST<Long>>(); | |
| private final int maxDoc; | |
| - private final AtomicLong ramBytesUsed; | |
| + private final List<RamUsage> ramUsage; | |
| static final byte NUMBER = 0; | |
| static final byte BYTES = 1; | |
| @@ -97,7 +100,7 @@ | |
| // read in the entries from the metadata file. | |
| IndexInput in = state.directory.openInput(metaName, state.context); | |
| boolean success = false; | |
| - ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass())); | |
| + ramUsage = new ArrayList<RamUsage>(); | |
| final int version; | |
| try { | |
| version = CodecUtil.checkHeader(in, metaCodec, | |
| @@ -191,8 +194,8 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - return ramBytesUsed.get(); | |
| + public synchronized RamUsage ramBytesUsed() { | |
| + return new RamUsage(getClass().getName(), new ArrayList<RamUsage>(ramUsage)); | |
| } | |
| private NumericDocValues loadNumeric(FieldInfo field) throws IOException { | |
| @@ -211,7 +214,9 @@ | |
| final int formatID = data.readVInt(); | |
| final int bitsPerValue = data.readVInt(); | |
| final PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), entry.packedIntsVersion, maxDoc, bitsPerValue); | |
| - ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed()); | |
| + ramUsage.add(new RamUsage(field.name, | |
| + new RamUsage(RamUsageEstimator.sizeOf(decode), "table"), | |
| + new RamUsage(ordsReader.ramBytesUsed(), "ords "))); | |
| return new NumericDocValues() { | |
| @Override | |
| public long get(int docID) { | |
| @@ -221,12 +226,14 @@ | |
| case DELTA_COMPRESSED: | |
| final int blockSize = data.readVInt(); | |
| final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false); | |
| - ramBytesUsed.addAndGet(reader.ramBytesUsed()); | |
| + ramUsage.add(new RamUsage(field.name, | |
| + new RamUsage(reader.ramBytesUsed(), "values (delta-compressed)"))); | |
| return reader; | |
| case UNCOMPRESSED: | |
| final byte bytes[] = new byte[maxDoc]; | |
| data.readBytes(bytes, 0, bytes.length); | |
| - ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(bytes)); | |
| + ramUsage.add(new RamUsage(field.name, | |
| + new RamUsage(RamUsageEstimator.sizeOf(bytes), "values (uncompressed)"))); | |
| return new NumericDocValues() { | |
| @Override | |
| public long get(int docID) { | |
| @@ -238,7 +245,8 @@ | |
| final long mult = data.readLong(); | |
| final int quotientBlockSize = data.readVInt(); | |
| final BlockPackedReader quotientReader = new BlockPackedReader(data, entry.packedIntsVersion, quotientBlockSize, maxDoc, false); | |
| - ramBytesUsed.addAndGet(quotientReader.ramBytesUsed()); | |
| + ramUsage.add(new RamUsage(field.name, | |
| + new RamUsage(quotientReader.ramBytesUsed(), "quotients (gcd compression)"))); | |
| return new NumericDocValues() { | |
| @Override | |
| public long get(int docID) { | |
| @@ -268,7 +276,9 @@ | |
| final PagedBytes.Reader bytesReader = bytes.freeze(true); | |
| if (entry.minLength == entry.maxLength) { | |
| final int fixedLength = entry.minLength; | |
| - ramBytesUsed.addAndGet(bytes.ramBytesUsed()); | |
| + ramUsage.add(new RamUsage(field.name, | |
| + new RamUsage(bytes.ramBytesUsed(), "term bytes"), | |
| + new RamUsage(bytesReader.ramBytesUsed(), "term bytes reader"))); | |
| return new BinaryDocValues() { | |
| @Override | |
| public void get(int docID, BytesRef result) { | |
| @@ -277,7 +287,10 @@ | |
| }; | |
| } else { | |
| final MonotonicBlockPackedReader addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false); | |
| - ramBytesUsed.addAndGet(bytes.ramBytesUsed() + addresses.ramBytesUsed()); | |
| + ramUsage.add(new RamUsage(field.name, | |
| + new RamUsage(bytes.ramBytesUsed(), "term bytes"), | |
| + new RamUsage(bytesReader.ramBytesUsed(), "term bytes reader"), | |
| + new RamUsage(addresses.ramBytesUsed(), "addresses"))); | |
| return new BinaryDocValues() { | |
| @Override | |
| public void get(int docID, BytesRef result) { | |
| @@ -298,7 +311,7 @@ | |
| if (instance == null) { | |
| data.seek(entry.offset); | |
| instance = new FST<Long>(data, PositiveIntOutputs.getSingleton()); | |
| - ramBytesUsed.addAndGet(instance.sizeInBytes()); | |
| + ramUsage.add(new RamUsage(field.name, new RamUsage(instance.sizeInBytes(), "terms index (fst)"))); | |
| fstInstances.put(field.number, instance); | |
| } | |
| } | |
| @@ -373,7 +386,7 @@ | |
| if (instance == null) { | |
| data.seek(entry.offset); | |
| instance = new FST<Long>(data, PositiveIntOutputs.getSingleton()); | |
| - ramBytesUsed.addAndGet(instance.sizeInBytes()); | |
| + ramUsage.add(new RamUsage(field.name, new RamUsage(instance.sizeInBytes(), "terms index (fst)"))); | |
| fstInstances.put(field.number, instance); | |
| } | |
| } | |
| Index: lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java (working copy) | |
| @@ -29,7 +29,10 @@ | |
| import java.io.Closeable; // javadocs | |
| import java.io.IOException; | |
| +import java.util.ArrayList; | |
| +import java.util.Collections; | |
| import java.util.HashMap; | |
| +import java.util.List; | |
| import java.util.Map; | |
| import org.apache.lucene.codecs.CodecUtil; | |
| @@ -53,6 +56,7 @@ | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.IOUtils; | |
| import org.apache.lucene.util.LongValues; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.packed.BlockPackedReader; | |
| import org.apache.lucene.util.packed.MonotonicBlockPackedReader; | |
| import org.apache.lucene.util.packed.PackedInts; | |
| @@ -67,6 +71,7 @@ | |
| private final IndexInput data; | |
| private final int maxDoc; | |
| private final int version; | |
| + private final List<RamUsage> ramUsage; | |
| // memory-resident structures | |
| private final Map<Integer,MonotonicBlockPackedReader> addressInstances = new HashMap<Integer,MonotonicBlockPackedReader>(); | |
| @@ -78,6 +83,7 @@ | |
| // read in the entries from the metadata file. | |
| IndexInput in = state.directory.openInput(metaName, state.context); | |
| this.maxDoc = state.segmentInfo.getDocCount(); | |
| + ramUsage = Collections.synchronizedList(new ArrayList<RamUsage>()); | |
| boolean success = false; | |
| try { | |
| version = CodecUtil.checkHeader(in, metaCodec, | |
| @@ -285,15 +291,8 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - long sizeInBytes = 0; | |
| - for(MonotonicBlockPackedReader monotonicBlockPackedReader: addressInstances.values()) { | |
| - sizeInBytes += Integer.SIZE + monotonicBlockPackedReader.ramBytesUsed(); | |
| - } | |
| - for(MonotonicBlockPackedReader monotonicBlockPackedReader: ordIndexInstances.values()) { | |
| - sizeInBytes += Integer.SIZE + monotonicBlockPackedReader.ramBytesUsed(); | |
| - } | |
| - return sizeInBytes; | |
| + public RamUsage ramBytesUsed() { | |
| + return new RamUsage(getClass().getName(), new ArrayList<RamUsage>(ramUsage)); | |
| } | |
| LongValues getNumeric(NumericEntry entry) throws IOException { | |
| @@ -376,6 +375,8 @@ | |
| if (addrInstance == null) { | |
| data.seek(bytes.addressesOffset); | |
| addrInstance = new MonotonicBlockPackedReader(data, bytes.packedIntsVersion, bytes.blockSize, bytes.count, false); | |
| + ramUsage.add(new RamUsage(field.name, | |
| + new RamUsage(addrInstance.ramBytesUsed(), "addresses"))); | |
| addressInstances.put(field.number, addrInstance); | |
| } | |
| addresses = addrInstance; | |
| @@ -497,6 +498,8 @@ | |
| if (ordIndexInstance == null) { | |
| data.seek(entry.offset); | |
| ordIndexInstance = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, false); | |
| + ramUsage.add(new RamUsage(field.name, | |
| + new RamUsage(ordIndexInstance.ramBytesUsed(), "ords"))); | |
| ordIndexInstances.put(field.number, ordIndexInstance); | |
| } | |
| ordIndex = ordIndexInstance; | |
| Index: lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java (working copy) | |
| @@ -19,8 +19,10 @@ | |
| import java.io.Closeable; | |
| import java.io.IOException; | |
| +import java.util.ArrayList; | |
| import java.util.HashMap; | |
| import java.util.IdentityHashMap; | |
| +import java.util.List; | |
| import java.util.Map; | |
| import java.util.ServiceLoader; | |
| import java.util.TreeMap; | |
| @@ -39,6 +41,7 @@ | |
| import org.apache.lucene.util.Bits; | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.RamUsageEstimator; | |
| /** | |
| @@ -302,13 +305,12 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - long size = 0; | |
| - for (Map.Entry<String,DocValuesProducer> entry : formats.entrySet()) { | |
| - size += (entry.getKey().length() * RamUsageEstimator.NUM_BYTES_CHAR) + | |
| - entry.getValue().ramBytesUsed(); | |
| + public RamUsage ramBytesUsed() { | |
| + List<RamUsage> details = new ArrayList<RamUsage>(); | |
| + for (DocValuesProducer producer : formats.values()) { | |
| + details.add(producer.ramBytesUsed()); | |
| } | |
| - return size; | |
| + return new RamUsage(getClass().getName(), details); | |
| } | |
| } | |
| Index: lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java (working copy) | |
| @@ -18,15 +18,18 @@ | |
| */ | |
| import java.io.IOException; | |
| +import java.util.ArrayList; | |
| import java.util.Collections; | |
| import java.util.HashMap; | |
| import java.util.Iterator; | |
| +import java.util.List; | |
| import java.util.Map; | |
| import java.util.ServiceLoader; // javadocs | |
| import java.util.Set; | |
| import java.util.TreeMap; | |
| import java.util.TreeSet; | |
| +import org.apache.lucene.codecs.DocValuesProducer; | |
| import org.apache.lucene.codecs.FieldsConsumer; | |
| import org.apache.lucene.codecs.FieldsProducer; | |
| import org.apache.lucene.codecs.PostingsFormat; | |
| @@ -36,6 +39,7 @@ | |
| import org.apache.lucene.index.SegmentWriteState; | |
| import org.apache.lucene.index.Terms; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.RamUsageEstimator; | |
| import static org.apache.lucene.index.FilterAtomicReader.FilterFields; | |
| @@ -238,13 +242,12 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - long sizeInBytes = 0; | |
| - for(Map.Entry<String,FieldsProducer> entry: formats.entrySet()) { | |
| - sizeInBytes += entry.getKey().length() * RamUsageEstimator.NUM_BYTES_CHAR; | |
| - sizeInBytes += entry.getValue().ramBytesUsed(); | |
| + public RamUsage ramBytesUsed() { | |
| + List<RamUsage> details = new ArrayList<RamUsage>(); | |
| + for (FieldsProducer producer : formats.values()) { | |
| + details.add(producer.ramBytesUsed()); | |
| } | |
| - return sizeInBytes; | |
| + return new RamUsage(getClass().getName(), details); | |
| } | |
| } | |
| Index: lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java (working copy) | |
| @@ -21,6 +21,7 @@ | |
| import java.util.Collections; | |
| import java.util.HashMap; | |
| import java.util.LinkedHashSet; | |
| +import java.util.List; | |
| import java.util.Map; | |
| import java.util.Set; | |
| import java.util.concurrent.atomic.AtomicInteger; | |
| @@ -38,6 +39,7 @@ | |
| import org.apache.lucene.store.IOContext; | |
| import org.apache.lucene.util.CloseableThreadLocal; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| /** Holds core readers that are shared (unchanged) when | |
| * SegmentReader is cloned or reopened */ | |
| @@ -191,10 +193,18 @@ | |
| } | |
| /** Returns approximate RAM bytes used */ | |
| - public long ramBytesUsed() { | |
| - return ((normsProducer!=null) ? normsProducer.ramBytesUsed() : 0) + | |
| - ((fields!=null) ? fields.ramBytesUsed() : 0) + | |
| - ((fieldsReaderOrig!=null)? fieldsReaderOrig.ramBytesUsed() : 0) + | |
| - ((termVectorsReaderOrig!=null) ? termVectorsReaderOrig.ramBytesUsed() : 0); | |
| + public void ramBytesUsed(List<RamUsage> details) { | |
| + if (normsProducer != null) { | |
| + details.add(new RamUsage("norms", normsProducer.ramBytesUsed())); | |
| + } | |
| + if (fields != null) { | |
| + details.add(new RamUsage("fields", fields.ramBytesUsed())); | |
| + } | |
| + if (fieldsReaderOrig != null) { | |
| + details.add(new RamUsage("stored fields", fieldsReaderOrig.ramBytesUsed())); | |
| + } | |
| + if (termVectorsReaderOrig != null) { | |
| + details.add(new RamUsage("term vectors", termVectorsReaderOrig.ramBytesUsed())); | |
| + } | |
| } | |
| } | |
| Index: lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java (working copy) | |
| @@ -1,6 +1,7 @@ | |
| package org.apache.lucene.index; | |
| import java.io.IOException; | |
| +import java.util.ArrayList; | |
| import java.util.HashMap; | |
| import java.util.List; | |
| import java.util.Map; | |
| @@ -10,6 +11,7 @@ | |
| import org.apache.lucene.store.Directory; | |
| import org.apache.lucene.store.IOContext; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.RefCount; | |
| /* | |
| @@ -97,12 +99,12 @@ | |
| } | |
| /** Returns approximate RAM bytes used. */ | |
| - synchronized long ramBytesUsed() { | |
| - long ramBytesUsed = 0; | |
| + synchronized RamUsage ramBytesUsed() { | |
| + List<RamUsage> details = new ArrayList<RamUsage>(); | |
| for (RefCount<DocValuesProducer> dvp : genDVProducers.values()) { | |
| - ramBytesUsed += dvp.get().ramBytesUsed(); | |
| + details.add(dvp.get().ramBytesUsed()); | |
| } | |
| - return ramBytesUsed; | |
| + return new RamUsage("doc values", details); | |
| } | |
| } | |
| Index: lucene/core/src/java/org/apache/lucene/index/SegmentReader.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (revision 1553093) | |
| +++ lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (working copy) | |
| @@ -36,6 +36,7 @@ | |
| import org.apache.lucene.store.IOContext; | |
| import org.apache.lucene.util.Bits; | |
| import org.apache.lucene.util.CloseableThreadLocal; | |
| +import org.apache.lucene.util.RamUsage; | |
| /** | |
| * IndexReader implementation over a single segment. | |
| @@ -542,15 +543,15 @@ | |
| } | |
| /** Returns approximate RAM Bytes used */ | |
| - public long ramBytesUsed() { | |
| + public RamUsage ramBytesUsed() { | |
| ensureOpen(); | |
| - long ramBytesUsed = 0; | |
| + final List<RamUsage> details = new ArrayList<RamUsage>(); | |
| if (segDocValues != null) { | |
| - ramBytesUsed += segDocValues.ramBytesUsed(); | |
| + details.add(segDocValues.ramBytesUsed()); | |
| } | |
| if (core != null) { | |
| - ramBytesUsed += core.ramBytesUsed(); | |
| + core.ramBytesUsed(details); | |
| } | |
| - return ramBytesUsed; | |
| + return new RamUsage(toString(), details); | |
| } | |
| } | |
| Index: lucene/core/src/java/org/apache/lucene/util/RamUsage.java | |
| =================================================================== | |
| --- lucene/core/src/java/org/apache/lucene/util/RamUsage.java (revision 0) | |
| +++ lucene/core/src/java/org/apache/lucene/util/RamUsage.java (working copy) | |
| @@ -0,0 +1,88 @@ | |
| +package org.apache.lucene.util; | |
| + | |
| +/* | |
| + * Licensed to the Apache Software Foundation (ASF) under one or more | |
| + * contributor license agreements. See the NOTICE file distributed with | |
| + * this work for additional information regarding copyright ownership. | |
| + * The ASF licenses this file to You under the Apache License, Version 2.0 | |
| + * (the "License"); you may not use this file except in compliance with | |
| + * the License. You may obtain a copy of the License at | |
| + * | |
| + * http://www.apache.org/licenses/LICENSE-2.0 | |
| + * | |
| + * Unless required by applicable law or agreed to in writing, software | |
| + * distributed under the License is distributed on an "AS IS" BASIS, | |
| + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| + * See the License for the specific language governing permissions and | |
| + * limitations under the License. | |
| + */ | |
| + | |
| +import java.util.Arrays; | |
| +import java.util.List; | |
| + | |
| +/** Description of the RAM usage of an object. */ | |
| +public class RamUsage { | |
| + | |
| + /** {@link RamUsage} description for an object that reads everyting from disk. */ | |
| + public static RamUsage noRamUsage(Object o) { | |
| + return new RamUsage(RamUsageEstimator.shallowSizeOf(o), o.getClass().getName()); | |
| + } | |
| + | |
| + private static long totalMemory(List<RamUsage> details) { | |
| + long sum = 0; | |
| + for (RamUsage usage : details) { | |
| + sum += usage.bytes; | |
| + } | |
| + return sum; | |
| + } | |
| + | |
| + /** Number of bytes used. */ | |
| + public final long bytes; | |
| + | |
| + public final String description; | |
| + | |
| + /** Details about the RAM usage. */ | |
| + public final List<RamUsage> details; | |
| + | |
| + /** Constructor. */ | |
| + public RamUsage(long bytes, String description, List<RamUsage> details) { | |
| + this.bytes = bytes; | |
| + this.description = description; | |
| + this.details = details; | |
| + } | |
| + | |
| + /** Constructor that sums up contributions. */ | |
| + public RamUsage(String description, List<RamUsage> details) { | |
| + this(totalMemory(details), description, details); | |
| + } | |
| + | |
| + /** Constructor. */ | |
| + public RamUsage(long bytes, String description, RamUsage... details) { | |
| + this(bytes, description, Arrays.asList(details)); | |
| + } | |
| + | |
| + /** Constructor that sums up contributions. */ | |
| + public RamUsage(String description, RamUsage... details) { | |
| + this(description, Arrays.asList(details)); | |
| + } | |
| + | |
| + private final void toString(int depth, StringBuilder buffer) { | |
| + for (int i = 0; i < depth; i++) { | |
| + buffer.append(" "); | |
| + } | |
| + buffer.append(RamUsageEstimator.humanReadableUnits(bytes)).append(": ").append(description); | |
| + buffer.append("\n"); | |
| + | |
| + for (RamUsage detail : details) { | |
| + detail.toString(depth + 1, buffer); | |
| + } | |
| + } | |
| + | |
| + @Override | |
| + public String toString() { | |
| + final StringBuilder buffer = new StringBuilder(); | |
| + toString(0, buffer); | |
| + return buffer.toString(); | |
| + } | |
| + | |
| +} | |
| Property changes on: lucene/core/src/java/org/apache/lucene/util/RamUsage.java | |
| ___________________________________________________________________ | |
| Added: svn:eol-style | |
| ## -0,0 +1 ## | |
| +native | |
| \ No newline at end of property | |
| Index: lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java | |
| =================================================================== | |
| --- lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java (revision 1553093) | |
| +++ lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java (working copy) | |
| @@ -37,6 +37,7 @@ | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.FixedBitSet; | |
| import org.apache.lucene.util.OpenBitSet; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.RamUsageEstimator; | |
| /** | |
| @@ -304,7 +305,7 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| + public RamUsage ramBytesUsed() { | |
| return in.ramBytesUsed(); | |
| } | |
| } | |
| Index: lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java | |
| =================================================================== | |
| --- lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java (revision 1553093) | |
| +++ lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java (working copy) | |
| @@ -34,6 +34,7 @@ | |
| import org.apache.lucene.index.Terms; | |
| import org.apache.lucene.index.TermsEnum; | |
| import org.apache.lucene.util.BytesRef; | |
| +import org.apache.lucene.util.RamUsage; | |
| /** | |
| * Just like {@link Lucene41PostingsFormat} but with additional asserts. | |
| @@ -86,7 +87,7 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| + public RamUsage ramBytesUsed() { | |
| return in.ramBytesUsed(); | |
| } | |
| } | |
| Index: lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java | |
| =================================================================== | |
| --- lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java (revision 1553093) | |
| +++ lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java (working copy) | |
| @@ -30,6 +30,7 @@ | |
| import org.apache.lucene.index.StoredFieldVisitor; | |
| import org.apache.lucene.store.Directory; | |
| import org.apache.lucene.store.IOContext; | |
| +import org.apache.lucene.util.RamUsage; | |
| /** | |
| * Just like {@link Lucene41StoredFieldsFormat} but with additional asserts. | |
| @@ -73,7 +74,7 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| + public RamUsage ramBytesUsed() { | |
| return in.ramBytesUsed(); | |
| } | |
| } | |
| Index: lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java | |
| =================================================================== | |
| --- lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java (revision 1553093) | |
| +++ lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java (working copy) | |
| @@ -31,6 +31,7 @@ | |
| import org.apache.lucene.store.Directory; | |
| import org.apache.lucene.store.IOContext; | |
| import org.apache.lucene.util.BytesRef; | |
| +import org.apache.lucene.util.RamUsage; | |
| /** | |
| * Just like {@link Lucene40TermVectorsFormat} but with additional asserts. | |
| @@ -72,7 +73,7 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| + public RamUsage ramBytesUsed() { | |
| return in.ramBytesUsed(); | |
| } | |
| } | |
| Index: lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java | |
| =================================================================== | |
| --- lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (revision 1553093) | |
| +++ lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (working copy) | |
| @@ -49,6 +49,7 @@ | |
| import org.apache.lucene.util.BytesRef; | |
| import org.apache.lucene.util.FixedBitSet; | |
| import org.apache.lucene.util.IOUtils; | |
| +import org.apache.lucene.util.RamUsage; | |
| import org.apache.lucene.util.RamUsageEstimator; | |
| /** Stores all postings data in RAM, but writes a small | |
| @@ -87,12 +88,12 @@ | |
| } | |
| @Override | |
| - public long ramBytesUsed() { | |
| - long sizeInBytes = 0; | |
| - for(RAMField field : fieldToTerms.values()) { | |
| - sizeInBytes += field.ramBytesUsed(); | |
| + public RamUsage ramBytesUsed() { | |
| + List<RamUsage> details = new ArrayList<RamUsage>(); | |
| + for (Map.Entry<String, RAMField> entry : fieldToTerms.entrySet()) { | |
| + details.add(new RamUsage(entry.getValue().ramBytesUsed(), entry.getKey())); | |
| } | |
| - return sizeInBytes; | |
| + return new RamUsage(getClass().getName(), details); | |
| } | |
| } | |
| Index: solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java | |
| =================================================================== | |
| --- solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java (revision 1553093) | |
| +++ solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java (working copy) | |
| @@ -576,7 +576,7 @@ | |
| for(AtomicReaderContext atomicReaderContext : reader.leaves()) { | |
| AtomicReader atomicReader = atomicReaderContext.reader(); | |
| if (atomicReader instanceof SegmentReader) { | |
| - indexHeapRamBytesUsed += ((SegmentReader) atomicReader).ramBytesUsed(); | |
| + indexHeapRamBytesUsed += ((SegmentReader) atomicReader).ramBytesUsed().bytes; | |
| } else { | |
| // Not supported for any reader that is not a SegmentReader | |
| return -1; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment