Xapian core Document::Internal buffered terms map

xapian-core-documentinternal_maps_buffered.diff

diff --git a/src/xapian/backends/documentinternal.cc b/src/xapian/backends/documentinternal.cc
index 7c87292fc..a20fda9a4 100644
--- a/src/xapian/backends/documentinternal.cc
+++ b/src/xapian/backends/documentinternal.cc
@@ -38,13 +38,16 @@ namespace Xapian {
 void
 Document::Internal::ensure_terms_fetched() const
 {
-    if (terms)
+    if (terms || terms_buffer)
 	return;
 
+    if (!database.get()) {
+	terms_buffer.reset(new std::deque<terms_buffer_value_type>);
+	return;
+    }
+
     terms.reset(new terms_type());
     termlist_size = 0;
-    if (!database.get())
-	return;
 
     unique_ptr<TermList> t(database->open_term_list(did));
     while (t->next(), !t->at_end()) {
@@ -72,6 +75,38 @@ Document::Internal::ensure_values_fetched() const
     }
 }
 
+void
+Document::Internal::apply_terms_buffer() const
+{
+    if (!terms_buffer)
+	return;
+
+    terms.reset(new terms_type());
+    termlist_size = 0;
+
+    std::vector<terms_buffer_value_type> v(terms_buffer->begin(), terms_buffer->end());
+    std::sort(v.begin(), v.end(), [](const auto& a, const auto& b) {
+	return a.term < b.term;
+    });
+    terms_buffer.reset();
+
+    for (const auto& t : v) {
+	auto i = terms->find(t.term);
+	if (i == terms->end()) {
+	    ++termlist_size;
+	    terms->emplace_hint(terms->end(),
+				t.term,
+				t.has_term_pos ? TermInfo(t.wdf_inc, t.term_pos) : TermInfo(t.wdf_inc));
+	} else if (t.has_term_pos) {
+	    if (i->second.add_position(t.wdf_inc, t.term_pos))
+		++termlist_size;
+	} else {
+	    if (i->second.increase_wdf(t.wdf_inc))
+		++termlist_size;
+	}
+    }
+}
+
 string
 Document::Internal::fetch_data() const
 {
@@ -105,6 +140,8 @@ Document::Internal::set_database(const Database& db) const
 TermList*
 Document::Internal::open_term_list() const
 {
+    apply_terms_buffer();
+
     if (terms)
 	return new DocumentTermList(this);
 
@@ -139,6 +176,8 @@ Document::Internal::get_description() const
 	description_append(desc, *data);
     }
 
+    apply_terms_buffer();
+
     if (terms) {
 	desc += ", terms[";
 	desc += str(terms->size());
diff --git a/src/xapian/backends/documentinternal.h b/src/xapian/backends/documentinternal.h
index a1aaab178..3ba3fa636 100644
--- a/src/xapian/backends/documentinternal.h
+++ b/src/xapian/backends/documentinternal.h
@@ -32,6 +32,7 @@
 #include "xapian/backends/databaseinternal.h"
 #include "xapian/common/overflow.h"
 
+#include <deque>
 #include <map>
 #include <memory>
 #include <string>
@@ -76,6 +77,14 @@ class Document::Internal : public Xapian::Internal::intrusive_base {
     using terms_type = std::map<std::string, TermInfo, std::less<std::string>, allocators::memory_pool_allocator<std::pair<const std::string, TermInfo>>>;
     mutable std::unique_ptr<terms_type> terms;
 
+    struct terms_buffer_value_type {
+	std::string term;
+	Xapian::termcount wdf_inc;
+	bool has_term_pos;
+	Xapian::termpos term_pos;
+    };
+    mutable std::unique_ptr<std::deque<terms_buffer_value_type>> terms_buffer;
+
     /** The number of distinct terms in @a terms.
      *
      *  Only valid when terms is non-NULL.
@@ -110,6 +119,8 @@ class Document::Internal : public Xapian::Internal::intrusive_base {
      */
     void ensure_values_fetched() const;
 
+    void apply_terms_buffer() const;
+
   protected:
     /** Document value slots and their contents.
      *
@@ -198,7 +209,7 @@ class Document::Internal : public Xapian::Internal::intrusive_base {
      *  compared to the version read, otherwise it means modifications
      *  compared to an empty database.
      */
-    bool terms_modified() const { return terms != NULL; }
+    bool terms_modified() const { return terms != NULL || terms_buffer != NULL; }
 
     /** Return true if the document's values might have been modified.
      *
@@ -251,6 +262,11 @@ class Document::Internal : public Xapian::Internal::intrusive_base {
     void add_term(const std::string& term, Xapian::termcount wdf_inc) {
 	ensure_terms_fetched();
 
+	if (terms_buffer) {
+	    terms_buffer->push_back({term, wdf_inc, false, 0});
+	    return;
+	}
+
 	auto i = terms->find(term);
 	if (i == terms->end()) {
 	    ++termlist_size;
@@ -265,6 +281,8 @@ class Document::Internal : public Xapian::Internal::intrusive_base {
     bool remove_term(const std::string& term) {
 	ensure_terms_fetched();
 
+	apply_terms_buffer();
+
 	auto i = terms->find(term);
 	if (i == terms->end()) {
 	    return false;
@@ -286,6 +304,11 @@ class Document::Internal : public Xapian::Internal::intrusive_base {
 	ensure_terms_fetched();
 	positions_modified_ = true;
 
+	if (terms_buffer) {
+	    terms_buffer->push_back({term, wdf_inc, true, term_pos});
+	    return;
+	}
+
 	auto i = terms->find(term);
 	if (i == terms->end()) {
 	    ++termlist_size;
@@ -305,6 +328,8 @@ class Document::Internal : public Xapian::Internal::intrusive_base {
 		   Xapian::termcount wdf_dec) {
 	ensure_terms_fetched();
 
+	apply_terms_buffer();
+
 	auto i = terms->find(term);
 	if (i == terms->end() || i->second.is_deleted()) {
 	    return remove_posting_result::NO_TERM;
@@ -330,6 +355,8 @@ class Document::Internal : public Xapian::Internal::intrusive_base {
 		    Xapian::termpos& n_removed) {
 	ensure_terms_fetched();
 
+	apply_terms_buffer();
+
 	auto i = terms->find(term);
 	if (i == terms->end() || i->second.is_deleted()) {
 	    return remove_posting_result::NO_TERM;
@@ -352,6 +379,9 @@ class Document::Internal : public Xapian::Internal::intrusive_base {
 
     /// Clear all terms from the document.
     void clear_terms() {
+	if (terms_buffer) {
+	    terms_buffer.reset();
+	}
 	if (!terms) {
 	    if (database.get()) {
 		terms.reset(new terms_type());
@@ -370,6 +400,8 @@ class Document::Internal : public Xapian::Internal::intrusive_base {
 
     /// Return the number of distinct terms in this document.
     Xapian::termcount termlist_count() const {
+	apply_terms_buffer();
+
 	if (terms)
 	    return termlist_size;