bleve.go

package main

import (
	"fmt"
	"github.com/blevesearch/bleve"
	"github.com/blevesearch/bleve/analysis"
	_ "github.com/blevesearch/bleve/analysis/analyzer/custom"
	_ "github.com/blevesearch/bleve/analysis/analyzer/standard"
	_ "github.com/blevesearch/bleve/analysis/char/zerowidthnonjoiner"
	_ "github.com/blevesearch/bleve/analysis/lang/en"
	_ "github.com/blevesearch/bleve/analysis/lang/ru"
	"github.com/blevesearch/bleve/document"
	"github.com/blevesearch/bleve/index/scorch"
	"github.com/blevesearch/bleve/mapping"
	"github.com/blevesearch/bleve/search/query"
	"os"
	"strconv"
	"sync"
	"time"
)

const anylyzerName = "app"

func buildIndexMapping() (*mapping.IndexMappingImpl, error) {
	indexMapping := bleve.NewIndexMapping()

	var err error
	err = indexMapping.AddCustomAnalyzer(anylyzerName,
		map[string]interface{}{
			"type": `custom`,
			"char_filters": []interface{}{
				`zero_width_spaces`,
			},
			"tokenizer": `unicode`,
			"token_filters": []interface{}{
				`possessive_en`,
				`to_lower`,
				`stop_en`,
				`stop_ru`,
				`stemmer_porter`,
				`stemmer_ru_snowball`,
			},
		})
	if err != nil {
		return nil, err
	}

	myDocMapping := bleve.NewDocumentMapping()

	nameMapping := bleve.NewTextFieldMapping()
	nameMapping.Analyzer = anylyzerName
	nameMapping.Store = false
	//nameMapping.IncludeInAll = true
	//nameMapping.IncludeTermVectors = true
	//nameMapping.DocValues = true
	price := bleve.NewNumericFieldMapping()
	myDocMapping.AddFieldMappingsAt("Price", price)

	myDocMapping.AddFieldMappingsAt("Name", nameMapping)
	myDocMapping.AddFieldMappingsAt("Flag", bleve.NewBooleanFieldMapping())

	myDocMapping.AddFieldMappingsAt("Size", bleve.NewNumericFieldMapping())

	myDocMapping.Dynamic = false

	indexMapping.AddDocumentMapping("doc", myDocMapping)
	//indexMapping.DocValuesDynamic = false
	//indexMapping.IndexDynamic = false
	//indexMapping.StoreDynamic = false


	return indexMapping, nil
}

type MyDoc struct {
	Name  string
	Flag  bool
	Size  int64
	Price float64
	PriceString string
}
type ExportDocument struct {
	ID     string                 `json:"id"`
	Fields map[string]interface{} `json:"fields"`
	Size   int
}

func (d MyDoc) Type() string {
	return "doc"
}
func dirExists(filename string) bool {
	info, err := os.Stat(filename)
	if os.IsNotExist(err) {
		return false
	}
	return info.IsDir()
}
func main() {
	// open a new searchIndex
	m, _ := buildIndexMapping()

	//searchIndex, _ := bleve.NewMemOnly(m)
	if dirExists("bleve.searchIndex") {
		_ = os.RemoveAll("bleve.searchIndex")
	}
	bleve.Config.DefaultIndexType = scorch.Name
	searchIndex, _ := bleve.New("bleve.searchIndex", m)
	//searchIndex, _ := bleve.NewMemOnly(m)
	analyzer := searchIndex.Mapping().AnalyzerNamed(anylyzerName)

	// searchIndex some data
	//_ = searchIndex.Index("1", MyDoc{
	//	Name:  "foo bar biz приветы tests",
	//	Flag:  false,
	//	Size:  1,
	//	Price: 100,
	//})
	//_ = searchIndex.Index("2", MyDoc{
	//	Name:  "foo biz bar приветы test",
	//	Flag:  false,
	//	Size:  2,
	//	Price: 150,
	//})
	//_ = searchIndex.Index("3", MyDoc{
	//	Name:  "foo biz bar приветы test BOOOBS",
	//	Flag:  true,
	//	Size:  2,
	//	Price: 200,
	//})
	var wg sync.WaitGroup
	wg.Add(1)
	b := searchIndex.NewBatch()
	fmt.Printf("Before for\n")

	for i := 1; i <= 10000; i++ {
		_ = b.Index(strconv.Itoa(i), MyDoc{
			Name:  "foo biz bar приветы test BOOOBS",
			Flag:  true,
			Size:  2,
			Price: float64(i),
			PriceString: strconv.Itoa(i),
		})
	}

	b.SetPersistedCallback(func(e error) {
		fmt.Printf("batch ended?\n")
		wg.Done()
	})
	fmt.Printf("Before batch\n")
	_ = searchIndex.Batch(b)
	fmt.Printf("Waiting\n")
	wg.Wait()

	// search for some text

	q := buildQuery("tests", analyzer)
	searchRequest := bleve.NewSearchRequest(q)
	searchRequest.Size = 40
	//searchRequest.Explain = true
	//sizeFacet := bleve.NewFacetRequest("Price", 2)
	//minR := 150.0
	////minRD := minR + 0.000001
	//sizeFacet.AddNumericRange("min", nil, &minR)
	//sizeFacet.AddNumericRange("max", &minR, nil)
	//
	//searchRequest.AddFacet("size", sizeFacet)

	searchResults, _ := searchIndex.Search(searchRequest)
	fmt.Printf("\n\nResults count: %d\n\nTime: %s\n\n", searchResults.Hits.Len(), searchResults.Took)
	searchResults, _ = searchIndex.Search(searchRequest)
	fmt.Printf("\n\nResults count: %d\n\nTime: %s\n\n", searchResults.Hits.Len(), searchResults.Took)
	searchResults, _ = searchIndex.Search(searchRequest)
	fmt.Printf("\n\nResults count: %d\n\nTime: %s\n\n", searchResults.Hits.Len(), searchResults.Took)

	d, _ := searchIndex.Document("1")
	exported := docExport(d)
	//stats := searchIndex.StatsMap()
	//fmt.Printf("%#v", stats)
	//fmt.Printf("%#v", d)
	fmt.Printf("%#v", exported)

}

func buildQuery(searchQuery string, analyzer *analysis.Analyzer) *query.ConjunctionQuery {
	q := bleve.NewConjunctionQuery()

	terms := analyzer.Analyze([]byte(searchQuery))
	queries := make([]query.Query, len(terms))
	for i, term := range terms {
		termQuery := bleve.NewTermQuery(string(term.Term))
		termQuery.SetField("Name")
		queries[i] = termQuery
	}
	q.AddQuery(queries...)
	//priceString := bleve.NewTermQuery("100")
	//priceString.SetField("PriceString")
	//q.AddQuery(priceString)

	//from := 0.0
	//to := 2000.0
	//price := bleve.NewNumericRangeQuery(&from, &to)
	//price.SetField("Price")
	//q.AddQuery(price)


	return q
}

func docExport(doc *document.Document) ExportDocument {
	rv := ExportDocument{
		ID:     doc.ID,
		Fields: map[string]interface{}{},
		Size:   doc.Size(),
	}
	for _, field := range doc.Fields {
		var newval interface{}
		switch field := field.(type) {
		case *document.TextField:
			newval = string(field.Value())
		case *document.NumericField:
			n, err := field.Number()
			if err == nil {
				newval = n
			}
		case *document.DateTimeField:
			d, err := field.DateTime()
			if err == nil {
				newval = d.Format(time.RFC3339Nano)
			}
		case *document.BooleanField:
			b, err := field.Boolean()
			if err == nil {
				newval = b
			}
		}
		existing, existed := rv.Fields[field.Name()]
		if existed {
			switch existing := existing.(type) {
			case []interface{}:
				rv.Fields[field.Name()] = append(existing, newval)
			case interface{}:
				arr := make([]interface{}, 2)
				arr[0] = existing
				arr[1] = newval
				rv.Fields[field.Name()] = arr
			}
		} else {
			rv.Fields[field.Name()] = newval
		}
	}
	return rv
}