filter.go

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	"runtime"
//	"runtime/debug"
//	"github.com/pkg/profile"
)

type filterFuncc func(*Item, string) bool
type registerFuncType map[string]filterFuncc
type registerGroupByFunc map[string]func(*Item) string
type filterType map[string][]string

//Item as Example
type Item struct {
	ID        int    `json:"id"`
	Name      string `json:"name"`
	Timestamp int64  `json:"time"`
	HumanTime string `json:"humanTime"`
}

//Items as Example
type Items []*Item

type ItemsGroupedBy map[string]Items

func createItems(amount int) Items {
	ll := Items{}
	startTime, _ := time.Parse("2006-01-02", "1800-01-01")
	for i := 0; i < amount; i++ {
		itemTime := startTime.AddDate(0, i, 0)
		item := &Item{
			ID:        i,
			Name:      fmt.Sprintf("Name:%d", i),
			Timestamp: itemTime.Unix(),
			HumanTime: itemTime.Format("2006-01-02"),
		}
		ll = append(ll, item)
	}
	return ll
}

// Filter Functions
func filterNameContains(i *Item, s string) bool {
	return strings.Contains(i.Name, s)
}

func filterIDisEven(i *Item, s string) bool {
	return i.ID%2 == 0
}

func filterBeforeDate(i *Item, s string) bool {
	itemTime := time.Unix(i.Timestamp, 0)
	filterTime, err := time.Parse("2006-01-02", s)
	if err != nil {
		return false
	}
	return itemTime.Before(filterTime)
}

func filterAfterDate(i *Item, s string) bool {
	itemTime := time.Unix(i.Timestamp, 0)
	filterTime, err := time.Parse("2006-01-02", s)
	if err != nil {
		return false
	}
	return itemTime.After(filterTime)
}

func filterMatchDate(i *Item, s string) bool {
	itemTime := time.Unix(i.Timestamp, 0)
	filterTime, err := time.Parse("2006-01-02", s)
	if err != nil {
		return false
	}
	y1, m1, d1 := itemTime.Date()
	y2, m2, d2 := filterTime.Date()
	return y1 == y2 && m1 == m2 && d1 == d2
}

//Runner of filter functions, Item Should pass all
func all(item *Item, filters filterType, registerFuncs registerFuncType) bool {
	for funcName, args := range filters {
		filterFunc := registerFuncs[funcName]
		if filterFunc == nil {
			continue
		}
		for _, arg := range args {
			if !filterFunc(item, arg) {
				return false
			}
		}
	}
	return true
}

//Runner of exlude functions, Item Should pass all
func exclude(item *Item, excludes filterType, registerFuncs registerFuncType) bool {
	for funcName, args := range excludes {
		excludeFunc := registerFuncs[funcName]
		if excludeFunc == nil {
			continue
		}
		for _, arg := range args {
			if excludeFunc(item, arg) {
				return false
			}
		}
	}
	return true
}

func filtered(items Items, filters filterType, excludes filterType, registerFuncs registerFuncType) Items {
	filteredItems := Items{}
	for _, item := range items {
		if !all(item, filters, registerFuncs) {
			continue
		}
		if !exclude(item, excludes, registerFuncs) {
			continue
		}
		filteredItems = append(filteredItems, item)
	}
	return filteredItems
}

func mapIndex(items Items, indexes []int) Items {
	o := Items{}
	for _, index := range indexes {
		o = append(o, items[index])
	}
	return o
}

func filtered1(items Items, filters filterType, excludes filterType, registerFuncs registerFuncType) []Item {
	filteredItems := make([]Item, 0, 100000)
	for _, item := range items {
		if !all(item, filters, registerFuncs) {
			continue
		}
		if !exclude(item, excludes, registerFuncs) {
			continue
		}
		filteredItems = append(filteredItems, *item)
	}
	return filteredItems
}

var registerFuncMap registerFuncType
var registerGroupBy registerGroupByFunc
var itemsUnfilterd Items

func init() {
	registerFuncMap = make(registerFuncType)
	registerFuncMap["namecontains"] = filterNameContains
	registerFuncMap["idiseven"] = filterIDisEven
	registerFuncMap["before"] = filterBeforeDate
	registerFuncMap["after"] = filterAfterDate
	registerFuncMap["match"] = filterMatchDate

	registerGroupBy = make(registerGroupByFunc)
	registerGroupBy["day"] = groupByDay
	registerGroupBy["weekend"] = groupByWeekend
	registerGroupBy["evenid"] = groupByEvenID
	registerGroupBy["year"] = groupByYear

}
// API

func listRest(w http.ResponseWriter, r *http.Request) {
	filterMap, excludeMap := parseURLParameters(r)

	items := filtered(itemsUnfilterd, filterMap, excludeMap, registerFuncMap)

	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Total-Items", strconv.Itoa(len(items)))
	w.WriteHeader(http.StatusOK)

	groupByS, groupByFound := r.URL.Query()["groupby"]
	if !groupByFound {
		json.NewEncoder(w).Encode(items)
		return
	}

	groupByItems := groupByRunner(items, groupByS[0])
	json.NewEncoder(w).Encode(groupByItems)
	go func() {
			time.Sleep(2 * time.Second)
			runtime.GC()
	}()
}

func helpRest(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")

	response := make(map[string][]string)
	registeredFilters := []string{}
	for k := range registerFuncMap {
		registeredFilters = append(registeredFilters, k)
	}
	registeredGroupbys := []string{}
	for k := range registerGroupBy {
		registeredGroupbys = append(registeredGroupbys, k)
	}
	response["filters"] = registeredFilters
	response["groupby"] = registeredGroupbys
	w.WriteHeader(http.StatusOK)

	json.NewEncoder(w).Encode(response)
}

// util for api
func parseURLParameters(r *http.Request) (filterType, filterType) {
	filterMap := make(filterType)
	excludeMap := make(filterType)
	for k := range registerFuncMap {
		parameter, parameterFound := r.URL.Query()[k]
		if parameterFound {
			filterMap[k] = parameter
		}
		parameter, parameterFound = r.URL.Query()["!"+k]
		if parameterFound {
			excludeMap[k] = parameter
		}
	}
	return filterMap, excludeMap
}

// Group by functions

func groupByYear(i *Item) string {
	itemTime := time.Unix(i.Timestamp, 0)
	return strconv.Itoa(int(itemTime.Year()))
}

func groupByEvenID(i *Item) string {
	isEven := i.ID%2 == 0
	if isEven {
		return "even"
	}
	return "oneven"
}

func groupByWeekend(i *Item) string {
	itemTime := time.Unix(i.Timestamp, 0)
	isWeekend := itemTime.Weekday() >= 5
	if isWeekend {
		return "weekend"
	}
	return "weekday"
}

func groupByDay(i *Item) string {
	itemTime := time.Unix(i.Timestamp, 0)
	return strings.ToLower(itemTime.Weekday().String())
}

func groupByRunner(items Items, groubByParameter string) ItemsGroupedBy {
	grouping := make(ItemsGroupedBy)
	groupingFunc := registerGroupBy[groubByParameter]
	if groupingFunc == nil {
		return grouping
	}
	for _, item := range items {
		GroupingKey := groupingFunc(item)
		grouping[GroupingKey] = append(grouping[GroupingKey], item)
	}
	return grouping
}


func main() {
	//defer profile.Start(profile.MemProfile).Stop()
	go runPrintMem()
	//go func() {
	//	for {
	//		time.Sleep(5 * time.Second)
	//		runtime.GC()
	//	}	
	//}()
	//defer profile.Start().Stop()
	//amount := 125000000	//100M
	amount := 45000000	// 10M
	//amount := 1000000	// 1M
	itemsUnfilterd = createItems(amount)
	runserver := true
	groupedByExample := false
	runScript := false
	runProblem := false

	if runserver {
		http.HandleFunc("/", listRest)
		http.HandleFunc("/help/", helpRest)
		fmt.Println("starting server, with:", len(itemsUnfilterd), "items")
		log.Fatal(http.ListenAndServe("0:8080", nil))
	}
	if runScript {

		//Examples how to use filtered function

		filterMap := make(filterType)
		//filterMap["ideven"] = []string{""}
		//filterMap["namecontains"] = []string{"", "8"}
		//filterMap["before"] = []string{"2010-06-01"}
		filterMap["after"] = []string{"0001-03-01"}
		//filterMap["match"] = []string{"2840-03-28"}
		//filterMap["FInName"] = []string{"a", "9"}

		exludeMap := make(filterType)
		items := filtered(itemsUnfilterd, filterMap, exludeMap, registerFuncMap)
		var itemJSON []byte
		//groupByItems := groupByRunner(items, groupByS[0])
		if !groupedByExample {
			itemJSON, _ = json.Marshal(items)
		} else {
			groupByItems := groupByRunner(items, "year")
			itemJSON, _ = json.Marshal(groupByItems)
		}
		fmt.Println(string(itemJSON), "amount:", len(items))
	}

	runtime.GC()
	fmt.Println("start first")
	runtime.GC()
	if runProblem {
		time.Sleep(1 * time.Second)
		for i:=0;i<1;i++ {
			time.Sleep(100 * time.Millisecond)
			go problem(i)
		}
	}
	time.Sleep(20 * time.Second)
	//debug.FreeOSMemory()
	fmt.Println("start second")
	if runProblem {
		time.Sleep(10 * time.Second)
		for i:=0;i<1;i++ {
			time.Sleep(1000 * time.Millisecond)
			//runtime.GC()
			go problem(i)
		}
	}
	time.Sleep(20 * time.Second)
	fmt.Println("Done!!!")
	//debug.FreeOSMemory()
	time.Sleep(30 * time.Second)
}


func problem(workerID int) {
	filterMap := make(filterType)
	exludeMap := make(filterType)
	items := filtered(itemsUnfilterd, filterMap, exludeMap, registerFuncMap)
	fmt.Println(workerID, len(items))
	//for i, _ := range items {
	//	items[i] = nil
	//}
	//items = nil
	runtime.GC()
}

func PrintMemUsage() {
        var m runtime.MemStats
        runtime.ReadMemStats(&m)
        // For info on each, see: https://golang.org/pkg/runtime/#MemStats
        fmt.Printf("Alloc = %v MiB", bToMb(m.Alloc))
        fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc))
        fmt.Printf("\tSys = %v MiB", bToMb(m.Sys))
        fmt.Printf("\tNumGC = %v\n", m.NumGC)
}

func bToMb(b uint64) uint64 {
    return b / 1024 / 1024
}

func runPrintMem() {
	for {
		PrintMemUsage()
		time.Sleep(1 * time.Second)
	}
}