is.go

gistfile1.txt

package main

import (
	"crypto/sha256"
	"crypto/tls"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

type NamespaceRepository struct {
	Name         string `json:"name"`
	Namespace    string `json:"namespace"`
	Type         string `json:"type"`
	Status       int    `json:"status"`
	Description  string `json:"description"`
	Is_automated bool   `json:"is_automated"`
	Is_private   bool   `json:"is_private"`
	Is_official  bool   `json:"is_official"`
	Star_count   int    `json:"star_count"`
	Pull_count   int    `json:"pull_count"`
	Last_updated string `json:"last_updated"`
}

type QueryRepository struct {
	Name         string `json:"name"`
	Namespace    string `json:"namespace"`
	Description  string `json:"description"`
	Is_automated bool   `json:"is_automated"`
	Is_official  bool   `json:"is_official"`
	Star_count   int    `json:"star_count"`
	Pull_count   int    `json:"pull_count"`
}

type FileMu struct {
	// 'count' is the # of threads acting on this file.
	// When its zero then we can remove it from the file map
	count int
	mu    *sync.RWMutex
}

var cache_path = "." + string(os.PathSeparator)
var expire = 10
var used_files map[string]*FileMu = map[string]*FileMu{}
var used_files_mu sync.Mutex
var verbose = 1

var https = &http.Client{Transport: &http.Transport{
	TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}}

func debug(v int, format string, args ...interface{}) {
	if v > verbose {
		return
	}
	fmt.Printf(format, args...)
}

func cacheGet(url string) []byte {
	debug(3, "Get URL: %s\n", url)

	h := sha256.New()
	h.Write([]byte(url))
	name := fmt.Sprintf("%x", h.Sum(nil))
	cache_file := cache_path + name

	debug(3, "  Cache File: %s\n", cache_file)

	defer func() {
		if r := recover(); r != nil {
			debug(0, "Runtime error: %#v\n", r)
		}
	}()

	debug(4, "  Locking used_files_mu\n")
	used_files_mu.Lock()

	// Find file specific lock (create it needed), bump count, and lock it
	uf, ok := used_files[name]
	if !ok {
		debug(4, "  Not in used_files map, adding it\n")
		uf = &FileMu{
			count: 1,
			mu:    &sync.RWMutex{},
		}
		used_files[name] = uf
	} else {
		uf.count++
	}

	debug(4, "  Unlocking used_files_my\n")
	used_files_mu.Unlock()

	uf.mu.RLock() // Read-lock
	bytes, err := ioutil.ReadFile(cache_file)
	uf.mu.RUnlock()

	// Assume any error means we need to go to the server
	if err != nil {
		// Block other threads from trying to create the same file
		uf.mu.Lock() // write-lock

		// First make sure the file is still missing because some
		// other thread could have created it while we were locked out
		bytes, err = ioutil.ReadFile(cache_file)

		// ok, still not there so go ahead and GET it and save it
		if err != nil {
			if resp, err := https.Get(url); err == nil {
				if bytes, err = ioutil.ReadAll(resp.Body); err == nil {
					debug(3, "  Writing to cache file\n")
					err = ioutil.WriteFile(cache_file, bytes, 0644)
					if err != nil {
						debug(1, "  Can't write cache(%s): %s\n",
							cache_file, err)
					}
				} else {
					debug(1, "  Error in reading http response: %s\n", err)
				}
				resp.Body.Close()
			} else {
				debug(1, "  Error on GET: %s\n", err)
			}
		}
		uf.mu.Unlock()

	} else {
		debug(3, "  Using cache\n")
	}

	used_files_mu.Lock()
	if uf, ok := used_files[name]; ok {
		uf.count--
		if uf.count == 0 {
			delete(used_files, name)
		}
	}
	used_files_mu.Unlock()

	return bytes
}

func checkCache(minutes int) {
	for {
		files, err := ioutil.ReadDir(cache_path)
		if err != nil {
			debug(0, "Error reading cache dir: %s\n", err)

			// No point continuing if we can't cache stuff
			os.Exit(1)
		}

		for _, f := range files {
			// Assume files of 64 chars in length are cache files
			if len(f.Name()) != 64 {
				continue
			}

			// Skip any file not older than "minutes" minutes
			if f.ModTime().Unix()+int64(minutes*60) >= time.Now().Unix() {
				continue
			}

			used_files_mu.Lock()
			// If the file is in our map then lock it
			uf, ok := used_files[f.Name()]
			if ok {
				uf.mu.Lock()
			}

			os.Remove(f.Name())

			if ok {
				uf.mu.Unlock()
			}
			used_files_mu.Unlock()
		}

		// Look for old files every minute
		time.Sleep(time.Duration(1 * int(time.Minute)))
	}
}

func main() {
	usage := flag.Usage
	flag.Usage = func() {
		fmt.Println("The Container Search server that can be used to find " +
			"container images.")
		usage()
	}

	port := 8080

	if os.Getenv("PORT") != "" {
		if p, _ := strconv.Atoi(os.Getenv("PORT")); p != 0 {
			port = p
		}
	}

	expire := flag.Int("expire", 10, "time (mins) items live in the cache")
	flag.StringVar(&cache_path, "path", cache_path, "dir path to the cache")
	flag.IntVar(&port, "port", port, "port number for the server to listen on")
	flag.IntVar(&verbose, "v", verbose, "verbose/debug level")
	flag.Parse()

	if cache_path == "" {
		cache_path = "."
	}

	if os.MkdirAll(cache_path, 0644) != nil {
		fmt.Printf("unable to create directory: \"%s\"", cache_path)
		os.Exit(1)
	}

	if cache_path[len(cache_path)-1] != os.PathSeparator {
		cache_path += string(os.PathSeparator)
	}
	go checkCache(*expire)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()

		bytes, err := ioutil.ReadFile(r.URL.Path[1:])
		if err == nil {
			w.Write(bytes)
		} else {
			bytes, err := ioutil.ReadFile("index.html")
			if err == nil {
				w.Write(bytes)
			} else {
				w.Write([]byte("404: \"" + r.URL.Path[1:] + "\" not found"))
			}
		}
	})

	http.HandleFunc("/search", func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()

		w.Header().Set("Content-Type", "application/json")

		query := strings.ToLower(r.FormValue("q"))
		namespace := strings.ToLower(r.FormValue("n"))
		page_size, _ := strconv.Atoi(r.FormValue("s"))
		page, _ := strconv.Atoi(r.FormValue("p"))
		order := r.FormValue("r")

		official := -1
		if o := r.FormValue("o"); len(o) != 0 {
			if o == "1" {
				official = 1
			} else {
				official = 0
			}
		}

		automated := -1
		if a := r.FormValue("a"); len(a) != 0 {
			if a == "1" {
				automated = 1
			} else {
				automated = 0
			}
		}

		if page_size < 1 {
			page_size = 100
		}

		if page < 0 {
			page = 0
		}

		if len(namespace) != 0 {
			var repos []NamespaceRepository
			start := page * page_size
			end := start + page_size
			if !(official == 0 && namespace == "library") && (official != 1 || namespace == "library") {
				is_official := namespace == "library"

				for i := 1; ; i++ {
					bytes := cacheGet("https://hub.docker.com/v2/repositories/" + namespace + "?page_size=1000&page=" + strconv.Itoa(i))

					if len(bytes) > 0 {
						var data map[string]interface{}
						err := json.Unmarshal(bytes, &data)
						if err == nil {
							if data["detail"] != nil && data["detail"].(string) == "Not found" {
								break
							}

							results := data["results"].([]interface{})
							for e := 0; e < len(results); e++ {
								result := results[e].(map[string]interface{})

								name := ""
								if r := result["name"]; r != nil {
									name = r.(string)
								}

								description := ""
								if r := result["description"]; r != nil {
									description = r.(string)
								}

								if len(query) > 0 && !(strings.Contains(strings.ToLower(name), query) || strings.Contains(strings.ToLower(description), query)) {
									continue
								}

								is_automated := false
								if r := result["is_automated"]; r != nil {
									is_automated = r.(bool)
								}
								if (automated == 0 && is_automated) || (automated == 1 && !is_automated) {
									continue
								}

								is_private := false
								if r := result["is_private"]; r != nil {
									is_private = r.(bool)
								}

								namespace := ""
								if r := result["namespace"]; r != nil {
									namespace = r.(string)
								}

								type_ := ""
								if r := result["repository_type"]; r != nil {
									type_ = r.(string)
								}

								status := 0
								if r := result["status"]; r != nil {
									status = int(r.(float64))
								}

								star_count := 0
								if r := result["star_count"]; r != nil {
									star_count = int(r.(float64))
								}

								pull_count := 0
								if r := result["pull_count"]; r != nil {
									pull_count = int(r.(float64))
								}

								last_updated := ""
								if r := result["last_updated"]; r != nil {
									last_updated = r.(string)
								}

								repos = append(repos, NamespaceRepository{name, namespace, type_, status, description, is_automated, is_private, is_official, star_count, pull_count, last_updated})
							}
						} else {
							break
						}
					} else {
						break
					}
				}

				switch order {
				case "star_count":
					sort.Slice(repos[:], func(a, b int) bool {
						return repos[a].Star_count > repos[b].Star_count
					})
					break
				case "-star_count":
					sort.Slice(repos[:], func(a, b int) bool {
						return repos[a].Star_count < repos[b].Star_count
					})
					break
				case "pull_count":
					sort.Slice(repos[:], func(a, b int) bool {
						return repos[a].Pull_count > repos[b].Pull_count
					})
					break
				case "-pull_count":
					sort.Slice(repos[:], func(a, b int) bool {
						return repos[a].Pull_count < repos[b].Pull_count
					})
					break
				}

			}

			response := "{\"count\":" + strconv.Itoa(len(repos))

			if start >= len(repos) {
				repos = repos[:0]
			} else {
				if end > len(repos) {
					end = len(repos)
				}

				repos = repos[start:end]
			}

			results, err := json.Marshal(repos)
			if err == nil {
				response += ",\"results\":" + string(results)
			} else {
				response += ",\"results\":\"[]"
			}

			w.Write([]byte(response + "}\n"))
		} else if len(query) != 0 {
			var repos []QueryRepository
			start := page * page_size
			offset := int(start % 100)
			count := 0
			query = url.QueryEscape(query)

			// swap negatives for the docker hub server
			if len(order) > 0 {
				if order[0:1] == "-" {
					order = "&ordering=" + order[1:]
				} else {
					order = "&ordering=-" + order
				}
			}

			// -1 = either, 0 = not official, 1 = official
			if official != -1 {
				if official == 1 {
					order += "&is_official=1"
				} else {
					order += "&is_official=0"
				}
			}

			if automated != -1 {
				if automated == 1 {
					order += "&is_automated=1"
				} else {
					order += "&is_automated=0"
				}
			}

		query_loop:
			for i := int(start/100) + 1; len(repos) < page_size; i++ {
				bytes := cacheGet("https://hub.docker.com/v2/search/repositories/" + "?page_size=100&page=" + strconv.Itoa(i) + "&query=" + query + order)

				if len(bytes) > 0 {
					var data map[string]interface{}
					err := json.Unmarshal(bytes, &data)
					if err == nil {
						if data["message"] != nil || data["text"] != nil {
							break
						}

						if count == 0 {
							count = int(data["count"].(float64))
						}

						results := data["results"].([]interface{})
						for e := offset; e < len(results); e++ {
							offset = 0
							if len(repos)+1 > page_size {
								break query_loop
							}

							result := results[e].(map[string]interface{})

							name := ""
							if r := result["repo_name"]; r != nil {
								name = r.(string)
							}

							namespace := ""
							if s := strings.Index(name, "/"); s > -1 {
								namespace = name[0:s]
								name = name[s+1:]
							} else {
								namespace = "library"
							}

							description := ""
							if r := result["short_description"]; r != nil {
								description = r.(string)
							}

							is_automated := false
							if r := result["is_automated"]; r != nil {
								is_automated = r.(bool)
							}

							is_official := false
							if r := result["is_official"]; r != nil {
								is_official = r.(bool)
							}

							star_count := 0
							if r := result["star_count"]; r != nil {
								star_count = int(r.(float64))
							}

							pull_count := 0
							if r := result["pull_count"]; r != nil {
								pull_count = int(r.(float64))
							}

							repos = append(repos, QueryRepository{name, namespace, description, is_automated, is_official, star_count, pull_count})
						}
					} else {
						break
					}
				} else {
					break
				}
			}

			response := "{\"count\":" + strconv.Itoa(count)

			results, err := json.Marshal(repos)
			if err == nil {
				response += ",\"results\":" + string(results)
			}

			w.Write([]byte(response + "}\n"))
		} else {
			w.WriteHeader(400)
			w.Write([]byte("{\"error\":{\"message\":\"Incorrect query parameters\"}}\n"))
		}
	})

	// TODO add -v to control debugging info printed
	debug(0, "Listening on port %d\n", port)

	http.ListenAndServe(":"+strconv.Itoa(port), nil)

	// TODO split into files
}