crawl.go

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"time"
	"log"
	"crypto/md5"
)

type workItem struct {
	ID  int    `json:"id"`
	URL string `json:"url"`
}

type workItemQueryResult struct {
	WorkItems []workItem `json:"workItems"`
}

func generateRequestsTFS(client *http.Client, baseURL string, token string) []*http.Request {
	requests := make([]*http.Request, 0)

	queryString := `{"query":"SELECT * FROM workitems WHERE [System.WorkItemType] = 'Code Review' OR [System.WorkItemType] = 'Code Review Participant' OR [System.WorkItemType] = 'Code Review Request' OR [System.WorkItemType] = 'Code Review Response'"}`
	req, _ := http.NewRequest("POST", baseURL, bytes.NewBuffer([]byte(queryString)))
	req.SetBasicAuth("", token)
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Connection", "keep-alive")

	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}

	b, _ := ioutil.ReadAll(resp.Body)
	fmt.Println(string(b))

	resp.Body.Close()

	if resp.StatusCode != 200 {
		fmt.Println(resp.StatusCode)
		fmt.Println(string(b))
		panic(string(b))
	}

	wiqr := workItemQueryResult{}
	json.Unmarshal(b, &wiqr)

	fmt.Println(len(wiqr.WorkItems), "work items to be downloaded")

	if len(wiqr.WorkItems) > 0 {
		for _, wi := range wiqr.WorkItems {
			reqWorkItem, _ := http.NewRequest("GET", wi.URL+"?$expand=all", nil)
			reqWorkItemComments, _ := http.NewRequest("GET", wi.URL+"/comments", nil)

			for _, req := range [2]*http.Request{reqWorkItem, reqWorkItemComments} {
				req.SetBasicAuth("", token)
				req.Header.Set("Content-Type", "application/json")
				req.Header.Set("Connection", "keep-alive")
				requests = append(requests, req)
			}
		}
	}
	return requests
}


func main() {

	if len(os.Args[1:]) != 2 {
		fmt.Println("Usage:", "https://{instance}/{team-project}", "API_TOKEN")
        return
	}
	
	url := os.Args[1] + "/_apis/wit/wiql?api-version=4.1"	
	token := os.Args[1]

	logFileName := string(time.Now().Format("2006-01-02T15_04_05")) + ".log"
	file, err := os.OpenFile(logFileName, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
    if err != nil {
        log.Fatal(err)
    }
    defer file.Close()
	log.SetOutput(file)

	os.Mkdir("data", 0700)

	client := &http.Client{Timeout: 60 * time.Second}
	requests := generateRequestsTFS(client, url, token)

	lenRequests := len(requests)

	for i, req := range requests {
		urlMD5 := md5.Sum([]byte(req.URL.EscapedPath()))
		filePath := fmt.Sprintf("./data/%x.json", urlMD5)

		if _, fs := os.Stat(filePath); os.IsNotExist(fs) {
			var err error
			for retry := 0; retry < 10; retry++ {
				resp, err := client.Do(req)
				if err == nil {
					data, err := ioutil.ReadAll(resp.Body)
					resp.Body.Close()
					err = ioutil.WriteFile(filePath, data, 0644)
					if err != nil {
						log.Panicln(err)
					} else {
						log.Println(fmt.Sprintf("[%d] %s", resp.StatusCode, req.URL))
					}
					break
				}
				fmt.Println("must wait", retry)
				time.Sleep(time.Duration(retry) * time.Second)	
			}

			if err != nil {
				fmt.Println(req.URL)
				panic(err)
			} 
		}
		fmt.Printf("\r%3.2f %%", float64(i)/float64(lenRequests)*100.0)
	}
}