GitHub Issue CSV Exporter

github-issue-csv.go

// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/csv"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
)

func main() {
	token := flag.String("token", "", "GitHub access token")
	headers := flag.Bool("headers", true, "output column headers")
	sort := flag.String("sort", "", "comments, created, or updated")
	order := flag.String("order", "desc", "either asc or desc")
	numsRepo := flag.String("nums", "", "if specified, the query arguments are interpreted as numbers in this repository")
	flag.Parse()
	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(64)
	}

	var next func() (*issue, error)
	if *numsRepo != "" {
		i := 0
		next = func() (*issue, error) {
			if i >= flag.NArg() {
				return nil, nil
			}
			n, err := strconv.Atoi(flag.Arg(i))
			if err != nil {
				return nil, err
			}
			i++
			return readIssue(*token, *numsRepo, n)
		}
	} else {
		query := strings.Join(flag.Args(), " ")
		results := search(*token, query, *sort, *order)
		next = results.next
	}
	if err := write(os.Stdout, *headers, next); err != nil {
		fmt.Fprintln(os.Stderr, "github-issue-csv:", err)
		os.Exit(1)
	}
}

func write(w io.Writer, headers bool, next func() (*issue, error)) error {
	c := csv.NewWriter(os.Stdout)
	wroteHeaders := false
	for {
		iss, err := next()
		if err != nil {
			c.Flush()
			return err
		}
		if iss == nil {
			break
		}
		if headers && !wroteHeaders {
			c.Write([]string{"ID", "#", "Title", "State", "Assignee", "Created", "Updated", "URL"})
			wroteHeaders = true
		}
		assignee := ""
		for i, a := range iss.Assignees {
			if i > 0 {
				assignee += ","
			}
			assignee += a.Name
		}
		c.Write([]string{
			strconv.Itoa(iss.ID),
			strconv.Itoa(iss.Number),
			iss.Title,
			iss.State,
			assignee,
			iss.CreatedAt,
			iss.UpdatedAt,
			iss.URL,
		})
	}
	c.Flush()
	return c.Error()
}

type issue struct {
	ID        int     `json:"id"`
	Number    int     `json:"number"`
	Title     string  `json:"title"`
	State     string  `json:"state"`
	URL       string  `json:"html_url"`
	Assignees []*user `json:"assignees"`
	CreatedAt string  `json:"created_at"`
	UpdatedAt string  `json:"updated_at"`
}

type user struct {
	Name string `json:"login"`
}

func readIssue(token string, repo string, num int) (*issue, error) {
	r := newRequest("GET", "https://api.github.com/repos/"+repo+"/issues/"+strconv.Itoa(num), token)
	res, err := http.DefaultClient.Do(r)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != http.StatusOK {
		// TODO: error text
		return nil, fmt.Errorf("http status %d", res.StatusCode)
	}
	result := new(issue)
	if err := json.NewDecoder(res.Body).Decode(result); err != nil {
		return nil, err
	}
	return result, nil
}

type pager struct {
	token   string
	nextURL string
	nextErr error
	results []*issue
}

func search(token string, q string, sort string, order string) *pager {
	params := url.Values{"q": []string{q}}
	if sort != "" {
		params["sort"] = []string{sort}
	}
	if order != "" {
		params["order"] = []string{order}
	}
	return &pager{
		nextURL: "https://api.github.com/search/issues?" + params.Encode(),
	}
}

func (p *pager) next() (*issue, error) {
	if len(p.results) > 0 {
		iss := p.results[0]
		p.results = p.results[1:]
		return iss, nil
	}
	if p.nextErr != nil {
		return nil, p.nextErr
	}
	if p.nextURL == "" {
		return nil, nil
	}
	res, err := http.DefaultClient.Do(newRequest("GET", p.nextURL, p.token))
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != http.StatusOK {
		// TODO: error text
		return nil, fmt.Errorf("http status %d", res.StatusCode)
	}
	var results struct {
		Items []*issue
	}
	if err := json.NewDecoder(res.Body).Decode(&results); err != nil {
		return nil, err
	}
	if len(results.Items) == 0 {
		p.nextURL = ""
		return nil, nil
	}
	p.results = results.Items[1:]
	links, err := parseLinkHeader(res.Header.Get("Link"))
	if err != nil {
		p.nextURL = ""
		p.nextErr = err
	} else if l := findRelLink(links, "next"); l != nil {
		p.nextURL = l.uri
	} else {
		p.nextURL = ""
	}
	return results.Items[0], nil
}

type link struct {
	uri    string
	params map[string]string
}

func parseLinkHeader(h string) ([]link, error) {
	p := linkParser{h}
	var vals []link
	for {
		l, err := p.link()
		if err != nil {
			return vals, fmt.Errorf("parsing link header %q: %v", h, err)
		}
		if l.uri == "" && l.params == nil {
			break
		}
		vals = append(vals, l)
	}
	return vals, nil
}

func findRelLink(vals []link, rel string) *link {
	for i := range vals {
		if vals[i].params["rel"] == rel {
			return &vals[i]
		}
	}
	return nil
}

type linkParser struct {
	s string
}

func (p *linkParser) link() (link, error) {
	p.skipSpace()
	if len(p.s) == 0 {
		return link{}, nil
	}
	if err := p.consume('<'); err != nil {
		return link{}, err
	}
	uri := p.run(func(b byte) bool { return b != '>' })
	if err := p.consume('>'); err != nil {
		return link{uri: uri}, err
	}
	if uri == "" {
		return link{}, errors.New("empty uri")
	}

	var params map[string]string
	for {
		p.skipSpace()
		if len(p.s) == 0 {
			break
		}
		if err := p.consume(','); err == nil {
			break
		}
		if err := p.consume(';'); err != nil {
			return link{uri, params}, errors.New("expected ; or ,")
		}
		p.skipSpace()
		name := p.run(func(b byte) bool {
			return isAlphaNum(b) || strings.IndexByte("!#$&+-.^_|~`*", b) != -1
		})
		p.skipSpace()
		if err := p.consume('='); err != nil {
			return link{uri, params}, err
		}
		p.skipSpace()
		if len(p.s) == 0 {
			return link{uri, params}, errors.New("expected link param value")
		}
		if params == nil {
			params = make(map[string]string)
		}
		if p.s[0] == '"' {
			var err error
			params[name], err = p.qstr()
			if err != nil {
				return link{uri, params}, fmt.Errorf("link param value %s for %s: %v", name, uri, err)
			}
		} else {
			params[name] = p.run(func(b byte) bool {
				return isAlphaNum(b) || strings.IndexByte("!#$%&'()*+-./:<=>?@[]^_`{|}~", b) != -1
			})
		}
	}
	return link{uri, params}, nil
}

func (p *linkParser) run(f func(b byte) bool) string {
	for i := 0; i < len(p.s); i++ {
		if !f(p.s[i]) {
			s := p.s[:i]
			p.s = p.s[i:]
			return s
		}
	}
	s := p.s
	p.s = ""
	return s
}

func (p *linkParser) qstr() (string, error) {
	if err := p.consume('"'); err != nil {
		return "", err
	}
	var s string
	for {
		s += p.run(func(b byte) bool { return b != '"' && b != '\\' })
		if len(p.s) == 0 {
			return s, errors.New("unterminated quoted string")
		}
		if err := p.consume('"'); err == nil {
			break
		}
		// Otherwise, this should be an escape.
		if err := p.consume('\\'); err != nil {
			panic(err)
		}
		if len(p.s) == 0 {
			return s, errors.New("unexpected end of input in quoted string escape")
		}
		s += string(p.s[0])
		p.s = p.s[1:]
	}
	return s, nil
}

func (p *linkParser) consume(b byte) error {
	if len(p.s) == 0 {
		return fmt.Errorf("expected %c; got EOF", b)
	}
	if p.s[0] != b {
		return fmt.Errorf("expected %c; got %c", b, p.s[0])
	}
	p.s = p.s[1:]
	return nil
}

func (p *linkParser) skipSpace() {
	p.s = strings.TrimLeftFunc(p.s, isSpace)
}

func isSpace(r rune) bool { return r == ' ' || r == '\t' }
func isAlphaNum(b byte) bool {
	return b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z' || b >= '0' && b <= '9'
}

func newRequest(method string, url string, token string) *http.Request {
	r, err := http.NewRequest(method, url, nil)
	if err != nil {
		panic(err)
	}
	r.Header.Set("Accept", "application/vnd.github.v3+json")
	r.Header.Set("User-Agent", "@zombiezen CSV issue downloader")
	if token != "" {
		r.Header.Set("Authorization", "token "+token)
	}
	return r
}