github issue comment crawler

gistfile1.txt

package main

import (
  "os"
  "log"
  "encoding/json"
	"golang.org/x/oauth2"
	"github.com/google/go-github/github"
)

func main() {
  ts := oauth2.StaticTokenSource(
    &oauth2.Token{AccessToken: "your token here"},
  )
  tc := oauth2.NewClient(oauth2.NoContext, ts)

  client := github.NewClient(tc)

  file, err := os.OpenFile("./comments.json", os.O_RDWR | os.O_CREATE, 0666)
  if err != nil {
    log.Fatalf("fail to open file: %v", err)
  }
  
  // list all repositories for the authenticated user
  required := 3000
  i := 0
  texts := make([]string, required)
  for n := 1; n < 20000; n++ {
    comments, _, err := client.Issues.ListComments("docker", "docker", n, nil)
    if err != nil {
      log.Fatalf("fail to get comment %v", err)
    }
    for j := 0; j < len(comments); j++ {
      texts[i + j] = *comments[j].Body 
    }
    log.Printf("get %d comments from %d", len(comments), n)
    i = i + len(comments)
    if i >= required {
      break
    }
  }

  enc := json.NewEncoder(file)
  if err := enc.Encode(texts); err != nil {
    log.Fatalf("fail to encode as json:%v", err)
  }

  file.Close()
}