OpenFrameworks addons finder (NodeJS)

addon_crawler.coffee

config = require '../config'
config.setEnvironment 'development'

fs      = require 'fs'
path    = require 'path'
colors  = require 'colors'
github  = require 'octonode'

client = github.client 
  id     : config.GITHUB_CLIENT_ID
  secret : config.GITHUB_SECRET

data_path = path.join __dirname, '..', '..', 'data'

# We retrieve addons year by year because github only gives the first 1000 results
get_addons = (callback, opt_) ->

  opt =
    addons  : []
    skipped : []
    year    : 2010
    page    : 1
    num_results : 0
    increment_num_results : true
  opt[k] = v for k, v of opt_

  console.log "→ fetching YEAR : ".cyan + "#{opt.year}".bold.magenta + " (page #{opt.page})".cyan
  
  q = "ofx in:name created:#{opt.year}-01-01..#{opt.year+1}-01-01"
  client.search().repos
    q        : q
    per_page : 100
    page     : opt.page
  , (err, res) ->

    if err
      github_error_handling err, q, get_addons, [callback, opt]
      return
    else

      if opt.increment_num_results
        opt.num_results += res.total_count

      skipped = []
      for r in res.items
        if /(^)ofx[iA-Z0-9]\w*\S/.test r.name
          opt.addons.push r
        else
          skipped.push r.name

      opt.skipped        = opt.skipped.concat skipped
      num_results_parsed = opt.addons.length + opt.skipped.length

      console.log 'total repos parsed :' + " #{num_results_parsed}/#{opt.num_results}".green
      console.log 'total addons found :' + " #{opt.addons.length}".green
      console.log "skipped #{skipped.length} repos : #{skipped.join ', '}".grey.italic

      if num_results_parsed < opt.num_results
        opt.page++
        opt.increment_num_results = false
        get_addons callback, opt
      else if opt.year < new Date().getFullYear()
        opt.year++
        opt.page = 1
        opt.increment_num_results = true
        get_addons callback, opt
      else callback opt.addons


get_addon_forks = (addon, callback, opt_) ->

  opt =
    forks   : []
    skipped : []
    page    : 1
  opt[k] = v for k, v of opt_

  console.log "→ fetching forks of #{addon.name} (page #{opt.page})"
  
  # client.repo(addon.full_name).forks opt.page, 100, addon.full_name, (err, res) ->
  q = "#{addon.name} in:name fork:only"
  client.search().repos
    q        : q
    per_page : 100
    page     : opt.page
  , (err, res) ->

    if err
      github_error_handling err, q, get_addon_forks, [addon, callback, opt_]
      return
    else

      skipped = []
      for r in res.items
        regex = new RegExp "^#{addon.name}$", 'g'
        if regex.test r.name
          opt.forks.push r
        else
          skipped.push r.name

      opt.skipped        = opt.skipped.concat skipped
      num_results        = res.total_count
      num_results_parsed = opt.skipped.length + opt.forks.length

      if num_results > 0
        console.log 'total repos parsed :' + " #{num_results_parsed}/#{num_results}".green
        console.log 'total forks found  :' + " #{opt.forks.length}".green
        if skipped.length
          console.log "skipped #{skipped.length} repos : #{skipped.join ', '}".grey.italic

      if num_results_parsed < num_results
        opt.page++
        get_addon_forks addon, callback, opt
      else callback opt.forks


github_error_handling = (err, q, callee, args) ->
  diff = -1
  if err.message and err.message.indexOf('rate limit exceeded') > -1
    reset = new Date parseInt(err.headers['x-ratelimit-reset']) * 1000
    today = new Date()
    diff = reset.getTime() - today.getTime()
    console.log "→ API rate limit exceeded, retrying in #{diff/1000} seconds...".yellow
  else
    console.log 'ERROR:'.red
    console.log "→ for query : #{q}".yellow
    console.log err 
    if true or err.code is 'ECONNRESET'
      diff = 1000
      console.log "→ retrying in #{diff/1000} seconds...".yellow
  setTimeout(->
    callee.apply null, args
  , diff)


load_features = (addons, callback, i = 0) ->
  console.log "#{i+1}/#{addons.length}".green + " → loading features of addon " + "#{addons[i].owner}/#{addons[i].name}".bold

  addon = addons[i]
  client.repo(addon.full_name).contents '', (err, data) ->
    if err
      if err.message and (err.message.indexOf('This repository is empty') > -1 or err.statusCode is 500 or err.statusCode is 404)
        console.log "#{err.message}".red
        console.log "invalid repository. skipping..".yellow
        addons.splice(i, 1)
        load_features addons, callback, i
      else
        github_error_handling err, '', load_features, [addons, callback, i]
      return
    
    addon.num_examples = 0
    for file in data
      if file.name == "addon_config.mk" || file.name == "addon.make"
        addon.has_makefile = true
        console.log ' > found ' + 'makefile'.magenta
      else if file.name.match /example/i
        addon.num_examples += 1
        console.log ' > found ' + 'example'.cyan
      else if file.name.match /ofxaddons_thumbnail.png/i
        addon.has_thumbnail = true
        console.log ' > found ' + 'thumbnail'.blue

    if i==addons.length-1
      callback()
    else load_features addons, callback, i+1

load_json = (filename) ->
  filepath = path.join data_path, 'json', filename
  data = fs.readFileSync filepath, 'utf8'
  return JSON.parse(data)

export_json = (data, filename) ->
  filepath = path.join data_path, 'json', filename
  err = fs.writeFileSync filepath, JSON.stringify(data)#, (err) ->
  if err
    console.log "ERROR during export: \n #{err}".red
  else console.log 'JSON exported!'.green


load_forks = (addons, callback, i = 0) ->
  console.log "→ loading forks of addon #{i+1}/#{addons.length}"
  while addons[i].forks is 0
    console.log '0 fork > skipping'.cyan
    i++
  get_addon_forks addons[i], (forks) ->
    if forks.length
      export_json forks, path.join('forks', addons[i].name + '.json')
    if i==addons.length-1
      callback()
    else load_forks addons, callback, i+1

export_compressed_json = (addons) ->
  
  out = []
  for addon in addons
    a = 
      name          : addon.name
      owner         : addon.full_name.split('/')[0] #addon.owner.login
      description   : addon.description
      created_at    : addon.created_at
      updated_at    : addon.updated_at
      num_stars     : addon.stargazers_count
      num_forks     : addon.forks
      has_makefile  : addon.has_makefile
      has_thumbnail : addon.has_thumbnail
      num_examples  : addon.num_examples

    # add fresher forks only
    forks_path = path.join 'forks', "#{addon.name}.json"
    if fs.existsSync path.join(data_path, 'json', forks_path)
      a.forks = []
      forks = load_json forks_path
      for f in forks
        if new Date(f.updated_at).getTime() > new Date(a.updated_at).getTime()
          a.forks.push
            owner : f.owner
            created_at    : f.created_at
            updated_at    : f.updated_at
            num_stars     : f.stargazers_count

    out.push a

  export_json out, 'addons_and_features_compressed.json'


# get_addons (addons) ->
#   export_json addons, 'addons.json'
#   console.log "saved!".green

addons = load_json 'addons.json'

load_forks addons, ->
  console.log '>> done!'.bold.green

# load_features addons, ->
#   export_json addons, 'full_addons.json'
#   console.log '>> done!'.bold.green

# addons = load_json '1_with_features.json'

# export_compressed_json addons