github API examples/collection

github_api_examples.sh

# WRONG: list all repos for an organisation (for example for atlasoflivingaustralia):
bash-3.2$ curl -s -i https://api.github.com/orgs/atlasoflivingaustralia/repos | grep '"name":' | sed -e 's/^.*name"://g'
 "ala-install",
 "collectory",
 "amrin-hub",
 "appd-hub",
 "asbp-hub",
 "avh-hub",
 "biocache-hubs",
 "generic-hub",
 "obis-hub",
 "ozcam-hub",
 "tepapa-hub",
 "ala-hub",
 "atlasoflivingaustralia.github.io",
 "misc",
 "biocache-jms",
 "biocache-service",
 "biocache-store",
 "analysis-service",
 "layer-ingestion",
 "layers-service",
 "layers-store",
 "spatial-portal",
 "fieldcapture",
 "ecodata",
 "ozatlas",
 "ozatlas-android",
 "fieldcapture-mobile",
 "userdetails",
 "fielddata-android",
 "fielddata-mobile",

# CORRECT:
bash-3.2$ curl -s -i "https://api.github.com/orgs/atlasoflivingaustralia/repos?page=1" | grep '"name":' | sed -e 's/^.*name"://g'
 "ala-install",
 "collectory",
 "amrin-hub",
 "appd-hub",
 "asbp-hub",
 "avh-hub",
 "biocache-hubs",
 "generic-hub",
 "obis-hub",
 "ozcam-hub",
 "tepapa-hub",
 "ala-hub",
 "atlasoflivingaustralia.github.io",
 "misc",
 "biocache-jms",
 "biocache-service",
 "biocache-store",
 "analysis-service",
 "layer-ingestion",
 "layers-service",
 "layers-store",
 "spatial-portal",
 "fieldcapture",
 "ecodata",
 "ozatlas",
 "ozatlas-android",
 "fieldcapture-mobile",
 "userdetails",
 "fielddata-android",
 "fielddata-mobile",
bash-3.2$ curl -s -i "https://api.github.com/orgs/atlasoflivingaustralia/repos?page=2" | grep '"name":' | sed -e 's/^.*name"://g'
 "fielddata-proxy",
 "ozatlas-proxy",
 "image-loader",
 "image-service",
 "image-tiling-agent",
 "image-utils",
 "images-client-plugin",
 "fieldcapture-hubs",
 "sandbox",
 "ala-name-matching",
 "ala-names-generator",
 "sds-webapp2",
 "sensitive-species",
 "ala-cas",
 "ala-cas-client",
 "ala-downloads",
 "ala-fieldguide",
 "ala-logger",
 "ala-logger-service",
 "bie-profile",
 "bie-service",
 "bie-webapp2",
 "specieslist-webapp",
 "bhl-ftindex-manage",
 "bhl-demo-app",
 "bhl-ftindexer",
 "bhl-solr-plugin",
 "ala-expert",
 "specimenbrowser",
 "volunteer-portal",
bash-3.2$ curl -s -i "https://api.github.com/orgs/atlasoflivingaustralia/repos?page=3" | grep '"name":' | sed -e 's/^.*name"://g'
 "dashboard",
 "fielddata",
 "sightings",
 "tviewer",
 "fieldcapture.test",
 "ala-web-theme",
 "alerts",
 "regions",
 "apikey",
 "webapi",
 "documentation",
 "ala-soils2sat",

# how many page-s are there?
bash-3.2$ curl -s -I "https://api.github.com/users/atlasoflivingaustralia/repos"
HTTP/1.1 200 OK
Server: GitHub.com
Date: Mon, 28 Jul 2014 06:58:28 GMT
Content-Type: application/json; charset=utf-8
Status: 200 OK
X-RateLimit-Limit: 60
X-RateLimit-Remaining: 42
X-RateLimit-Reset: 1406532185
Cache-Control: public, max-age=60, s-maxage=60
ETag: "52c8a953ab8e99040ecb9a0bb7faec64"
Vary: Accept
X-GitHub-Media-Type: github.v3
Link: <https://api.github.com/user/7296572/repos?page=2>; rel="next", <https://api.github.com/user/7296572/repos?page=3>; rel="last"
X-XSS-Protection: 1; mode=block
X-Frame-Options: deny
Content-Security-Policy: default-src 'none'
Content-Length: 179554
Access-Control-Allow-Credentials: true
Access-Control-Expose-Headers: ETag, Link, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval
Access-Control-Allow-Origin: *
X-GitHub-Request-Id: 9853C1C5:7B2F:41A5E84:53D5F493
Strict-Transport-Security: max-age=31536000; includeSubdomains
X-Content-Type-Options: nosniff
Vary: Accept-Encoding
X-Served-By: 132026e9262a0093e437f99db5f1e499

# the important part is in:
bash-3.2$ curl -s -I "https://api.github.com/orgs/atlasoflivingaustralia/repos" | grep '^Link:'
Link: <https://api.github.com/organizations/7296572/repos?page=2>; rel="next", <https://api.github.com/organizations/7296572/repos?page=3>; rel="last"

# extract the number:
bash-3.2$ curl -s -I "https://api.github.com/orgs/atlasoflivingaustralia/repos" | grep '^Link:' | sed -e 's/^Link:.*page=//g' | sed -e 's/>.*$//g'
3

# sort the output, replace the 3 curl lines with a normal loop from 1..3 later
bash-3.2$ curl -s -i "https://api.github.com/orgs/atlasoflivingaustralia/repos?page=1" | grep '"name":' | sed -e 's/^.*name"://g' >> /tmp/repo_names.out
bash-3.2$ curl -s -i "https://api.github.com/orgs/atlasoflivingaustralia/repos?page=2" | grep '"name":' | sed -e 's/^.*name"://g' >> /tmp/repo_names.out
bash-3.2$ curl -s -i "https://api.github.com/orgs/atlasoflivingaustralia/repos?page=3" | grep '"name":' | sed -e 's/^.*name"://g' >> /tmp/repo_names.out
bash-3.2$ cat /tmp/repo_names.out | sort
 "ala-cas",
 "ala-cas-client",
 "ala-downloads",
 "ala-expert",
 "ala-fieldguide",
 "ala-hub",
 "ala-install",
 "ala-logger",
 "ala-logger-service",
 "ala-name-matching",
 "ala-names-generator",
 "ala-soils2sat",
 "ala-web-theme",
 "alerts",
 "amrin-hub",
 "analysis-service",
 "apikey",
 "appd-hub",
 "asbp-hub",
 "atlasoflivingaustralia.github.io",
 "avh-hub",
 "bhl-demo-app",
 "bhl-ftindex-manage",
 "bhl-ftindexer",
 "bhl-solr-plugin",
 "bie-profile",
 "bie-service",
 "bie-webapp2",
 "biocache-hubs",
 "biocache-jms",
 "biocache-service",
 "biocache-store",
 "collectory",
 "dashboard",
 "documentation",
 "ecodata",
 "fieldcapture",
 "fieldcapture-hubs",
 "fieldcapture-mobile",
 "fieldcapture.test",
 "fielddata",
 "fielddata-android",
 "fielddata-mobile",
 "fielddata-proxy",
 "generic-hub",
 "image-loader",
 "image-service",
 "image-tiling-agent",
 "image-utils",
 "images-client-plugin",
 "layer-ingestion",
 "layers-service",
 "layers-store",
 "misc",
 "obis-hub",
 "ozatlas",
 "ozatlas-android",
 "ozatlas-proxy",
 "ozcam-hub",
 "regions",
 "sandbox",
 "sds-webapp2",
 "sensitive-species",
 "sightings",
 "spatial-portal",
 "specieslist-webapp",
 "specimenbrowser",
 "tepapa-hub",
 "tviewer",
 "userdetails",
 "volunteer-portal",
 "webapi",

githubapi-get-all-repos.sh

#!/bin/bash

if [ -z "$1" ]; then
    echo "usage: ./githubapi-get-all-repos.sh [github username]"
    exit 1;
fi

# "For unauthenticated requests, the rate limit allows you to make up to 60 requests per hour."
# https://developer.github.com/v3/#rate-limiting
#
GITHUB_REQUEST_RATE=2

temp=`basename $0`
TMPFILE=`mktemp /tmp/${temp}.XXXXXX` || exit 1

# single page result-s (no pagination), have no Link: section, the grep result is empty
last_page=`curl -s -I "https://api.github.com/users/$1/repos" | grep '^Link:'`
sleep $GITHUB_REQUEST_RATE

# does this result use pagination?
if [ -z "$last_page" ]; then
    # no - this result has only one page
    curl -s -i "https://api.github.com/users/$1/repos" | grep '"name":' >> $TMPFILE;

else
    # yes - this result is on multiple pages; extract the last_page number
    last_page=`echo $last_page | sed -e 's/^Link:.*page=//g' -e 's/>.*$//g'`

    p=1
    while [ "$p" -le "$last_page" ]; do
	curl -s -i "https://api.github.com/users/$1/repos?page=$p" | grep '"name":' >> $TMPFILE
	p=$(($p + 1))
	sleep $GITHUB_REQUEST_RATE
    done
fi

cat $TMPFILE | sed -e 's/^ *"name": "//g' -e 's/",$//g' | sort