Google I/O 2014 redeem easter egg crawler

ioCrawler.go

//before running make sure you setup a GOPATH env variable and ran: "go get code.google.com/p/go.net/html"

//to run: go run ioCrawler.go -url="http://developers.google.com/"
//also try http://developer.android.com/index.html
//output goo.gl links to try and redeem will be sent to foundLinks.txt

//by the way there's an artificial "rate limit" in func crawler, you can lower that or raise it as you wish
//You can also comment out the onlyGoogleRegex code if you don't want to limit to google.com/youtube
//if you're getting I/O timeout errors, then you might need to increase the timeouts on line 231

package main

import (
	"code.google.com/p/go.net/html"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"regexp"
	"strings"
	"time"
	"sync"
	"math/rand"
)

//prevent from downloading files we can't even read
var extToIgnore = regexp.MustCompile(`(?i)\.(gz|tar|tgz|bz2|jpg|png|jpeg|gif|psd|mp3|webm|ogg|mp4|mpeg|flv|avi|wmv|fla|mov|bin|wav|rpm|dmg|exe|zip|rar|7z|doc|ppt|pdf|iso|torrent)$`)

//if you want to index more domains feel free to update this or remove it entirely (update validateUri too to remove the check at the bottom)
var onlyGoogleRegex = regexp.MustCompile(`(?i)(google|youtube|android|golang|opers.blogspot|oogle.blogspot|youtu|angularjs|angulardart|dartlang|html5rocks)\.(com|org|net|io|be)`)

var googlRegex = regexp.MustCompile(`(?im)((?:goo\.gl/[a-zA-Z0-9]{3,9})|(?:[a-zA-Z0-9]{3,9}(?:/|\\)lg\.oog))`)

//if you're getting a lot of I/O (pun not intended...) timeouts then raise this
var defaultConnectTimeout = time.Duration(2*time.Second)
var defaultRWTimeout = time.Duration(5*time.Second)

var guessTimeout = time.Duration(1000*time.Millisecond)
var rateLimitedGoog bool

var queuedLock = sync.RWMutex{}
var knownLock = sync.RWMutex{}
var knownGooglLock = sync.RWMutex{}
var queuedGooglLock = sync.RWMutex{}

var queuedUris = map[string]bool{}
var knownUris = map[string]bool{}
var queuedGoogl = map[string]bool{}
var knownGoogl = map[string]bool{
	"goo.gl/ZV63Cr": true,
	"goo.gl/1nToQ4": true,
	"goo.gl/q7wKp8": true,
	"goo.gl/dDZIsO": true,
	"goo.gl/R1jVTj": true,
	"goo.gl/B92rvj": true,
	"goo.gl/Au49Um": true,
	"goo.gl/uQvdDQ": true,
	"goo.gl/KBXDW9": true,
	"goo.gl/m3uwvU": true,
	"goo.gl/bhLDq0": true,
	"goo.gl/gdEknO": true,
	"goo.gl/8AUTRP": true,
	"goo.gl/5nz6cg": true,
	"goo.gl/T3mzjk": true,
	"goo.gl/5FwCJy": true,
	"goo.gl/3QGGK0": true,
	"goo.gl/K6E7l8": true,
	"goo.gl/xZM9jt": true,
	"goo.gl/rMFYNX": true,
	"goo.gl/F3noUV": true,
	"goo.gl/JYSlLQ": true,
	"goo.gl/hmgbrW": true,
	"goo.gl/0gNnSE": true,
	"goo.gl/Tw6kP9": true,
	"goo.gl/RB2zhx": true,
	"goo.gl/kJPqT3": true,
	"goo.gl/qllLsG": true,
	"goo.gl/zj9bO0": true,
	"goo.gl/bjAPLg": true,
	"goo.gl/7tKTyd": true,
	"goo.gl/eXeVao": true,
	"goo.gl/Dpoyqi": true,
	"goo.gl/iztHMd": true,
	"goo.gl/HACUad": true,
	"goo.gl/oJDnYs": true,
	"goo.gl/ia7jLd": true,
	"goo.gl/AJRPcI": true,
	"goo.gl/bS7lhD": true,
	"goo.gl/lyBKa0": true,
	"goo.gl/OmM6DN": true,
	"goo.gl/puUsyX": true,
	"goo.gl/Gc8Dsd": true,
	"goo.gl/P3cEUu": true,
	"goo.gl/9lT4ZV": true,
	"goo.gl/HNA9YG": true,
	"goo.gl/xjP4Ke": true,
	"goo.gl/ykmzUC": true,
	"goo.gl/EOiis9": true,
	"goo.gl/wgzU7N": true,
	"goo.gl/5ixjt4": true,
	"goo.gl/L2N5sY": true,
	"goo.gl/SWNo61": true,
	"goo.gl/06Hov3": true,
	"goo.gl/JOzJ9j": true,
	"goo.gl/8JoXn0": true,
	"goo.gl/y1y9Lh": true,
	"goo.gl/FDTdEx": true,
	"goo.gl/LgqaL0": true,
	"goo.gl/OQwgds": true,
	"goo.gl/hZWUau": true,
	"goo.gl/wfkUr7": true,
	"goo.gl/sK6kAA": true,
	"goo.gl/DpYUwO": true,
	"goo.gl/vLppsl": true,
	"goo.gl/KVDTVy": true,
	"goo.gl/dGrMwY": true,
	"goo.gl/YOTeO1": true,
	"goo.gl/PltckZ": true,
	"goo.gl/sUJ1r7": true,
	"goo.gl/okXi4W": true,
	"goo.gl/ENWVvN": true,
	"goo.gl/BcLOM9": true,
	"goo.gl/R9MgF3": true,
	"goo.gl/egnhgc": true,
	"goo.gl/Jca6KP": true,
	"goo.gl/1AiXEk": true,
	"goo.gl/iMmkIj": true,
	"goo.gl/E1vTJk": true,
	"goo.gl/pikTTi": true,
	"goo.gl/9KK0k0": true,
	"goo.gl/Oof8mG": true,
	"goo.gl/X1R54k": true,
	"goo.gl/ykH2OP": true,
	"goo.gl/cuSgmK": true,
	"goo.gl/I6Lpbz": true,
	"goo.gl/JFKDGw": true,
	"goo.gl/o0YxL2": true,
	"goo.gl/cCpQrU": true,
	"goo.gl/sWHuKQ": true,
	"goo.gl/FhGPJW": true,
	"goo.gl/v6jFEL": true,
	"goo.gl/hBKyB5": true,
	"goo.gl/mH9rXs": true,
	"goo.gl/3QXZ9r": true,
	"goo.gl/6M31VD": true,
	"goo.gl/ZaQQyr": true,
	"goo.gl/FbPhW1": true,
	"goo.gl/EePxH9": true,
	"goo.gl/LLy4UO": true,
	"goo.gl/yGQsaV": true,
	"goo.gl/MtXamT": true,
	"goo.gl/SaKrk1": true,
	"goo.gl/Jtc6vr": true,
	"goo.gl/UkoouX": true,
	"goo.gl/JUAYZY": true,
	"goo.gl/1ZTcPQ": true,
	"goo.gl/IUJ41u": true,
	"goo.gl/3A5sUu": true,
	"goo.gl/bPPmeA": true,
	"goo.gl/KF857p": true,
	"goo.gl/PrJELw": true,
	"goo.gl/TDxlVd": true,
	"goo.gl/B9W5Mm": true,
	"goo.gl/m3JCoz": true,
	"goo.gl/ZObZyF": true,
	"goo.gl/aQ1W4k": true,
	"goo.gl/IpXSvB": true,
	"goo.gl/dBPRBg": true,
	"goo.gl/s1HvIH": true,
	"goo.gl/ttepYl": true,
	"goo.gl/IWa8N2": true,
	"goo.gl/Rduuii": true,
	"goo.gl/rvhMqc": true,
	"goo.gl/DJi6x0": true,
	"goo.gl/nEj7wW": true,
	"goo.gl/eFJLBs": true,
	"goo.gl/7jXWOU": true,
	"goo.gl/KiYy3o": true,
	"goo.gl/wS3Ari": true,
	"goo.gl/HH7XXz": true,
	"goo.gl/txZSBa": true,
	"goo.gl/fIabRh": true,
	"goo.gl/iu38dw": true,
	"goo.gl/qa4Frv": true,
	"goo.gl/1QTi2T": true,
	"goo.gl/Ql8Ap8": true,
	"goo.gl/pXX3oV": true,
	"goo.gl/eFJLB": true,
}
var uriQueue = make(chan *url.URL, 16384)
var googlQueue = make(chan *url.URL, 4096)

var googlOutputFile *os.File
var shouldGuess bool

func main() {
	startURL := flag.String("url", "", "the url to start crawling")
	guess := flag.Bool("guess", false, "guess goo.gl links")
	flag.Parse()
	shouldGuess = *guess
	startUri, err := url.Parse(*startURL)
	if err != nil || *startURL == "" {
		fmt.Printf("Failed to parse starting URL: %s. Response: %s\n", *startURL, err)
		return
	}

	googlOutputFile, err = os.OpenFile("./foundLinks.txt", os.O_APPEND | os.O_CREATE | os.O_WRONLY, 0644)
	if err != nil {
		fmt.Printf("Failed to open foundLinks.txt. Error: %s\n", err)
		return
	}
	defer func() {
		if err := googlOutputFile.Close(); err != nil {
			fmt.Printf("Failed to close output file: %s\n", err)
		}
	}()

	for i := 0; i < 48; i++ {
		go crawler()
	}
	for i := 0; i < 8; i++ {
		go googlChecker()
	}
	for i := 0; i < 1; i++ {
		go guesser()
	}
	//fakeGoogl, _ := url.Parse("http://goo.gl/ZV63Cr")
	//googlQueue <- fakeGoogl
	uriQueue <- startUri
	waitForSignal()
}

func crawler() {
	var nextUri *url.URL
	for {
		<-time.After(500*time.Millisecond)
		select {
		case nextUri = <-uriQueue:
			crawl(nextUri)
		}
	}
}

func googlChecker() {
	var nextUri *url.URL
	for {
		if rateLimitedGoog {
			fmt.Println("RATE LIMITED!! Waiting...")
			<-time.After(120*time.Second)
		}
		select {
		case nextUri = <-googlQueue:
			verifyGoogl(nextUri)
		}
	}
}

func normalizeUri(uri *url.URL) string {
	host := uri.Host
	//remove www from the host
	if len(host) > 4 && host[0:4] == "www." {
		host = host[4:]
	}
	path := uri.Path
	if path == "" || path == "/" {
		return host
	}
	pathLen := len(path)
	//remove any trailing slashes
	if path[pathLen-1] == '/' {
		path = path[:pathLen-1]
	}
	//unfortunately including RawQuery in here to fix youtube links not working anymore
	return strings.Join([]string{host, path, uri.RawQuery}, "")
}


func validateUri(uri *url.URL) (bool, string) {
	if uri.Scheme != "http" && uri.Scheme != "https" {
		return false, ""
	}
	normalUri := normalizeUri(uri)
	knownLock.RLock()
	if _, ok := knownUris[normalUri]; ok {
		knownLock.RUnlock()
		//already checked this
		return false, normalUri
	}
	knownLock.RUnlock()
	if uri.Host == "" {
		return false, normalUri
	}
	if extToIgnore.MatchString(uri.Path) {
		return false, normalUri
	}
	if !onlyGoogleRegex.MatchString(uri.Host) {
		return false, normalUri
	}
	return true, normalUri
}

//from http://stackoverflow.com/questions/16895294/how-to-set-timeout-for-http-get-requests-in-golang
func TimeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(net, addr string) (c net.Conn, err error) {
	return func(netw, addr string) (net.Conn, error) {
		conn, err := net.DialTimeout(netw, addr, cTimeout)
		if err != nil {
			return nil, err
		}
		conn.SetDeadline(time.Now().Add(rwTimeout))
		return conn, nil
	}
}
func NewTimeoutClient(connectTimeout time.Duration, readWriteTimeout time.Duration, baseNormalUri string) *http.Client {
	return &http.Client{
		Transport: &http.Transport{
			Dial: TimeoutDialer(connectTimeout, readWriteTimeout),
		},
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			valid, newNormalUri := validateUri(req.URL)
			//allow redirects to the *same* url since we might already have this url in knownUris because we're processing it right now
			if !valid && baseNormalUri != newNormalUri {
				return errors.New("Redirect:Invalid")
			}
			return nil
		},
	}
}

func crawl(uri *url.URL) {
	valid, normalUri := validateUri(uri)
	if !valid || uri == nil {
		queuedLock.Lock()
		delete(queuedUris, normalUri)
		queuedLock.Unlock()
		return
	}
	knownLock.Lock()
	knownUris[normalUri] = true
	knownLock.Unlock()

	queuedLock.Lock()
	delete(queuedUris, normalUri)
	queuedLock.Unlock()

	findYoutubeAnnotations(uri)

	urlStr := uri.String()
	fmt.Printf("Fetching: %s\n", urlStr)
	client := NewTimeoutClient(defaultConnectTimeout, defaultRWTimeout, normalUri)
	resp, err := client.Get(urlStr)
	if err != nil {
		var errMsg string
		urlErr, ok := err.(*url.Error)
		if !ok {
			errMsg = err.Error()
		} else {
			errMsg = urlErr.Err.Error()
		}

		if errMsg != "Redirect:Invalid" {
			fmt.Printf("Failed to Get URL: %s. Response: %s\n", urlStr, errMsg)
		}
		knownLock.Lock()
		delete(knownUris, normalUri)
		knownLock.Unlock()
		return
	}
	htmlBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Printf("Failed to read from server to get full HTML for %s. Error: %s\n", urlStr, err)
		knownLock.Lock()
		delete(knownUris, normalUri)
		knownLock.Unlock()
		return
	}
	resp.Body.Close()
	htmlStr := string(htmlBytes)
	locateAndSaveGoogl(htmlStr)

	doc, err := html.Parse(strings.NewReader(htmlStr))
	if err != nil {
		fmt.Printf("Failed to parse HTML for %s. Error: %s\n", urlStr, err)
		return
	}
	findAnchors(doc, uri)
}

func findAnchors(node *html.Node, baseUri *url.URL) {
	if node.Type == html.ElementNode && (node.Data == "a" || node.Data == "url") {
		for _, attr := range node.Attr {
			if (node.Data == "a" && attr.Key == "href") || (node.Data == "url" && attr.Key == "value") {
				newUri, err := url.Parse(attr.Val)
				if err != nil || newUri == nil {
					fmt.Printf("Invalid url detected: %s. Error %s\n", attr.Val, err)
					break
				}
				if !newUri.IsAbs() {
					newUri = baseUri.ResolveReference(newUri)
				}
				valid, newNorm := validateUri(newUri)
				if valid {
					//fmt.Printf("Found new URL! %s\n", newUri.String())
					queuedLock.Lock()
					if _, ok := queuedUris[newNorm]; !ok {
						queuedUris[newNorm] = true
						queuedLock.Unlock()

						select {
						case uriQueue <- newUri:
						case <-time.After(time.Second * 1):
							//fmt.Println("Queue is full!!")
							return
						}
					} else {
						queuedLock.Unlock()
					}
				}
				break
			}
		}
	} else if node.Type == html.ElementNode && node.Data == "head" {
		//skip looping through head
		node = node.NextSibling
		if node == nil {
			return
		}
	}
	for nextNode := node.FirstChild; nextNode != nil; nextNode = nextNode.NextSibling {
		findAnchors(nextNode, baseUri)
	}
}

func findYoutubeAnnotations(uri *url.URL) {
	if (uri.Host == "youtube.com" || uri.Host == "www.youtube.com") {
		if len(uri.Path) < 6 || uri.Path[0:6] != "/watch" {
			return
		}
		queryVals := uri.Query();
		if queryVals == nil {
			return
		}
		videoIDs, ok := queryVals["v"]
		if !ok || len(videoIDs) < 1 {
			videoIDsCommas, ok := queryVals["video_ids"]
			if !ok || len(videoIDsCommas) < 1 {
				return
			}
			videoIDs = strings.Split(videoIDsCommas[0], ",")
		}
		for _, videoID := range videoIDs {
			addYoutubeAnnotations(videoID)
		}
	} else if (uri.Host == "youtu.be") {
		addYoutubeAnnotations(uri.Path);
	} else {
		return
	}
}

//thanks https://plus.google.com/+MarcLesterTan
func addYoutubeAnnotations(videoID string) {
	if len(videoID) < 5 {
		return
	}
	newUri, err := url.Parse("https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=" + videoID)
	if err != nil || newUri == nil {
		fmt.Printf("Invalid youtube annotations url generated. Error %s\n", err)
		return
	}
	valid, newNorm := validateUri(newUri)
	if valid {
		//fmt.Printf("Added new youtube annotations URL! %s\n", newUri.String())
		queuedLock.Lock()
		if _, ok := queuedUris[newNorm]; !ok {
			queuedUris[newNorm] = true
			queuedLock.Unlock()

			select {
			case uriQueue <- newUri:
			case <-time.After(time.Second * 1):
				//fmt.Println("Queue is full!!")
				return
			}
		} else {
			queuedLock.Unlock()
		}
	}
}

func ReverseString(str string) string {
	i := len(str)
	final := make([]rune, i)
    for _, r := range str {
		i--
        final[i] = r
    }
    return string(final)
}

func locateAndSaveGoogl(htmlStr string) {
	matches := googlRegex.FindAllString(htmlStr, -1)
	if matches == nil {
		return
	}
	for _, match := range matches {
		//detect backwards goo.gl links
		if len(match) > 6 && match[len(match)-6:] == "lg.oog" {
			match = ReverseString(match)
		}
		//fmt.Printf("Found goo.gl link!: %s\n", match)
		//prevent backwards slashes from breaking this
		uri, err := url.Parse("http://" + strings.Replace(match, `\`, "/", -1))
		if err != nil {
			fmt.Printf("Failed to parse goo.gl link: %s. Error: %s\n", match, err)
			continue
		}
		normalUri := normalizeUri(uri)
		knownGooglLock.RLock()
		if _, ok := knownGoogl[normalUri]; ok {
			knownGooglLock.RUnlock()
			continue
		}
		knownGooglLock.RUnlock()

		queuedGooglLock.Lock()
		if _, ok := queuedGoogl[normalUri]; ok {
			queuedGooglLock.Unlock()
			continue
		}
		queuedGoogl[normalUri] = true
		queuedGooglLock.Unlock()

		googlQueue <- uri
	}
}

var validGooglChars = "avE1k4yiQhAzl9B6DpIRdCqnF8tbSr7J2NKGT3VmoMPuc5UHeLjOZWxfwXgsY0"

func guesser() { //s float64
	if !shouldGuess {
		return
	}

	var i int
	n := make([]uint8, 6)
	for {
		if rateLimitedGoog {
			<-time.After(180*time.Second)
			continue
		}

		i = 0
		for i < 6 {
			n[i] = validGooglChars[int(rand.Float64() * 62)]
			i++
		}
		normal := "goo.gl/" + string(n)
		knownGooglLock.RLock()
		if _, ok := knownGoogl[normal]; ok {
			knownGooglLock.RUnlock()
			continue
		}
		knownGooglLock.RUnlock()

		queuedGooglLock.Lock()
		if _, ok := queuedGoogl[normal]; ok {
			queuedGooglLock.Unlock()
			continue
		}
		queuedGoogl[normal] = true
		queuedGooglLock.Unlock()

		uri, err := url.Parse("http://" + normal)
		if err != nil {
			fmt.Printf("Failed to generate valid goo.gl link: %s. Error: %s\n", normal, err)
			continue
		}
		if guessTimeout > 0 {
			<-time.After(guessTimeout)
		}
		googlQueue <- uri
	}
}

var googlClient = &http.Client{
	CheckRedirect: func(req *http.Request, via []*http.Request) error {
		// /events/io/2014/redeem
		//yes... this is ghetto how it works
		if req.URL.Host == "developers.google.com" && len(req.URL.Path) > 22 && req.URL.Path[0:22] == "/events/io/2014/redeem" {
			return errors.New("Redirect:Valid")
		}
		return errors.New("Redirect:Invalid")
	},
}

func verifyGoogl(uri *url.URL) {
	normalUri := normalizeUri(uri)

	queuedGooglLock.Lock()
	delete(queuedGoogl, normalUri)
	queuedGooglLock.Unlock()

	knownGooglLock.Lock()
	if _, ok := knownGoogl[normalUri]; ok {
		knownGooglLock.Unlock()
		return
	}
	knownGoogl[normalUri] = true
	knownGooglLock.Unlock()

	urlStr := uri.String()
	fmt.Println("Fetching goo.gl link: " + urlStr)
	resp, err := googlClient.Head(urlStr)
	if err != nil {
		var errMsg string
		urlErr, ok := err.(*url.Error)
		if !ok {
			errMsg = err.Error()
		} else {
			errMsg = urlErr.Err.Error()
		}

		if errMsg == "Redirect:Valid" {
			rateLimitedGoog = false
			fmt.Printf("Found a valid goo.gl redirect!! %s\n", urlStr)

			if _, err := googlOutputFile.WriteString(urlStr + "\n"); err != nil {
				fmt.Printf("ERROR! Failed to write goo.gl redirect to output file! Error: %s/n", err)
			}
		} else if errMsg == "Redirect:Invalid" {
			rateLimitedGoog = false
			//do nothing
		} else {
			fmt.Printf("Failed to Head URL: %s. Response: %s\n", urlStr, errMsg)
			knownGooglLock.Lock()
			delete(knownGoogl, normalUri)
			knownGooglLock.Unlock()
		}
	}
	if resp.StatusCode == 403 {
		rateLimitedGoog = true
	} else {
		rateLimitedGoog = false
	}
}

func waitForSignal() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	<-c
	fmt.Println("Received SIGINT!")
}

knownUsedLinks.txt

http://goo.gl/ZV63Cr
http://goo.gl/1nToQ4
http://goo.gl/q7wKp8
http://goo.gl/dDZIsO
http://goo.gl/R1jVTj
http://goo.gl/B92rvj
http://goo.gl/Au49Um
http://goo.gl/uQvdDQ
http://goo.gl/KBXDW9
http://goo.gl/m3uwvU
http://goo.gl/bhLDq0
http://goo.gl/gdEknO
http://goo.gl/8AUTRP
http://goo.gl/5nz6cg
http://goo.gl/T3mzjk
http://goo.gl/5FwCJy
http://goo.gl/3QGGK0
http://goo.gl/K6E7l8
http://goo.gl/xZM9jt
http://goo.gl/rMFYNX
http://goo.gl/F3noUV
http://goo.gl/JYSlLQ
http://goo.gl/hmgbrW
http://goo.gl/0gNnSE
http://goo.gl/Tw6kP9
http://goo.gl/RB2zhx
http://goo.gl/kJPqT3
http://goo.gl/qllLsG
http://goo.gl/zj9bO0
http://goo.gl/bjAPLg
http://goo.gl/7tKTyd
http://goo.gl/eXeVao
http://goo.gl/Dpoyqi
http://goo.gl/iztHMd
http://goo.gl/Dpoyqi
http://goo.gl/HACUad
http://goo.gl/oJDnYs
http://goo.gl/ia7jLd
http://goo.gl/AJRPcI
http://goo.gl/bS7lhD
http://goo.gl/lyBKa0
http://goo.gl/OmM6DN
http://goo.gl/puUsyX
http://goo.gl/Gc8Dsd
http://goo.gl/P3cEUu
http://goo.gl/9lT4ZV
http://goo.gl/HNA9YG
http://goo.gl/xjP4Ke
http://goo.gl/ykmzUC
http://goo.gl/EOiis9
http://goo.gl/wgzU7N
http://goo.gl/5ixjt4
http://goo.gl/L2N5sY
http://goo.gl/SWNo61
http://goo.gl/06Hov3
http://goo.gl/JOzJ9j
http://goo.gl/8JoXn0
http://goo.gl/y1y9Lh
http://goo.gl/FDTdEx
http://goo.gl/LgqaL0
http://goo.gl/OQwgds
http://goo.gl/hZWUau
http://goo.gl/wfkUr7
http://goo.gl/sK6kAA
http://goo.gl/DpYUwO
http://goo.gl/vLppsl
http://goo.gl/KVDTVy
http://goo.gl/dGrMwY
http://goo.gl/YOTeO1
http://goo.gl/PltckZ
http://goo.gl/sUJ1r7
http://goo.gl/okXi4W
http://goo.gl/ENWVvN
http://goo.gl/BcLOM9
http://goo.gl/R9MgF3
http://goo.gl/egnhgc
http://goo.gl/Jca6KP
http://goo.gl/1AiXEk
http://goo.gl/iMmkIj
http://goo.gl/E1vTJk
http://goo.gl/pikTTi
http://goo.gl/9KK0k0
http://goo.gl/Oof8mG
http://goo.gl/X1R54k
http://goo.gl/ykH2OP
http://goo.gl/cuSgmK
http://goo.gl/I6Lpbz
http://goo.gl/JFKDGw
http://goo.gl/o0YxL2
http://goo.gl/cCpQrU
http://goo.gl/sWHuKQ
http://goo.gl/FhGPJW
http://goo.gl/v6jFEL
http://goo.gl/hBKyB5
http://goo.gl/mH9rXs
http://goo.gl/3QXZ9r
http://goo.gl/6M31VD
http://goo.gl/ZaQQyr
http://goo.gl/FbPhW1
http://goo.gl/EePxH9
http://goo.gl/LLy4UO
http://goo.gl/yGQsaV
http://goo.gl/SaKrk1
http://goo.gl/MtXamT
http://goo.gl/Jtc6vr
http://goo.gl/UkoouX
http://goo.gl/JUAYZY
http://goo.gl/1ZTcPQ
http://goo.gl/IUJ41u
http://goo.gl/3A5sUu
http://goo.gl/bPPmeA
http://goo.gl/KF857p
http://goo.gl/PrJELw
http://goo.gl/TDxlVd
http://goo.gl/B9W5Mm
http://goo.gl/m3JCoz
http://goo.gl/ZObZyF
http://goo.gl/aQ1W4k
http://goo.gl/IpXSvB
http://goo.gl/dBPRBg
http://goo.gl/s1HvIH
http://goo.gl/ttepYl
http://goo.gl/IWa8N2
http://goo.gl/Rduuii
http://goo.gl/rvhMqc
http://goo.gl/DJi6x0
http://goo.gl/nEj7wW
http://goo.gl/eFJLBs
http://goo.gl/7jXWOU
http://goo.gl/KiYy3o
http://goo.gl/wS3Ari
http://goo.gl/HH7XXz
http://goo.gl/txZSBa
http://goo.gl/fIabRh
http://goo.gl/iu38dw
http://goo.gl/qa4Frv
http://goo.gl/1QTi2T
http://goo.gl/Ql8Ap8
http://goo.gl/pXX3oV
http://goo.gl/eFJLB

if links are valid, they should show up in the found txt file?

Hey James, so I found a few more of the goo.gl URLs after scouring the developers.google.com site. Unfortunately all were taken. Also after doing some research - it seems that mostly all of these links were created on March 14, or 22. Also they are placed in videos and pages that only go as far back as 6 months from today.

You can tell if a link has been taken by checking the stats when adding ".info" after the URL. For example: "goo.gl/XS2wZh.info" shows that this link has been viewed 60 times. (at the time of this post).

I'm pretty sure Google has another set of codes waiting to be distributed. Let me know your thoughts.

Any luck?

http://goo.gl/f6awqb
http://goo.gl/h4h1s8

Came across http://goo.gl/6K9bLw but seems like already used.

More...
http://goo.gl/Ql8Ap8
http://goo.gl/h4h1s8
http://goo.gl/f6awqb
http://goo.gl/iu38dw

goo.gl/RiEzSX

http://goo.gl/022E2j

To get this script to work with the latest version of GO, you need to replace the URL in line 14 "code.google.com/p/go.net/html" with "golang.org/x/net/html".

I wonder if this will be relevant for 2015?