pokeball99

A Common Crawl Experiment

Introduction

At my company, we are building infrastructure that enables us to perform computations involving large bodies of text data.

To get familiar with the tech involved, I started with a simple experiment: using [Common Crawl metadata corpus][1], count crawled URLs grouped by top level domain (TLD).

A simple bookmarklet which makes our painful life a bit easier.

javascript:(function(b,a,c,d,e){if(/\.\w+$/.test(a))b.location=c+a.replace("/blob/","/");else if(e=prompt("Insert a filename:","index.html"))b.location=c+a.replace("/tree/","/")+(~a.indexOf(d)?"/":d)+e})(window,location.pathname,"http://rawgit.com","/master/");

The usage is quite simple:

Go to any repo where you like to preview a file, and execute the booklet. If the current url is a file, it should open the file directly, otherwise it will prompt for a filename (per default it is index.html).

	javascript:function tinycc_bookmarklet(){var script=document.createElement('script');script.src='http://tiny.cc/public/javascript/bookmarklet.js';scriptObj=document.body.appendChild(script);}tinycc_bookmarklet();
	javascript:void(window.open('http://3.ly/?bm=1&u='+encodeURIComponent(location.href)))
	javascript:(function(){var%20d=document,w=window,e=w.getSelection,k=d.getSelection,x=d.selection,s=(e?e():(k)?k():(x?x.createRange().text:0)),f='http://identi.ca//index.php?action=bookmarklet',l=d.location,e=encodeURIComponent,g=f+'&status_textarea=%E2%80%9C'+((e(s))?e(s):e(document.title))+'%E2%80%9D%20%E2%80%94%20'+e(l.href);function%20a(){if(!w.open(g,'t','toolbar=0,resizable=0,scrollbars=1,status=1,width=450,height=200')){l.href=g;}}a();})()
	javascript:var t=((window.getSelection&&window.getSelection())\|\|(document.getSelection&&document.getSelection())\|\|(document.selection&&document.selection.createRange&&document.selection.createRange().text));var e=(document.charset\|\|document.characterSet);if(t!=''){location.

	#!/bin/bash

	# If we get an argument, use it for ssh port, otherwise use default of 22
	if [ -n "$1" ]
	then
	port=$1
	else
	port=22
	fi

	#!/bin/bash

	if [ $# -eq 2 ]; then
	URL=$1
	MAILTO=$2
	else
	echo "usage : $0 check_url alert_mail_address"
	exit 1
	fi

	/**
	*
	* Base64 encode / decode
	* http://www.webtoolkit.info/
	*
	**/

	var Base64 = {

	// private property

	# Communicate with an IRC channel from the shell
	#
	# Runs in background, writes messages to stdout in tty
	# Depends on sic

	TMPF=ircsession.tmp; CHAN='#testircfrombash'; ME=bot001; cat /dev/null > $TMPF; (tail -f $TMPF \| sic -h irc.freenode.net -p 6667 -n $ME \| while read MSG; do case "$MSG" in "$ME") echo ":j $CHAN" >> $TMPF;; "$CHAN") echo "$MSG";; esac; done;) &

	# Then, to write to the channel:
	echo "Hello all!" >> $TMPF
	# or open a buffer to write many rows:

	ssh -NfL 6667:irc.redsrc.com:6667 [email protected]

	# then connect to localhost:6667

	// http://gamecenter.grantgalitz.org/
	var opts = $$("#dropdown_select option"); for(var i=0,cmd=[],val;i<opts.length;val=opts[i++].value){ if(/.+\.gbc?$/.test(val))cmd.push("wget http://gamecenter.grantgalitz.org/romStorage/"+val+".txt") }; cmd.join("\n");