Downloads the historic top Alexa domains from the Way Back Machine (Internet Archive)

download_historic_alexa_top_domains.sh

#!/bin/bash

# makes this MacOS compatible
DATE_CMD=$(which gdate || which date)

for DAY_AGO in {0..30};
do
	DATE_FILE=$(${DATE_CMD} -d "$DAY_AGO days ago" +%F)
	DATE_URL=$(${DATE_CMD} -d "$DAY_AGO days ago" +%F | sed 's/-//g')

	if [ -e "${DATE_FILE}-top-1m.csv" ]
	then
		echo "Skipping $DATE_FILE since file exists ..."
	else
		URL="https://web.archive.org/web/${DATE_URL}/http://s3.amazonaws.com/alexa-static/top-1m.csv.zip"
		curl -L "$URL" --output "${DATE_FILE}-top-1m.csv.zip" && \
			unzip "${DATE_FILE}-top-1m.csv.zip" && \
			mv top-1m.csv "${DATE_FILE}-top-1m.csv"  && \
			rm "${DATE_FILE}-top-1m.csv.zip"
	fi
done