OSM latest planet file torrent create and update scripts

create_torrent.sh

#!/bin/bash
#
# Downloads latest OpenStreetMap planet file and creates torrent for it.
# Based on http://osm-torrent.torres.voyager.hr/ scripts
#
# You should edit "WORKDIR=" line, and copy this script in your /etc/cron.daily
#

DEF_WORKDIR=.					# you must change this, if nothing else...
DEF_FILE_TYPE=planet				# "planet" or "pbfplanet" (or "changesets" for faster testing)
WGET_OPTIONS="--limit-rate=1000k"		# if you want to speed limit wget of PLANET etc.

# those can be overriden from environment, for example:
# env FILE_TYPE=changesets EXPIRE_DAYS=30 WORKDIR=/tmp DATE=101201 create_new_planet_torrent.sh

############################################
# no user configurable parts below
############################################

DEBUG=${DEBUG:-0}
WORKDIR=${WORKDIR:-$DEF_WORKDIR}
FILE_TYPE=${FILE_TYPE:-$DEF_FILE_TYPE}
DATE=${DATE:-$(date --date '2 days ago' +%y%m%d)}	# due to timezones and stuff, it seems we only see the planet when we're already two days ahead. compensate in order to get right planet name... This would not be needed if the script is run on OSMF servers themselves, and you'd have torrents immedaately instead of 48 hours delay....

cd "$WORKDIR" || exit 101

LICENSE="© OpenStreetMap contributors"
if [ "$FILE_TYPE" = "pbfplanet" ]
then
	FILE_PLANET="planet-latest.osm.pbf"
	URL_EXTRA_DIR="pbf/"
else
	exit
fi
FILE_MD5="${FILE_PLANET}.md5"
FILE_TORRENT="${FILE_PLANET}.torrent"
URL_PLANET2="http://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/${URL_EXTRA_DIR}${FILE_PLANET}"	# Germany, mirror planet.osm.org
URL_PLANET="http://planet.osm.org/${URL_EXTRA_DIR}${FILE_PLANET}"	# webseed fallback, original site
URL_MD5="${URL_PLANET}.md5"
CHUNKSIZE=22		# 2^20=1MB, 2^22=4MB, etc. mktorrent 1.0 default ( 2^18=256kB) is too small for our ~15GB files

[ "$DEBUG" -gt 0 ] && echo "source=$URL_PLANET, md5=$URL_MD5, torrent=$FILE_TORRENT"
[ "$DEBUG" -gt 8 ] && exit 9

# abort new download if download currently in progress!
[ -f "$FILE_PLANET" ] && fuser -s "$FILE_PLANET" && exit 1

LAST=`stat -c %Y $FILE_MD5`
echo $LAST
wget -Nq "$WGET_OPTIONS" "$URL_MD5"
CURRENT=`stat -c %Y $FILE_MD5`
echo $CURRENT
RET=$?
[ "$DEBUG" -gt 1 ] && echo "wget $URL_MD5 -- return code $RET"
if [ $CURRENT -le $LAST ]
then
    [ "$DEBUG" -gt 1 ] && echo "already latest planet file"
    exit
fi

# remove torrent file earlier
rm $FILE_TORRENT

# retrieve new planet file name
FILE_PLANET=`cat $FILE_MD5 | awk '{print $2}'`
FILE_TORRENT="${FILE_PLANET}.torrent"
URL_PLANET="http://planet.osm.org/${URL_EXTRA_DIR}${FILE_PLANET}"	# webseed fallback, original site

# download new planet (if remote file changed)
wget -qN -c "$WGET_OPTIONS" "$URL_PLANET"
RET=$?
[ "$DEBUG" -gt 1 ] && echo "wget $URL_PLANET -- return code $RET"

# exit silently if nothing downloaded
[ -f "$FILE_PLANET" ] || exit 0

# exit silently if planet file didn't change (torrent is newer than file)
[ "$FILE_TORRENT" -nt "$FILE_PLANET" ] && exit 0

if md5sum -c $FILE_MD5
then
	echo checksum ok
else
	echo checksum failed, aborting
      rm $FILE_TORRENT
      rm $FILE_MD5
	exit 2
fi

[ "$DEBUG" -gt 1 ] && echo "running mktorrent to create $FILE_TORRENT"

# this is our full featured torrent file: redundant trackers, tcp+udp, ipv4+ipv6, webseed
mktorrent -l $CHUNKSIZE \
  -c "$LICENSE" \
  -a http://ipv4.tracker.osm-torrent.torres.voyager.hr/announce \
  -a udp://tracker.ipv6tracker.org:80/announce,http://tracker.ipv6tracker.org:80/announce \
  -a udp://tracker.publicbt.com:80/announce,http://tracker.publicbt.com:80/announce \
  -a udp://tracker.ccc.de:80/announce,http://tracker.ccc.de/announce \
  -a udp://tracker.openbittorrent.com:80/announce \
  -w $URL_PLANET2 -w $URL_PLANET $FILE_PLANET -o ${FILE_TORRENT}

[ "$DEBUG" -gt 1 ] && echo "running post-process.d for $FILE_TORRENT"

# run scripts in post-process.d directory if any (we can use it to cache torrent on torrage.com, generate RSS, etc)
export DEBUG
if [ -d ./post-process.d ]
then
  run-parts --verbose --arg="$FILE_TORRENT" ./post-process.d
fi

exit 0