openstreetmap-tiles-update-expire

#!/bin/bash

set -e

# openstreetmap-tiles-update-expire will either initialize a replication system
# or run a replication to sync the OSM master database to a local database.
#
# Run openstreetmap-tiles-update-expire with a single argument in YYYY-MM-DD
# format to initialize the replication system to sync from that date. If you
# just did an osm2pgsql import, then use the date of the planet file as your
# argument to openstreetmap-tiles-update-expire.
#
# Run openstreetmap-tiles-update-expire with *no* arguments to have the script
# automatically determine the previous sync timestamp and download an update
# from OSM for a certain interval (interval is defined in
# $WORKOSM_DIR/configuration.txt).
#
#*************************************************************************
#*************************************************************************
OSMOSIS_BIN=/usr/bin/osmosis
OSM2PGSQL_BIN=/usr/local/bin/osm2pgsql
OSM2PGSQL_OPTIONS="-E 4326 --number-processes 2 --style /usr/local/share/osm2pgsql/default.style --hstore -G -v --username import --database osm -C 500"

BASE_DIR=/var/lib/mod_tile
LOG_DIR=/var/log/tiles/
WORKOSM_DIR=$BASE_DIR/.osmosis
RENDERD_SOCK=/var/run/renderd/renderd.sock

LOCK_FILE=/var/run/openstreetmap-update-expire-lock.txt
CHANGE_FILE=$BASE_DIR/changes.osc.gz
EXPIRY_FILE=$BASE_DIR/dirty_tiles
STOP_FILE=$BASE_DIR/stop.txt

OSMOSISLOG=$LOG_DIR/osmosis.log
PGSQLLOG=$LOG_DIR/osm2pgsql.log
EXPIRYLOG=$LOG_DIR/expiry.log
RUNLOG=$LOG_DIR/run.log

EXPIRY_MINZOOM=10
EXPIRY_MAXZOOM=18

#*************************************************************************
#*************************************************************************

m_info()
{
    echo "[`date -Ins`] $$ $1" >> "$RUNLOG"
}

m_error()
{
    echo "[`date -Ins`] $$ [error] $1" >> "$RUNLOG"

    m_info "resetting state"
    cp $WORKOSM_DIR/last.state.txt $WORKOSM_DIR/state.txt || true

    rm "$CHANGE_FILE" || true
    rm "$EXPIRY_FILE.$$" || true
    rm "$LOCK_FILE"
    exit
}

m_ok()
{
    echo "[`date -Ins`] $$ $1" >> "$RUNLOG"
}

getlock()
{
    if [ -s $1 ]; then
        if [ "$(ps -p `cat $1` | wc -l)" -gt 1 ]; then
            return 1 #false
        fi
    fi

    echo $$ >"$1"
    return 0 #true
}

freelock()
{
    rm "$1"
    rm "$CHANGE_FILE"
}


if [ $# -eq 1 ] ; then
    if [[ "$1" != [0-9][0-9][0-9][0-9]-[0-1][0-9]-[0-3][0-9] ]]; then
        m_info "$1 is not a valid date, it should by YYYY-MM-DD format."
        exit 1
    fi
    m_info "Initialising Osmosis replication system to $1"
    mkdir $WORKOSM_DIR
    $OSMOSIS_BIN --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG"
    /usr/bin/wget "http://osm.personalwerk.de/replicate-sequences/?"$1"T00:00:00Z" -O $WORKOSM_DIR/state.txt
else
    # make sure the lockfile is removed when we exit and then claim it

    if ! getlock "$LOCK_FILE"; then
        m_info "pid `cat $LOCK_FILE` still running"
        exit 3
    fi

    if [ -e $STOP_FILE ]; then
        m_info "stopped"
        exit 2
    fi

    seq=`cat $WORKOSM_DIR/state.txt | grep sequenceNumber | /usr/bin/cut -d= -f2`

    if ! [[ "$seq" =~ ^[0-9]+$ ]]; then
        m_error "Sequence number $seq is invalid. Check state.txt file."
        exit 1
    fi

    m_ok "start import from seq-nr $seq, replag is `osmosis-db_replag -h`"

    cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt
    m_ok "downloading diff"

    if ! $OSMOSIS_BIN --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then
        m_error "Osmosis error"
    fi

    m_ok "importing diff"
    EXPIRY_METAZOOM=`/usr/bin/expr $EXPIRY_MAXZOOM - 3`
    if ! $OSM2PGSQL_BIN -a --slim -e$EXPIRY_METAZOOM:$EXPIRY_METAZOOM $OSM2PGSQL_OPTIONS -o "$EXPIRY_FILE.$$" $CHANGE_FILE 1>&2 2> "$PGSQLLOG"; then
        m_error "osm2pgsql error"
    fi

    freelock "$LOCK_FILE"

    m_ok "expiring tiles"
    if ! /usr/local/bin/render_expired --min-zoom=$EXPIRY_MINZOOM --max-zoom=$EXPIRY_MAXZOOM --touch-from=$EXPIRY_MINZOOM -s $RENDERD_SOCK < "$EXPIRY_FILE.$$" 2>&1 | /usr/bin/tail -8 >> "$EXPIRYLOG"; then
        m_info "Expiry failed"
    fi

    rm "$EXPIRY_FILE.$$"

    m_ok "Done with import"
fi