Instantly share code, notes, and snippets.
Created
November 23, 2012 12:05
-
Star
1
(1)
You must be signed in to star a gist -
Fork
0
(0)
You must be signed in to fork a gist
-
Save norrs/4135334 to your computer and use it in GitHub Desktop.
NAV Developer deploy setup
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/sh | |
| # Script depends on gnu sed and perl. | |
| # Rest of the dependencies gets installed by apt-get. | |
| set -e | |
| YOUR_USER= | |
| SRCDIR= | |
| DEPLOYDIR= | |
| PIP_REQUIREMENTS= | |
| RAPID_DEVELOPMENT= | |
| DISPLAY_HELP= | |
| SKIP_MCC_RUN= | |
| die_with_message() { | |
| echo "$@" | |
| exit 1 | |
| } | |
| display_help() { | |
| echo "nav-dev.sh (Roy Sindre Norangshol aka Rockj)" | |
| echo "https://gist.github.com/norrs/4135334" | |
| echo "" | |
| echo "Required settings:" | |
| echo "-u username to chown deploy directory to" | |
| echo "-s source dir where to find NAV sources" | |
| echo "-d deploy dir where to deploy NAV virtualenv and install" | |
| echo "Optional settings:" | |
| echo "-m Skip mcc.py execution that builds cricket file, faster «redeployments»" | |
| echo "-p use provided pip resources file to deploy a virtualenv with specific/testing libraries" | |
| echo "-r rapid development, symlink media/*, templates and python code from source dir into deploydir" | |
| echo "-h display this help" | |
| echo "" | |
| echo "Important: flags who has no arguments need to come before flags with arguments!" | |
| echo "Example: $0 -m -r -u norangshol -s /source/nav-mirror -d /deploy/nav-mirror" | |
| [ -z "$1" ] | |
| exit $? | |
| } | |
| set -- $(getopt hmru:s:d:p: "$@") | |
| while [ $# -gt 0 ] | |
| do | |
| case "$1" in | |
| (-u) YOUR_USER="$2";shift;; | |
| (-s) SRCDIR=$2;shift;; | |
| (-d) DEPLOYDIR=$2;shift;; | |
| (-p) PIP_REQUIREMENTS=$2;shift;; | |
| (-r) RAPID_DEVELOPMENT=1;; | |
| (-h) DISPLAY_HELP=1;; | |
| (-m) SKIP_MCC_RUN=1;; | |
| (--) shift; break;; | |
| (-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;; | |
| (*) break;; | |
| esac | |
| shift | |
| done | |
| [ -n "$DISPLAY_HELP" ] && display_help | |
| [ -z "$YOUR_USER" ] && display_help 1 | |
| [ -z "$SRCDIR" ] && display_help 1 | |
| [ -z "$DEPLOYDIR" ] && display 1 | |
| echo "Using settings:" | |
| echo "User : $YOUR_USER" | |
| echo "Src : $SRCDIR" | |
| echo "Deploy dir : $DEPLOYDIR" | |
| echo "requirements.txt : $PIP_REQUIREMENTS (blank means default)" | |
| echo "rapid_development: $RAPID_DEVELOPMENT" | |
| setup_virtualenv() { | |
| test -z "$1" && echo dir missing && exit 1 | |
| VIRTENV="$1/.env" | |
| # Create virtualenv for some required python packages, and keep it around to | |
| # avoid hitting the network each time a job runs. | |
| if [ -d "$VIRTENV" ]; then | |
| echo "**> virtualenv exists" | |
| else | |
| echo "**> creating virtualenv" | |
| opt= | |
| test -n "$PYTHON_VERSION" && opt="-p $PYTHON_VERSION" | |
| virtualenv --no-site-packages $opt "$VIRTENV" | |
| fi | |
| } | |
| install_dependencies_and_build() { | |
| test -z "$PGDATABASE" && echo PGDATABASE missing && exit 1 | |
| test -z "$PGUSER" && echo PGUSER missing && exit 1 | |
| test -z "$1" && echo source dir missing && exit 1 | |
| test -z "$2" && echo «deploy» dir missing && exit 1 | |
| SRCDIR="$1" | |
| BUILDDIR="$2/build" | |
| VIRTENV="$2/.env" | |
| # Clear build directory | |
| test -d "$BUILDDIR" && rm -rf "$BUILDDIR" | |
| . "$VIRTENV/bin/activate" | |
| easy_install pip || exit 1 | |
| # hacky hacky fall back to always included cheetah and psycopg2 :-p | |
| pip install cheetah | |
| pip install psycopg2 | |
| [ -z "$PIP_REQUIREMENTS"] && PIP_REQUIREMENTS="$SRCDIR/tests/requirements.txt" | |
| pip install -r "$PIP_REQUIREMENTS" || exit 1 | |
| # Make install code into given directory | |
| ./autogen.sh || exit 1 | |
| ./configure --prefix "$BUILDDIR" || exit 1 | |
| make || exit 1 | |
| make install || exit 1 | |
| export PYTHONPATH="$BUILDDIR/lib/python" | |
| # Update db config | |
| sed -i'' -e "s,^db_nav\s*=\s*nav,db_nav=$PGDATABASE," "$BUILDDIR/etc/db.conf" | |
| sed -i'' -e "s/^script_default\s*=\s*nav/script_default=$PGUSER/" "$BUILDDIR/etc/db.conf" | |
| sed -i'' -e "s/^userpw_nav\s*=.*/userpw_$PGUSER=$PGPASSWORD/" "$BUILDDIR/etc/db.conf" | |
| if [ -n "$PGHOST" ]; then sed -i'' -e "s/^dbhost\s*=\s*localhost/dbhost=$PGHOST/" "$BUILDDIR/etc/db.conf"; fi | |
| if [ -n "$PGPORT" ]; then sed -i'' -e "s/^dbport\s*=.*/dbport=$PGPORT/" "$BUILDDIR/etc/db.conf"; fi | |
| # Remove existing DB, if any, and create new one | |
| #dropdb $PGDATABASE || true | |
| #sql/syncdb.py -c || exit 1 | |
| if [ -n "$ADMINPASSWORD" ]; then psql -c "UPDATE account SET password = '$ADMINPASSWORD' WHERE login = 'admin'"; fi | |
| # Add non-ASCII chars to the admin user's login name to test encoding | |
| # compliance for all Cheetah based web pages. | |
| psql -c "UPDATE account SET name = 'Administrator ÆØÅ' WHERE login = 'admin'" | |
| } | |
| check_python_version() { | |
| [ $# -eq 0 ] && die_with_message "No arguments given for check_python_version, bailing" | |
| VERSION="$1" | |
| [ -z "$VERSION" ] && die_with_message "No arguments given for check_python_version, bailing" | |
| TEST_PYTHON=$($DEPLOYDIR/.env/bin/python -V 2>&1 | grep -e "^Python $VERSION") | |
| [ -n "$TEST_PYTHON" ] | |
| return $? | |
| } | |
| export DEPLOYDIR="$DEPLOYDIR" | |
| if [ ! -f "$SRCDIR/tests/bootstrap-test-environment.sh" ]; then | |
| die_with_message "Are you sure you have NAV sources deployed in $SRCDIR? \$SRCDIR need to be set of root of checkout" | |
| fi | |
| DISTRO_CODENAME=$(lsb_release -c | tr '\t' ' ' | cut -d ' ' -f 2) | |
| if [ $DISTRO_CODENAME = "squeeze" ]; then | |
| apt-get install -t squeeze-backports -y mercurial | |
| else | |
| apt-get install -y mercurial | |
| fi | |
| apt-get install -y subversion | |
| apt-get install -y python-virtualenv build-essential autoconf | |
| apt-get install -y postgresql-client | |
| apt-get build-dep -y psycopg2 | |
| if [ ! -d /etc/nav ]; then | |
| mkdir /etc/nav | |
| echo "Creating /etc/nav" | |
| fi | |
| cd "$SRCDIR" | |
| cat <<DATABASECONFIG > /etc/nav/database | |
| #!/bin/sh | |
| export PGDATABASE=nav | |
| export PGUSER=nav | |
| export PGPASSWORD=nav | |
| export PGHOST=127.0.0.1 | |
| export PGPORT=5432 | |
| DATABASECONFIG | |
| . /etc/nav/database | |
| setup_virtualenv "$DEPLOYDIR" | |
| . "$DEPLOYDIR/.env/bin/activate" | |
| check_python_version 2.6 && [ $? -eq 0 ] && export PYTHON_VERSION=python2.6 | |
| check_python_version 2.7 && [ $? -eq 0 ] && export PYTHON_VERSION=python2.7 | |
| [ -z "$PYTHON_VERSION" ] && die_with_message "Could not detect python version in virtualenv in $DEPLOYDIR" | |
| # Build env should be fine now. | |
| install_dependencies_and_build "$SRCDIR" "$DEPLOYDIR" || exit 1 | |
| getent group nav || addgroup --system nav | |
| getent passwd navcron | |
| if [ $? -ne 0 ]; then | |
| adduser --system --no-create-home --home "$DEPLOYDIR/build" --shell /bin/sh --ingroup nav navcron | |
| else | |
| usermod -d "$DEPLOYDIR/build" navcron | |
| fi | |
| chown -R navcron:nav "$DEPLOYDIR/build/var" | |
| apt-get install -y cricket | |
| # not needed? | |
| #cp -r "$DEPLOYDIR/build/doc/cricket/cricket-config" "$DEPLOYDIR/build/etc/" | |
| chown -R navcron:nav "$DEPLOYDIR/build/etc/cricket-config" | |
| # should do a sed here. but /etc/cricket/cricket-conf.pl should have: | |
| # $gConfigRoot = "/deploy/nav-mirror/build/etc/cricket-config"; | |
| #sed 's/$gConfigRoot.*/$gConfigRoot = \"'"$DEPLOYDIR"'\/build\/etc\/cricket-config\";/g' -i /etc/cricket/cricket-conf.pl | |
| perl -pi -e 's/\$gConfigRoot.*/\$gConfigRoot = \"$ENV{DEPLOYDIR}\/build\/etc\/cricket-config\";/' /etc/cricket/cricket-conf.pl | |
| # confirm with: grep gConfigRoot /etc/cricket/cricket-conf.pl | |
| (grep gConfigRoot /etc/cricket/cricket-conf.pl | grep -o $DEPLOYDIR) | |
| [ ! -d "$DEPLOYDIR/build/var/cricket-data" ] && mkdir "$DEPLOYDIR/build/var/cricket-data" | |
| chown navcron "$DEPLOYDIR/build/var/cricket-data" | |
| su navcron -c cricket-compile | |
| # need to do something tricky here FIRST! :] | |
| # like rm rf python lib and symlink it . not delete whole lbi | |
| rm -rf "$DEPLOYDIR/build/lib/python/nav" | |
| ln -sf "$SRCDIR/python/nav" "$DEPLOYDIR/build/lib/python/nav" | |
| ln -sf "$DEPLOYDIR/build/lib/python/nav" "$DEPLOYDIR/.env/lib/$PYTHON_VERSION/site-packages/" | |
| # IN THE SOURCE ENVIRONMENT! | |
| pip install django==1.2.3 | |
| pip install ipy | |
| [ -z "$SKIP_MCC_RUN" ] && (echo "Running mcc.py, might take some time if your have a few devices already added in DB"; su navcron -s /bin/sh -c ". $DEPLOYDIR/.env/bin/activate && $DEPLOYDIR/build/bin/mcc.py") | |
| mkdir "$DEPLOYDIR/build/share/htdocs/cricket" | |
| cd /usr/share/cricket | |
| ln -sf $PWD/grapher.cgi "$DEPLOYDIR/build/share/htdocs/cricket/" | |
| ln -sf $PWD/mini-graph.cgi "$DEPLOYDIR/build/share/htdocs/cricket/" | |
| cd "$DEPLOYDIR/build/share/htdocs/cricket" | |
| ln -sf grapher.cgi index.cgi | |
| #todo update! | |
| #cp "$DEPLOYDIR/build/doc/cricket/public_html/cricket.css" . | |
| ln -sf /usr/share/cricket/images "$DEPLOYDIR/build/share/htdocs/cricket/" | |
| #Apache2 vhost: | |
| cat <<APACHEVHOST > /etc/apache2/sites-available/navdeploy | |
| <VirtualHost *:80> | |
| ServerAdmin webmaster@localhost | |
| Include /deploy/nav-mirror/build/etc/apache/apache.conf | |
| ErrorLog \${APACHE_LOG_DIR}/error.log | |
| # Possible values include: debug, info, notice, warn, error, crit, | |
| # alert, emerg. | |
| LogLevel warn | |
| CustomLog \${APACHE_LOG_DIR}/access.log combined | |
| </VirtualHost> | |
| APACHEVHOST | |
| apt-get install -y libapache2-mod-wsgi | |
| apt-get install -y libapache2-mod-python | |
| #vim $DEPLOYDIR/build/etc/apache/apache.conf | |
| #Insert: | |
| # requires gnu sed. | |
| sed -i '/PythonPath/d' "$DEPLOYDIR/build/etc/apache/apache.conf" | |
| perl -pi -e "s{^((\s*)PythonHandler django.core.handlers.modpython)}{\1\n\2PythonPath \"\(lambda oldpath:\(__import__\('site'\)\.addsitedir\('$DEPLOYDIR/.env/lib/$PYTHON_VERSION/site-packages'\),filter\( \(lambda x: x not in oldpath\),sys\.path\)\+oldpath\)\)\(list\(sys\.path\)\)\[\-1\]\"}" "$DEPLOYDIR/build/etc/apache/apache.conf" | |
| a2ensite navdeploy | |
| /etc/init.d/apache2 restart | |
| # nav.asyncdns.py | |
| # need to install from tarball due to pip not accepting names with spaces in its repository | |
| # blame pip-tool or twisted guys! | |
| # https://github.com/pypa/pip/issues/307 | |
| pip install twisted | |
| pip install http://twistedmatrix.com/Releases/Names/12.2/TwistedNames-12.2.0.tar.bz2 | |
| # freebsd doesnt have newer ... or yeah meh. | |
| pip install simplejson==2.0.6 | |
| #pip install pysnmp | |
| pip install http://downloads.sourceforge.net/project/twistedsnmp/pysnmp-se/3.5.2/pysnmp-se-3.5.2.tar.gz | |
| pip install networkx | |
| apt-get build-dep -y librrd-dev | |
| apt-get install -y librrd-dev | |
| #pip install rrdtool | |
| pip install python-rrdtool | |
| pip install xmpppy | |
| #PRØV libsnmp15 først! Skal ikke være nødvendig med de 2 neste for pynetsnmp | |
| #apt-get install libsnmp-base | |
| #apt-get install libsnmp-dev | |
| apt-get install -y libsnmp15 | |
| pip install svn+http://dev.zenoss.org/svn/tags/pynetsnmp-0.28.14 | |
| #------------------------------------ | |
| #FINAL LIST | |
| #----------------------------------- | |
| #pip freeze -l | |
| #Cheetah==2.4.4 | |
| #Django==1.2.3 | |
| #IPy==0.75 | |
| #Jinja2==2.6 | |
| #Markdown==2.2.0 | |
| #Pygments==1.5 | |
| #Sphinx==1.1.3 | |
| #Twisted==12.2.0 | |
| #Twisted-Names==12.2.0 | |
| #distribute==0.6.10 | |
| #docutils==0.9.1 | |
| #logilab-astng==0.24.0 | |
| #logilab-common==0.58.1 | |
| #mock==1.0.0 | |
| #networkx==1.7 | |
| #psycopg2==2.4.5 | |
| #py==1.4.9 | |
| #pyasn1==0.1.4 | |
| #pycrypto==2.6 | |
| #pylint==0.25.2 | |
| #pysnmp-se==3.5.2 | |
| #pytest==2.2.4 | |
| #python-rrdtool==1.4.7 | |
| #simplejson==2.0.6 | |
| #unittest2==0.5.1 | |
| #xmpppy==0.5.0rc1 | |
| #zope.interface==4.0.1 | |
| # | |
| #-------- | |
| #ln -sf /nav-mirror/media/ /deploy/nav-mirror/build/lib/ ?? | |
| if [ -n "$RAPID_DEVELOPMENT" ]; then | |
| echo "Checking for making RAPID DEVELOPEMENT symlinks..." | |
| [ ! -L "$DEPLOYDIR/build/share/htdocs/images" ] && rm -rf "$DEPLOYDIR/build/share/htdocs/images" | |
| ln -sf "$SRCDIR/media/images" "$DEPLOYDIR/build/share/htdocs/images" | |
| [ ! -L "$DEPLOYDIR/build/share/htdocs/js" ] && rm -rf "$DEPLOYDIR/build/share/htdocs/js" | |
| ln -sf "$SRCDIR/media/js" "$DEPLOYDIR/build/share/htdocs/js" | |
| [ ! -L "$DEPLOYDIR/build/share/htdocs/style" ] && rm -rf "$DEPLOYDIR/build/share/htdocs/style" | |
| ln -sf "$SRCDIR/media/style" "$DEPLOYDIR/build/share/htdocs/style" | |
| [ ! -L "$DEPLOYDIR/build/lib/templates" ] && rm -rf "$DEPLOYDIR/build/lib/templates" | |
| ln -sf "$SRCDIR/templates" "$DEPLOYDIR/build/lib/templates" | |
| fi | |
| #------------- | |
| #------ | |
| #FOR RUNNING TESTS | |
| pip install minimock | |
| apt-get build-dep -y python-lxml | |
| pip install lxml | |
| # Bør migrere til pytidylib .. og noke magi for å håndtere tidypakken i debian. | |
| #pip install pytidylib | |
| echo "We should be done, navdev" | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
The script has tags, so in case your trying to deploy an old version you could try rolling back the script in case it doesn't work.
(started tagging from NAV version 3.14.15 , PIIIIII ) :-)