Fabric everywhere

CERN_OpenStack.py

import StringIO
from fabric.api import *

env.hosts = [""]

krbconf = StringIO.StringIO("""
[libdefaults]
    default_realm = CERN.CH
    ticket_lifetime = 24h
    renew_lifetime = 7d
    forwardable = true
 
[realms]
    CERN.CH = {
        default_domain = cern.ch
        kdc = cerndc.cern.ch
    }
 
[domain_realm]
    cern.ch = CERN.CH
    .cern.ch = CERN.CH """)

def setup_afs(user="kostams"):
    packages = [
        "krb5-workstation", "krb5-libs", "krb5-auth-dialog",
        "openafs-client",  "openafs-server", "openafs-krb5",
        "kernel-module-openafs", "compat-readline5.x86_64",
        "openssl098e.x86_64",  "fuse-libs.x86_64"]

    run("yum -y install " + " ".join(packages))

    put(krbconf, "/etc/krb5.conf")

    run("kinit {0}@CERN.CH".format(user))
    with settings(warn_only=True):
       run("service afs start")
    run("aklog")


cvmfs = StringIO.StringIO("""
    CVMFS_HTTP_PROXY=\"http://ca-proxy.cern.ch:3128;http://ca-proxy1.cern.ch:3128|http://ca-proxy2.cern.ch:3128|http://ca-proxy3.cern.ch:3128|http://ca-proxy4.cern.ch:3128|http://ca-proxy5.cern.ch:3128\"
    CVMFS_REPOSITORIES=ams
""")

def setup_cvmfs():
    with  cd("/etc/yum.repos.d/"):
        run("wget http://cvmrepo.web.cern.ch/cvmrepo/yum/cernvm.repo")

    with cd("/etc/pki/rpm-gpg/"):
        run("wget http://cvmrepo.web.cern.ch/cvmrepo/yum/RPM-GPG-KEY-CernVM")

    run("yum -y install cvmfs.x86_64")

    put(cvmfs, "/etc/cvmfs/default.local")
    put(StringIO.StringIO(
        'CVMFS_SERVER_URL="http://cvmfs-stratum-one.cern.ch/opt/@org@"'
        ), "/etc/cvmfs/config.d/ams.cern.ch.local"
    )
    run("service autofs start")    
    run("cvmfs_config setup")
    run("cvmfs_config probe")


def mount_all(user="kostams"):
    afshome = '/afs/cern.ch/user/{0}/{1}'.format(user[0],user)
    run("kinit {0}@CERN.CH".format(user))
    run("aklog")
    run("GROUP=va /afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select -b fuse mount {0}/eos".format(afshome))
    run("if ! grep -qs '/dev/vda3' /proc/mounts; then mount /dev/vda3 /data; fi")

def create_partition():
    fdisk = run("fdisk -l -u /dev/vda")
    N, last, ptype = None, None, None
    for l in fdisk.split('\n'):
        if not l.startswith("/dev/vda"):
            continue
        pinfo = l.split()
        if pinfo[1] == '*': del pinfo[1]
        N, last, ptype = int(pinfo[0][-1]), int(pinfo[2]), pinfo[4]
        print "{0}: Found partition '{1}'".format(N,pinfo[0])
        print " -- last block {}".format(last)
        print " -- type {}".format(ptype)

    with settings(warn_only=True):
        run("echo -e 'u\nn\np\n{0}\n{1}\n\nt\n{0}\n{2}\nw\n' | fdisk /dev/vda".format(N+1,last+1,ptype))
    reboot()
    run("mkfs.ext4 /dev/vda{}".format(N+1))

fabric.sh

export BASEDIR=$PWD
export INSTALLDIR=$PWD/install
export DISTDIR=$PWD/distrib

mkdir -p $INSTALLDIR
mkdir -p $DISTDIR

cd $DISTDIR 
wget --no-check-certificate https://www.python.org/ftp/python/2.7.9/Python-2.7.9.tgz
wget --no-check-certificate https://pypi.python.org/packages/source/v/virtualenv/virtualenv-12.0.tar.gz

tar xvvf virtualenv-*.tar.gz
tar xvvf Python-*.tgz

cd $DISTDIR/Python-*/    
./configure CXXFLAGS="-I$INSTALLDIR/include" --prefix=$INSTALLDIR   --enable-shared --enable-unicode=ucs4
make && make install

cd $DISTDIR/virtualenv-*/
LD_LIBRARY_PATH=$INSTALLDIR/lib/ $INSTALLDIR/bin/python setup.py install --prefix=$INSTALLDIR

cd $BASEDIR
LD_LIBRARY_PATH=$INSTALLDIR/lib/ $INSTALLDIR/bin/virtualenv --python=$INSTALLDIR/bin/python fabric

cat > env.sh << "EOF"
MY_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source $MY_PATH/fabric/bin/activate
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MY_PATH/install/lib
EOF

. env.sh
pip install fabric

VirtualEnv.py

import StringIO
from fabric.api import *

BASEDIR    = "VirtualEnv"
DISTDIR    = BASEDIR + "/distribs"
INSTALLDIR = BASEDIR + "/install"

env.hosts = [
#    "root@kanishev-ams-vm",
    "root@kanishev-ams-vm1",
#    "root@kanishev-ams-vm2",
#    "root@kanishev-ams-vm3",
]

envsh = StringIO.StringIO("""
MY_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source $MY_PATH/install/bin/thisroot.sh
source $MY_PATH/ipython/bin/activate
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MY_PATH/install/lib
""")

def yum():
    packages = [
        "openssl-devel", "xorg-x11-server-devel", "gfortran",
        "gcc-c++", "gcc", "binutils", "libX11-devel", 
        "libXpm-devel", "libXft-devel", "libXext-devel"]
        
    run("yum -y install " + " ".join(packages))


def get_sources():
    packages = [
        "https://sqlite.org/2014/sqlite-autoconf-3080403.tar.gz",
        "http://download.zeromq.org/zeromq-4.0.3.tar.gz",
        "https://www.python.org/ftp/python/2.7.6/Python-2.7.6.tgz",
        "https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.9.tar.gz",
        "ftp://root.cern.ch/root/root_v5.34.18.source.tar.gz"
    ]
    run("mkdir -p " + DISTDIR)
    with cd(DISTDIR):
        for p in packages:
            run("wget " + p)
        for p in packages:
            run("tar xvvf " + p.split("/")[-1])


def compile():
    run("mkdir -p " + INSTALLDIR)
    with cd(DISTDIR + "/sqlite-autoconf-*"):
        run("./configure --prefix=$HOME/" + INSTALLDIR)
        run("make && make install")
 
    with cd(DISTDIR + "/zeromq-*"):
        run("./configure --prefix=$HOME/" + INSTALLDIR)
        run("make && make install")

    with cd(DISTDIR + "/Python-2.7.6"):
        run('./configure CXXFLAGS="-I$HOME/{0}/include" --prefix=$HOME/{0}   --enable-shared --enable-unicode=ucs4'.format(INSTALLDIR))
        run("make && make install")

    with cd("$HOME/{0}/lib".format(INSTALLDIR)):
        run("ln -s libpython2.7.so libpython.so")

    with cd("$HOME/{0}/root".format(DISTDIR)):
        version = "4.1.1"
        instdir = "$HOME/{0}/opt".format(INSTALLDIR)

        run("mkdir -p " + instdir)
        run("./build/unix/installXrootd.sh -v {0} $HOME/{1}/opt".format(version,INSTALLDIR)
        
        confcommand = [ 
            "./configure",
            "--with-python-incdir=$HOME/{0}/include/python2.7/".format(INSTALLDIR),
            "--with-python-libdir=$HOME/{0}/lib".format(INSTALLDIR),
            "--prefix=$HOME/{0}".format(INSTALLDIR),
            "--etcdir=$HOME/{0}/etc".format(INSTALLDIR),
            "--with-xrootd-incdir=$HOME/{0}/opt/xrootd-{1}/include/xrootd".format(INSTALLDIR, version),
            "--with-xrootd-libdir=$HOME/{0}/opt/xrootd-{1}/lib".format(INSTALLDIR, version) ]
        run(" ".join(confcommand))

        run("make && make install")

def install_numpy():
    with cd(DISTDIR):
        run("git clone git://github.com/xianyi/OpenBLAS")
        with("cd OpenBLAS"):
            run("make FC=gfortran")
            run("make PREFIX=$HOME/{0} install".format(INSTALLDIR))
 
    with cd(DISTDIR):
        run("mkdir -p numpy")
        run("pip install -d numpy numpy")
        with cd("numpy"):
            run("tar xvvf numpy-*.tar.gz")
            with cd("numpy-*"):
                put(StringIO.StringIO("""
                    [default]
                    library_dirs= $INSTALLDIR/lib/
                    [openblas]
                    libraries = openblas
                    library_dirs = $INSTALLDIR/lib
                    include_dirs = $INSTALLDIR/include"""), "site.cfg")
                run("python setup.py install")
                run("pip install scipy")

def prepare_venv():
    with cd("$HOME/{0}".format(BASEDIR)):
        run("cp $HOME/{0}/virtualenv-1.9/virtualenv.py .".format(DISTDIR))
        run("LD_LIBRARY_PATH=$HOME/{0}/lib/ $HOME/{0}/bin/python virtualenv.py --python=$HOME/{0}/bin/python ipython".format(INSTALLDIR))

        put(envsh, "env.sh")

        run('source env.sh && pip install pyzmq --install-option="--zmq=$HOME/{0}"'.format(INSTALLDIR))
        run('CPPFLAGS="-I$HOME/{0}/include" pip install pysqlite'.format(INSTALLDIR))
         
        run('pip install ipython[all]')