RHEL64-based script to build and start 3 Fedora 18 VM's for packstack.

packstack-vms-build.bash

#!/bin/bash
#
# This script is intended to ultimately "just work" on RHEL6, setting
# up a grizzly openstack implementation on three Fedora 18 guest VM's
# using the packstack installer.  It currently does not work due to a
# dependency error with openstack-nova-conductor not being available.
#
# It creates 3 Fedora-18 VM's for packstack/openstack-grizzly.  Each
# has two nic's. The first nic is connected to the virt default
# network. The second is connected to the virt openstackvms network
# (created by this script, see /tmp/openstackvms.xml) and corresponds
# to the packstack CONFIG_NOVA_NETWORK_FIXEDRANGE= config value.
#
# It exports an nfs share on /mnt/vm-share to place root's authorized
# keys, needed by the three VM's. In addition, the three hosts write
# their IP default-network addresses to .hello files in /mnt/vm-share
# after they boot up.
#
# For convenience, the host OS's /etc/hosts is updated with the IP
# addresses of the guests, so the user may just ssh into packstack1,
# packstack2 or packstack3.  Note that the packstack installer
# requires IP addresses and not hostnames, so we stick with IP
# addresses for the installer.
#
# Notable tweaks/workarounds in script (with nods to jistr and pblaho):
#
#  * Disabling selinux on the f18 hosts
#  * Updating boot parameters of f18 hosts with "noapic
#    ipv6.disable=1" The noapic is needed for cinder volume creation
#    to work.  The ipv6 disabling is for sanity.
#  * The kickstart intentionally does not use entire (qcow2) disk, so
#    that packstack/cinder may succeed in creating a physical volume
#    and volume group.
#  * Updates packstack answer.txt file to use a 1GB cinder volume
#    instead of 20GB.
#  * Don't copy over packstack's create_resources module, which
#    packstack wants to do since it makes the assumption the guest
#    OS'es are Fedora-or-not-Fedora based on the host OS (and it would
#    copy it over correctly *if* our guests were RHEL6).  The puppet
#    mysql module would fail without this workaround.
#
# VM Notes:
#
#  * Snapshots are created after the hosts are prepped but before they
#    packstack'ed to allow for retrying packstack without too much
#    difficulty.  To revert to the snapshot for a particular guest
#    (though it would really only make sense to do it for all three):
#
#      sudo sh -c 'ssh packstack1 "shutdown -h now"'
#      # shouldn't be needed, but make sure it's dead
#      sudo virsh destroy packstack1  
#      sudo qemu-img snapshot -a before_pack /var/lib/libvirt/images/packstack1.qcow2
#      sudo virsh start packstack1
#      # only needed if you ssh'ed into one of the hosts
#      rm -f /root/.ssh/known_hosts
#
# Debugging Tips:
#
# * Comment out remove_remote_var_dirs() in
#   /packstack/packstack/installer/run_setup.py to easily be able to retry puppet commands by hand
# * Add --debug after "( flock %s/ps.lock puppet apply " in
#   packstack/packstack/plugins/puppet_950.py
# * Of course, add --debug to the packstack command itself
#
# TODO:
#
# * Make the packstack install work.
#   * openstack-nova-conductor dependency error (not found).
#   * sometimes have to rerun packstack for tgtd.service to come up
# * Detect when packstack kickstart has completed rather than ask for
#   user input
#

#sudo yum install -y --nogpgcheck http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm

depends="nfs-utils libguestfs-tools libvirt virt-manager git mysql-server tigervnc-server tigervnc-server-module tigervnc xorg-x11-twm xorg-x11-server-utils"

for dep in $depends; do
  if ! `rpm -q --quiet --nodigest $dep`; then
    install_list="$install_list $dep"
  fi
done
# Install the needed packages
if [ "x$install_list" != "x" ]; then
  sudo yum install -y $install_list
fi
# Verify the dependencies did install
fail_list=""
for dep in $depends; do
  if ! `rpm -q --quiet --nodigest $dep`; then
    fail_list="$fail_list $dep"
  fi
done
# If anything failed verification, we tell the user and exit
if [ "x$fail_list" != "x" ]; then
  echo "ABORTING:  FAILED TO INSTALL $fail_list"
  exit 1
fi

sudo /sbin/iptables --flush
sudo sysctl -w net.ipv4.ip_forward=1
sudo sed -i 's/net.ipv4.ip_forward = 0/net.ipv4.ip_forward = 1/g' /etc/sysctl.conf 
sudo setenforce 0

sudo /sbin/service libvirtd start

cat >/tmp/openstackvms.xml <<EOF
<network>
  <name>openstackvms</name>
  <bridge name="virbr1" />
  <ip address="192.168.32.1" netmask="255.255.255.0">
    <dhcp>
      <range start="192.168.32.2" end="192.168.32.254" />
    </dhcp>
  </ip>
</network>
EOF

sudo virsh net-define /tmp/openstackvms.xml
sudo virsh net-start openstackvms

# this shouldn't be necessary, but i've seen issues on rhel6...
sudo virsh net-define /usr/share/libvirt/networks/default.xml
sudo virsh net-start default

sudo sh -c "if [ ! -f /root/.ssh/id_rsa.pub ]; then ssh-keygen -t rsa -N '' -f /root/.ssh/id_rsa ; fi"

sudo mkdir -p /mnt/vm-share
sudo chmod ugo+rwx /mnt/vm-share;
if ! `grep -q vm-share /etc/exports`; then
  sudo sh -c "echo '/mnt/vm-share 192.168.0.0/16(rw,sync,no_root_squash)' >> /etc/exports"
  sudo /sbin/service nfs restart
fi

sudo cp -f /root/.ssh/id_rsa.pub /mnt/vm-share/authorized_keys
sudo chmod ugo+r /mnt/vm-share/authorized_keys

################################################################
# kickstart def

cat >/tmp/f18-packstack.ks <<EOD
%packages
@standard
nfs-utils
%end

install
text
reboot
lang en_US.UTF-8
keyboard us
#Choose a saner password here.
rootpw packstack
firewall --disabled
selinux --disabled
timezone --utc America/New_York
#firstboot --disable
ignoredisk --only-use=vda
bootloader --location=mbr --append="console=tty0 console=ttyS0,115200 rd_NO_PLYMOUTH" --boot-drive=vda
clearpart --initlabel --drives=vda
part /boot --fstype ext4 --size=100
part swap --size=100
part pv.01 --size=5000
volgroup lv_admin --pesize=32768 pv.01
logvol / --fstype ext4 --name=lv_root --vgname=lv_admin --size=4000 --grow
zerombr
network --bootproto=dhcp --noipv6

%post

/usr/bin/yum update -y --skip-broken
/usr/bin/yum install -y net-tools git emacs-nox screen zlib-devel gcc gcc-c++ sqlite-devel tar bzip2 readline-devel openssl-devel patch
mkdir -p /mnt/vm-share
mount 192.168.122.1:/mnt/vm-share /mnt/vm-share
if [ ! -d /root/.ssh ]; then
  mkdir -p /root/.ssh
  chmod 700 /root/.ssh
fi
if [ -f /mnt/vm-share/authorized_keys ]; then
  cp /mnt/vm-share/authorized_keys /root/.ssh/authorized_keys
  chmod 0600 /root/.ssh/authorized_keys
fi

%end
EOD

################################################################
### create the first image

sudo /usr/bin/qemu-img create -f qcow2 -o preallocation=metadata /var/lib/libvirt/images/packstack1.qcow2 8G

sudo virt-install --connect=qemu:///system \
    --network network:default \
    --network network:openstackvms \
    --initrd-inject=/tmp/f18-packstack.ks \
    --extra-args="ks=file:/f18-packstack.ks" \
    --name=packstack1 \
    --location=http://download.fedoraproject.org/pub/fedora/linux/releases/18/Fedora/x86_64/os \
    --disk /var/lib/libvirt/images/packstack1.qcow2,format=qcow2 \
    --ram 1600 \
    --vcpus=1 \
    --autostart \
    --os-variant virtio26 \
    --vnc
#    --location=/$HOME/Fedora-18-x86_64-DVD.iso \

echo "view the install with:"
echo "   virt-viewer --connect qemu+ssh://root@`hostname`/system packstack1"
echo "press enter when install is complete."
echo "note that packstack1 MUST NOT BE RUNNING before continuing"
read

################################################################
### create the other images

sudo virt-clone -o packstack1 -n packstack2 -f /var/lib/libvirt/images/packstack2.qcow2
sudo virt-sysprep -d packstack2

sudo virt-clone -o packstack1 -n packstack3 -f /var/lib/libvirt/images/packstack3.qcow2
sudo virt-sysprep -d packstack3

## TODO:  DRY this.
## TODO:  wait on pid after unmounting (see http://libguestfs.org/guestmount.1.html)
sudo mkdir -p /mnt/guest1
# when the host boots up, write a "we're here" file to /mnt/vm-share
sudo guestmount -a /var/lib/libvirt/images/packstack1.qcow2 -i /mnt/guest1
echo '#!/bin/bash
mount /mnt/vm-share
firewall-cmd --permanent --zone=public --add-port=80/tcp' > /tmp/packstack1.rc.local
echo 'echo `ifconfig eth0 | grep "inet " | perl -p -e "s/.*inet .*?(\d\S+\d).*\\\$/\\\$1/"` packstack1 > /mnt/vm-share/packstack1.hello' >> /tmp/packstack1.rc.local
sudo cp /tmp/packstack1.rc.local /mnt/guest1/etc/rc.d/rc.local
sudo chmod ugo+x /mnt/guest1/etc/rc.d/rc.local
# disable selinux if the kickstart did not
sudo sh -c "echo 'SELINUX=disabled' > /mnt/guest1/etc/selinux/config"
# always mount /mnt/vm-share
sudo sh -c "echo '192.168.122.1:/mnt/vm-share /mnt/vm-share nfs defaults 0 0' >> /mnt/guest1/etc/fstab"
# noapic
sudo sh -c "perl -p -i -e 's/^(\s*linux\s+.*)\$/\$1 noapic ipv6.disable=1/' /mnt/guest1/boot/grub2/grub.cfg"
sleep 2
sudo umount /mnt/guest1

sudo mkdir -p /mnt/guest2
# when the host boots up, write a "we're here" file to /mnt/vm-share
sudo guestmount -a /var/lib/libvirt/images/packstack2.qcow2 -i /mnt/guest2
echo '#!/bin/bash
mount /mnt/vm-share
firewall-cmd --permanent --zone=public --add-port=80/tcp' > /tmp/packstack2.rc.local
echo 'echo `ifconfig eth0 | grep "inet " | perl -p -e "s/.*inet .*?(\d\S+\d).*\\\$/\\\$1/"` packstack2 > /mnt/vm-share/packstack2.hello' >> /tmp/packstack2.rc.local
sudo cp /tmp/packstack2.rc.local /mnt/guest2/etc/rc.d/rc.local
sudo chmod ugo+x /mnt/guest2/etc/rc.d/rc.local
# disable selinux if the kickstart did not
sudo sh -c "echo 'SELINUX=disabled' > /mnt/guest2/etc/selinux/config"
# always mount /mnt/vm-share
sudo sh -c "echo '192.168.122.1:/mnt/vm-share /mnt/vm-share nfs defaults 0 0' >> /mnt/guest2/etc/fstab"
# noapic
sudo sh -c "perl -p -i -e 's/^(\s*linux\s+.*)\$/\$1 noapic ipv6.disable=1/' /mnt/guest2/boot/grub2/grub.cfg"
sleep 2
sudo umount /mnt/guest2

sudo mkdir -p /mnt/guest3
# when the host boots up, write a "we're here" file to /mnt/vm-share
sudo guestmount -a /var/lib/libvirt/images/packstack3.qcow2 -i /mnt/guest3
echo '#!/bin/bash
mount /mnt/vm-share
firewall-cmd --permanent --zone=public --add-port=80/tcp' > /tmp/packstack3.rc.local
echo 'echo `ifconfig eth0 | grep "inet " | perl -p -e "s/.*inet .*?(\d\S+\d).*\\\$/\\\$1/"` packstack3 > /mnt/vm-share/packstack3.hello' >> /tmp/packstack3.rc.local
sudo cp /tmp/packstack3.rc.local /mnt/guest3/etc/rc.d/rc.local
sudo chmod ugo+x /mnt/guest3/etc/rc.d/rc.local
# disable selinux if the kickstart did not
sudo sh -c "echo 'SELINUX=disabled' > /mnt/guest3/etc/selinux/config"
# always mount /mnt/vm-share
sudo sh -c "echo '192.168.122.1:/mnt/vm-share /mnt/vm-share nfs defaults 0 0' >> /mnt/guest3/etc/fstab"
# noapic
sudo sh -c "perl -p -i -e 's/^(\s*linux\s+.*)\$/\$1 noapic ipv6.disable=1/' /mnt/guest3/boot/grub2/grub.cfg"
sleep 2
sudo umount /mnt/guest3
sleep 45

sudo qemu-img snapshot -c before_pack /var/lib/libvirt/images/packstack1.qcow2
sudo qemu-img snapshot -c before_pack /var/lib/libvirt/images/packstack2.qcow2
sudo qemu-img snapshot -c before_pack /var/lib/libvirt/images/packstack3.qcow2

sudo virsh start packstack1
sleep 45
sudo virsh start packstack2
sleep 45
sudo virsh start packstack3

################################################################
### get packstack

cd $HOME
if [ -d packstack ]; then
  echo 'packstack dir already exists, moving it out of the way'
  mv packstack packstack.$$.sav
fi
git clone --recursive https://github.com/stackforge/packstack.git

# don't let packstack copy over the create_resources module,
# which is only needed for older ruby/puppet.
# only needed because our host OS is not Fedora, which breaks
# a packstack assumption.  (see puppet_950.py)
cd packstack
perl -p -i -e "s/'create_resources',//" packstack/plugins/puppet_950.py

###############################################################
### add the packstack hosts to /etc/hosts

ATTEMPTS=30
FAILED=0
count=`ls /mnt/vm-share/packstack*.hello | wc -w`
while [ $count -lt 3  ]; do
  FAILED=$(expr $FAILED + 1)
  echo $FAILED
  if [ $FAILED -ge $ATTEMPTS ]; then
    echo "VM(s) did not write their IP info in /mnt/vm-share/packstack*.hello"
    echo "Manual investigation is required"
    exit 1
  fi
  sleep 10
  count=`ls /mnt/vm-share/packstack*.hello | wc -w`
done

# These host names don't buy us much since the packstack installer
# demands IP's, but maybe nice for humans
for packhost in packstack1 packstack2 packstack3; do
  packhost_line=$(cat /mnt/vm-share/$packhost.hello)
  if $(grep -qs $packhost /etc/hosts) ;then
    sudo perl -p -i -e "s|.*\b$packhost\b.*\$|$packhost_line|" /etc/hosts
  else
    sudo sh -c "cat /mnt/vm-share/$packhost.hello >> /etc/hosts"
  fi
done

# do the same thing on the vm's (poor man's DNS!)
#for sshhost in packstack1 packstack2 packstack3; do
#sudo ssh -o StrictHostKeyChecking=no $sshhost '
#for packhost in packstack1 packstack2 packstack3; do
#  packhost_line=$(cat /mnt/vm-share/$packhost.hello)
#  if $(grep -qs $packhost /etc/hosts) ;then
#    perl -p -i -e "s|.*\b$packhost\b.*\$|$packhost_line|" /etc/hosts
#  else
#    sh -c "cat /mnt/vm-share/$packhost.hello >> /etc/hosts"
#  fi
#done'
#done

################################################################
### generate and edit answer file

# we need to use ip addresses instead of host names
packstack1_ip=$(ping -q -c 1 -t 1 packstack1 | grep PING | perl -p -e 's/^.*?\((\d\S+)\).*$/$1/')
packstack2_ip=$(ping -q -c 1 -t 1 packstack2 | grep PING | perl -p -e 's/^.*?\((\d\S+)\).*$/$1/')
packstack3_ip=$(ping -q -c 1 -t 1 packstack3 | grep PING | perl -p -e 's/^.*?\((\d\S+)\).*$/$1/')

cd $HOME/packstack
python bin/packstack --gen-answer-file=ans.txt
perl -p -i -e 's|^CONFIG_SSH_KEY=$|CONFIG_SSH_KEY=/mnt/vm-share/authorized_keys|' ans.txt
perl -p -i -e 's|^CONFIG_CINDER_VOLUMES_SIZE=.*$|CONFIG_CINDER_VOLUMES_SIZE=1G|' ans.txt
perl -p -i -e "s/^(.*HOSTS?)=.*$/\$1=$packstack1_ip/" ans.txt
perl -p -i -e "s/^CONFIG_NOVA_COMPUTE_HOSTS=.*$/CONFIG_NOVA_COMPUTE_HOSTS=$packstack2_ip,$packstack3_ip/" ans.txt

sudo ./bin/packstack --answer-file=ans.txt