Skip to content

Instantly share code, notes, and snippets.

@evrardjp

evrardjp/cobbler-osa-multi-aio.sh

Forked from cloudnull/cobbler-osa-multi-aio.sh
Created June 14, 2016 14:46
Show Gist options
  • Save evrardjp/b2924162b4601d9ff65326ede7272301 to your computer and use it in GitHub Desktop.
Save evrardjp/b2924162b4601d9ff65326ede7272301 to your computer and use it in GitHub Desktop.
Download ZIP
Full Multi-Node OpenStack deployment using a single OnMetal host from the Rackspace Public Cloud. Read more in the "About this script" comment.
Raw
cobbler-osa-multi-aio.sh
#!/usr/bin/env bash
# Copyright [2016] [Kevin Carter]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# About this script:
# Full OpenStack deployment using a single OnMetal host from the
# Rackspace Public Cloud. This is a multi-node installation using
# VMs that have been PXE booted which was done to provide an environment
# that is almost exactly what is in production. This script will build, kick
# and deploy OpenStack using KVM, Cobbler, OpenStack-Ansible within 9 Nodes
# and 1 load balancer all using a Hyper Converged environment.
# This script assumes that the environment is a using the following details:
# FLAVOR: onmetal-io1
# IMAGE: OnMetal - Ubuntu 14.04 LTS (Trusty Tahr)
# More about this script
# Once deployed you can use virt-manager to manage the KVM instances on the host, similar to a drac or ilo.
# LINUX:
# If you're running a linux system as your workstation simply install virt-manager
# from your package manager and connect to the host via QEMU/KVM:SSH
# OSX:
# If you're running a MAC you can get virt-manager via X11 forwarding to the host
# or install it via BREW. http://stackoverflow.com/questions/3921814/is-there-a-virt-manager-alternative-for-mac-os-x
# WINDOWS:
# If you're running Windows, you can install virt-viewer from the KVM Download site.
# https://virt-manager.org/download/
if [ ! -f "/root/.ssh/id_rsa" ];then
ssh-keygen -t rsa -N ''
fi
cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys
apt-get update && apt-get install -y qemu-kvm libvirt-bin virtinst bridge-utils virt-manager
virsh net-autostart default --disable
virsh net-destroy default
if ! grep "^source.*cfg$" /etc/network/interfaces; then
echo 'source /etc/network/interfaces.d/*.cfg' | tee -a /etc/network/interfaces
fi
# create kvm bridges
cat > /etc/network/interfaces.d/kvm-bridges.cfg <<EOF
auto br-dhcp
iface br-dhcp inet static
bridge_stp off
bridge_waitport 0
bridge_fd 0
bridge_ports none
address 10.0.0.200
netmask 255.255.255.0
offload-sg off
auto br-mgmt
iface br-mgmt inet static
bridge_stp off
bridge_waitport 0
bridge_fd 0
bridge_ports none
address 172.29.236.200
netmask 255.255.252.0
offload-sg off
auto br-vxlan
iface br-vxlan inet static
bridge_stp off
bridge_waitport 0
bridge_fd 0
bridge_ports none
address 172.29.240.200
netmask 255.255.252.0
offload-sg off
auto br-storage
iface br-storage inet static
bridge_stp off
bridge_waitport 0
bridge_fd 0
bridge_ports none
address 172.29.244.200
netmask 255.255.252.0
offload-sg off
auto br-vlan
iface br-vlan inet static
bridge_stp off
bridge_waitport 0
bridge_fd 0
bridge_ports none
address 172.29.248.200
netmask 255.255.252.0
offload-sg off
EOF
for i in br-dhcp br-mgmt br-vlan br-storage br-vxlan; do
ifup $i;
done
sysctl -w net.ipv4.ip_forward=1 | tee -a /etc/sysctl.conf
# Add rules from the INPUT chain
iptables -w -I INPUT -i "br-dhcp" -p udp --dport 67 -j ACCEPT
iptables -w -I INPUT -i "br-dhcp" -p tcp --dport 67 -j ACCEPT
iptables -w -I INPUT -i "br-dhcp" -p udp --dport 53 -j ACCEPT
iptables -w -I INPUT -i "br-dhcp" -p tcp --dport 53 -j ACCEPT
# Add rules from the FORWARDING chain
iptables -w -I FORWARD -i "br-dhcp" -j ACCEPT
iptables -w -I FORWARD -o "br-dhcp" -j ACCEPT
# Add rules from the nat POSTROUTING chain
iptables -w -t nat \
-A POSTROUTING \
-s "10.0.0.0/24" ! \
-d "10.0.0.0/24" \
-j MASQUERADE
# Add rules from the mangle POSTROUTING chain
iptables -w -t mangle \
-A POSTROUTING \
-s "10.0.0.0/24" \
-o "br-dhcp" \
-p udp \
-m udp \
--dport 68 \
-j CHECKSUM \
--checksum-fill
# Create kvm storage pool
parted --script /dev/sdb mklabel gpt
parted --script /dev/sdb mkpart kvm ext4 0% 100%
mkfs.ext4 /dev/sdb1
echo "$(blkid | awk '/\/dev\/sdb1/ {print $2}') /var/lib/libvirt/images ext4 errors=remount-ro 0 1" | tee -a /etc/fstab
mount -a
# Install cobbler
wget -qO - http://download.opensuse.org/repositories/home:/libertas-ict:/cobbler26/xUbuntu_14.04/Release.key | apt-key add -
add-apt-repository "deb http://download.opensuse.org/repositories/home:/libertas-ict:/cobbler26/xUbuntu_14.04/ ./"
apt-get update && apt-get -y install cobbler dhcp3-server debmirror isc-dhcp-server ipcalc tftpd tftp fence-agents
# Move Cobbler Apache config to the right place
cp /etc/apache2/conf.d/cobbler.conf /etc/apache2/conf-available/
cp /etc/apache2/conf.d/cobbler_web.conf /etc/apache2/conf-available/
# Fix Apache conf to match 2.4 configuration
sed -i "/Order allow,deny/d" /etc/apache2/conf-enabled/cobbler*.conf
sed -i "s/Allow from all/Require all granted/" /etc/apache2/conf-enabled/cobbler*.conf
sed -i "s/^Listen 80/Listen 5150/" /etc/apache2/ports.conf
sed -i "s/\:80/\:5150/" /etc/apache2/sites-available/000-default.conf
# Enable the above config
a2enconf cobbler cobbler_web
# Enable Proxy modules
a2enmod proxy
a2enmod proxy_http
# Basic cobbler setup
sed -i 's/^manage_dhcp\:.*/manage_dhcp\: 1/g' /etc/cobbler/settings
sed -i 's/^restart_dhcp\:.*/restart_dhcp\: 1/g' /etc/cobbler/settings
sed -i 's/^next_server\:.*/next_server\: 10.0.0.200/g' /etc/cobbler/settings
sed -i 's/^server\:.*/server\: 10.0.0.200/g' /etc/cobbler/settings
sed -i 's/^http_port\:.*/http_port\: 5150/g' /etc/cobbler/settings
sed -i 's/^INTERFACES.*/INTERFACES="br-dhcp"/g' /etc/default/isc-dhcp-server
# Fix TFTP server arguments in cobbler template to enable it to work on Ubuntu
sed -i "s/server_args .*/server_args = -s \$args/" /etc/cobbler/tftpd.template
# Permission Workarounds
mkdir -p /tftpboot
chown www-data /var/lib/cobbler/webui_sessions
# when templated replace \$ with $
cat > /etc/cobbler/dhcp.template <<EOF
# ******************************************************************
# Cobbler managed dhcpd.conf file
#
# generated from cobbler dhcp.conf template (\$date)
# Do NOT make changes to /etc/dhcpd.conf. Instead, make your changes
# in /etc/cobbler/dhcp.template, as /etc/dhcpd.conf will be
# overwritten.
#
# ******************************************************************
ddns-update-style interim;
allow booting;
allow bootp;
ignore client-updates;
set vendorclass = option vendor-class-identifier;
option pxe-system-type code 93 = unsigned integer 16;
subnet 10.0.0.0 netmask 255.255.255.0 {
option routers 10.0.0.200;
option domain-name-servers 8.8.8.8;
option subnet-mask 255.255.255.0;
range dynamic-bootp 10.0.0.1 10.0.0.199;
default-lease-time 21600;
max-lease-time 43200;
next-server \$next_server;
class "pxeclients" {
match if substring (option vendor-class-identifier, 0, 9) = "PXEClient";
if option pxe-system-type = 00:02 {
filename "ia64/elilo.efi";
} else if option pxe-system-type = 00:06 {
filename "grub/grub-x86.efi";
} else if option pxe-system-type = 00:07 {
filename "grub/grub-x86_64.efi";
} else {
filename "pxelinux.0";
}
}
}
#for dhcp_tag in \$dhcp_tags.keys():
## group could be subnet if your dhcp tags line up with your subnets
## or really any valid dhcpd.conf construct ... if you only use the
## default dhcp tag in cobbler, the group block can be deleted for a
## flat configuration
# group for Cobbler DHCP tag: \$dhcp_tag
group {
#for mac in \$dhcp_tags[\$dhcp_tag].keys():
#set iface = \$dhcp_tags[\$dhcp_tag][\$mac]
host \$iface.name {
hardware ethernet \$mac;
#if \$iface.ip_address:
fixed-address \$iface.ip_address;
#end if
#if \$iface.hostname:
option host-name "\$iface.hostname";
#end if
#if \$iface.netmask:
option subnet-mask \$iface.netmask;
#end if
#if \$iface.gateway:
option routers \$iface.gateway;
#end if
#if \$iface.enable_gpxe:
if exists user-class and option user-class = "gPXE" {
filename "http://\$cobbler_server/cblr/svc/op/gpxe/system/\$iface.owner";
} else if exists user-class and option user-class = "iPXE" {
filename "http://\$cobbler_server/cblr/svc/op/gpxe/system/\$iface.owner";
} else {
filename "undionly.kpxe";
}
#else
filename "\$iface.filename";
#end if
## Cobbler defaults to \$next_server, but some users
## may like to use \$iface.system.server for proxied setups
next-server \$next_server;
## next-server \$iface.next_server;
}
#end for
}
#end for
EOF
cat > /var/www/html/trusty-sources.list <<EOF
deb http://archive.ubuntu.com/ubuntu trusty main restricted
deb-src http://archive.ubuntu.com/ubuntu trusty main restricted
deb http://archive.ubuntu.com/ubuntu trusty-updates main restricted
deb-src http://archive.ubuntu.com/ubuntu trusty-updates main restricted
deb http://archive.ubuntu.com/ubuntu trusty universe
deb-src http://archive.ubuntu.com/ubuntu trusty universe
deb http://archive.ubuntu.com/ubuntu trusty-updates universe
deb-src http://archive.ubuntu.com/ubuntu trusty-updates universe
deb http://archive.ubuntu.com/ubuntu trusty multiverse
deb-src http://archive.ubuntu.com/ubuntu trusty multiverse
deb http://archive.ubuntu.com/ubuntu trusty-updates multiverse
deb-src http://archive.ubuntu.com/ubuntu trusty-updates multiverse
deb http://archive.ubuntu.com/ubuntu trusty-backports main restricted universe multiverse
deb-src http://archive.ubuntu.com/ubuntu trusty-backports main restricted universe multiverse
deb http://security.ubuntu.com/ubuntu trusty-security main restricted
deb-src http://security.ubuntu.com/ubuntu trusty-security main restricted
deb http://security.ubuntu.com/ubuntu trusty-security universe
deb-src http://security.ubuntu.com/ubuntu trusty-security universe
deb http://security.ubuntu.com/ubuntu trusty-security multiverse
deb-src http://security.ubuntu.com/ubuntu trusty-security multiverse
EOF
# This is being set because sda is on hosts, vda is kvm, xvda is xen.
DEVICE_NAME="vda"
# This gets the root users SSH-public-key
SSHKEY=$(cat /root/.ssh/id_rsa.pub)
# This is set to instruct the preseed what the default network is expected to be
DEFAULT_NETWORK="eth0"
# when templated replace \$ with $ and \\ with \
cat > /var/lib/cobbler/kickstarts/ubuntu-server-14.04-unattended-cobbler.seed <<EOF
#
# Ubuntu Server 14.04 Preseed
#
# Usage:
#
# This Ubuntu Preseed file is meant to be used with Cobbler. It has been
# tested only on Cobbler 2.4.1. It is a completely unattended install.
#
# References:
#
# https://help.ubuntu.com/14.04/installation-guide/example-preseed.txt
# http://www.claudiokuenzler.com/blog/513/debian-ubuntu-preseed-create-two-volume-groups-same-disk
#############
#
# Kernel Options
#
#############
# Use the following option to add additional boot parameters for the
# installed system (if supported by the bootloader installer).
# Note: options passed to the installer will be added automatically.
d-i debian-installer/add-kernel-opts string \$kernel_options_post
#############
#
# Networking
#
#############
#if "${DEFAULT_NETWORK}" in \$interfaces
#set idata = \$interfaces["${DEFAULT_NETWORK}"]
#set static = \$idata['static']
#set ip_address = \$idata.get("ip_address", "127.0.0.1")
#set netmask = \$idata.get("netmask", "255.0.0.0")
#set gateway = \$getVar('gateway','127.0.0.1')
#end if
d-i netcfg/choose_interface select ${DEFAULT_NETWORK}
#if \$static
# Static network configuration.
d-i netcfg/disable_autoconfig boolean true
d-i netcfg/dhcp_failed note
d-i netcfg/dhcp_options select Configure network manually
d-i netcfg/get_ipaddress string \$idata.get("ip_address", "127.0.0.1")
d-i netcfg/get_netmask string \$idata.get("netmask", "255.0.0.0")
d-i netcfg/get_gateway string \$gateway
d-i netcfg/get_nameservers string \$name_servers[0]
d-i netcfg/confirm_static boolean true
#end if
d-i netcfg/get_hostname string \$system_name
d-i netcfg/get_domain string \$hostname
# Disable that annoying WEP key dialog.
d-i netcfg/wireless_wep string
#############
#
# Pre Install
#
#############
# Command Line 1: This is necessary otherwise you will be prompted to umount /dev/${DEVICE_NAME}. See Ubuntu bug #1347726.
d-i preseed/early_command string \
umount /media || true
#############
#
# Net Image
#
#############
# Required at least for 12.10+
d-i live-installer/net-image string http://\$http_server/cobbler/links/\$distro_name/install/filesystem.squashfs
#############
#
# Localization
#
#############
d-i debian-installer/locale string en
d-i debian-installer/country string US
d-i debian-installer/locale string en_US.UTF-8
d-i debian-installer/language string en
#############
#
# Keyboard
#
#############
# Disable automatic (interactive) keymap detection.
d-i console-setup/ask_detect boolean false
d-i console-setup/layoutcode string us
d-i console-setup/variantcode string
d-i keyboard-configuration/layoutcode string us
#############
#
# Mirror
#
#############
d-i mirror/country string manual
d-i mirror/http/proxy string
d-i mirror/http/hostname string \$http_server
d-i mirror/http/directory string \$install_source_directory
#############
#
# Clock and Time Zone
#
#############
# Controls whether to use NTP to set the clock during the install
d-i clock-setup/ntp boolean true
d-i clock-setup/ntp-server string ntp.ubuntu.com
# You may set this to any valid setting for \$TZ; see the contents of
# /usr/share/zoneinfo/ for valid values.
d-i time/zone string US/Central
# Controls whether or not the hardware clock is set to UTC.
d-i clock-setup/utc boolean true
#############
#
# Partitioning
#
#############
# If one of the disks that are going to be automatically partitioned
# contains an old LVM configuration, the user will normally receive a
# warning. This can be preseeded away...
d-i partman-lvm/device_remove_lvm boolean true
# The same applies to pre-existing software RAID array:
d-i partman-md/device_remove_md boolean true
# And the same goes for the confirmation to write the lvm partitions.
d-i partman-lvm/confirm boolean true
d-i partman-lvm/confirm_nooverwrite boolean true
d-i partman-auto/method string lvm
d-i partman-auto/disk string /dev/${DEVICE_NAME}
# For LVM partitioning, you can select how much of the volume group to use
# for logical volumes.
d-i partman-auto-lvm/guided_size string max
d-i partman-auto/choose_recipe select custompartitioning
d-i partman-auto/expert_recipe string \\
custompartitioning :: \\
512 1 512 ext2 \\
\$primary{ } \\
\$bootable{ } \\
method{ format } format{ } \\
use_filesystem{ } filesystem{ ext2 } \\
label{ boot } \\
mountpoint{ /boot } \\
. \\
1024 1 100% ext4 \\
\$primary{ } \\
method{ lvm } \\
device{ /dev/${DEVICE_NAME}2 } \\
vg_name{ lxc } \\
. \\
8192 1 8192 linux-swap \\
\$lvmok{ } in_vg{ lxc } \\
lv_name{ swap00 } \\
method{ swap } format{ } \\
. \\
51200 1 51200 ext4 \\
\$lvmok{ } in_vg{ lxc } \\
lv_name{ root00 } \\
method{ format } format{ } \\
use_filesystem{ } filesystem{ ext4 } \\
label{ root } \\
mountpoint{ / } \\
. \\
76800 1 76800 ext4 \\
\$lvmok{ } in_vg{ lxc } \\
lv_name{ openstack00 } \\
method{ format } format{ } \\
use_filesystem{ } filesystem{ ext4 } \\
label{ openstack00 } \\
mountpoint{ /openstack } \\
. \\
20480 1 20480 ext4 \\
\$lvmok{ } in_vg{ lxc } \\
lv_name{ deleteme00 } \\
method{ format } format{ } \\
use_filesystem{ } filesystem{ ext4 } \\
label{ deleteme00 } \\
mountpoint{ /deleteme } \\
. \\
1024 1 1000000000 ext4 \\
\$lvmok{ } in_vg{ lxc } \\
lv_name{ nova00 } \\
method{ format } format{ } \\
use_filesystem{ } filesystem{ ext4 } \\
label{ nova } \\
mountpoint{ /var/lib/nova } \\
. \\
# This makes partman automatically partition without confirmation, provided
# that you told it what to do using one of the methods above.
d-i partman-partitioning/confirm_write_new_label boolean true
d-i partman/choose_partition select finish
d-i partman/confirm boolean true
d-i partman/confirm_nooverwrite boolean true
#############
#
# Packages
#
#############
# Package selection
tasksel tasksel/first multiselect openssh-server
# Whether to upgrade packages after debootstrap.
# Allowed values: none, safe-upgrade, full-upgrade
d-i pkgsel/upgrade select none
d-i pkgsel/include string openssh-server bridge-utils vim python2.7
# Policy for applying updates. May be "none" (no automatic updates),
# "unattended-upgrades" (install security updates automatically), or
# "landscape" (manage system with Landscape).
d-i pkgsel/update-policy select none
# Some versions of the installer can report back on what software you have
# installed, and what software you use. The default is not to report back,
# but sending reports helps the project determine what software is most
# popular and include it on CDs.
popularity-contest popularity-contest/participate boolean false
#############
#
# Users and Password
#
#############
# Skip creation of a root account (normal user account will be able to
# use sudo). The default is false; preseed this to true if you want to set
# a root password.
d-i passwd/root-login boolean true
# Alternatively, to skip creation of a normal user account.
d-i passwd/make-user boolean false
# The installer will warn about weak passwords. If you are sure you know
# what you're doing and want to override it, uncomment this.
d-i user-setup/allow-password-weak boolean true
# Root password, either in clear text
d-i passwd/root-password password cobbler
d-i passwd/root-password-again password cobbler
#############
#
# Bootloader
#
#############
# This is fairly safe to set, it makes grub install automatically to the MBR
# if no other operating system is detected on the machine.
d-i grub-installer/only_debian boolean true
#############
#
# Post Install
#
#############
# Command Line 1: Allow root to SSH.
# Command Line 2: Cobbler overwrites sources.list with its own repos. Put the default Ubuntu sources.list back.
# Command Line 3: Cobbler API call to turn off netboot.
d-i preseed/late_command string \\
wget http://\$http_server/trusty-sources.list -O /target/etc/apt/sources.list; \\
wget http://\$http_server/cblr/svc/op/nopxe/system/\$system_name -O /dev/null; \\
sed -i '/PermitRootLogin / s/ .*/ yes/' /target/etc/ssh/sshd_config; \\
mkdir /target/root/.ssh; \\
chmod 0700 /target/root/.ssh; \\
echo "${SSHKEY}" >> /target/root/.ssh/authorized_keys \\
killall.sh; netcfg
#############
#
# Finish
#
#############
# Reboot after the install is finished.
finish-install finish-install/reboot_in_progress note
EOF
# Restart services again and configure autostart
service cobblerd restart
service apache2 restart
service xinetd restart
update-rc.d cobblerd defaults
# Get ubuntu server image
mkdir -p /var/cache/iso
pushd /var/cache/iso
wget http://releases.ubuntu.com/trusty/ubuntu-14.04.4-server-amd64.iso
popd
# import cobbler image
mkdir -p /mnt/iso
mount -o loop /var/cache/iso/ubuntu-14.04.4-server-amd64.iso /mnt/iso
cobbler import --name=ubuntu-14.04.4-server-amd64 --path=/mnt/iso
umount /mnt/iso
cobbler profile add \
--name ubuntu-14.04.4-server-unattended \
--distro ubuntu-14.04.4-server-x86_64 \
--kickstart /var/lib/cobbler/kickstarts/ubuntu-server-14.04-unattended-cobbler.seed
# sync cobbler
cobbler sync
# Get Loaders
cobbler get-loaders
# Update Cobbler Signatures
cobbler signature update
for network in br-dhcp br-mgmt br-vxlan br-storage br-vlan; do
cat > /etc/libvirt/qemu/networks/${network}.xml <<EOF
<network>
<name>${network}</name>
<forward mode="bridge"/>
<bridge name="${network}"/>
</network>
EOF
virsh net-define --file /etc/libvirt/qemu/networks/${network}.xml
virsh net-create --file /etc/libvirt/qemu/networks/${network}.xml
virsh net-autostart ${network}
done
count=0
for node in infra1 infra2 infra3 logging1 compute1 compute2 swift1 swift2 storage1; do
node_count="$((count++))"
cobbler system add \
--name=${node} \
--profile=ubuntu-14.04.4-server-unattended \
--hostname=${node}.openstackci.local \
--kopts="interface=${DEFAULT_NETWORK}" \
--interface=${DEFAULT_NETWORK} \
--mac="52:54:00:bd:81:d${node_count}" \
--ip-address="10.0.0.10${node_count}" \
--subnet=255.255.255.0 \
--gateway=10.0.0.200 \
--name-servers=8.8.8.8 8.8.4.4 \
--static=1
done
count=0
for node in infra1 infra2 infra3 logging1 compute1 compute2 swift1 swift2 storage1; do
node_count="$((count++))"
qemu-img create -f qcow2 /var/lib/libvirt/images/${node}.openstackci.local.img 252G
cat > /etc/libvirt/qemu/${node}.openstackci.local.xml <<EOF
<domain type='kvm'>
<name>${node}.openstackci.local</name>
<memory unit='KiB'>8392704</memory>
<currentMemory unit='KiB'>8392704</currentMemory>
<vcpu placement='static'>8</vcpu>
<resource>
<partition>/machine</partition>
</resource>
<os>
<type arch='x86_64' machine='pc-i440fx-trusty'>hvm</type>
</os>
<features>
<acpi/>
<apic/>
</features>
<cpu mode='custom' match='exact'>
<model fallback='allow'>SandyBridge</model>
</cpu>
<clock offset='utc'>
<timer name='rtc' tickpolicy='catchup'/>
<timer name='pit' tickpolicy='delay'/>
<timer name='hpet' present='no'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>restart</on_crash>
<pm>
<suspend-to-mem enabled='no'/>
<suspend-to-disk enabled='no'/>
</pm>
<devices>
<emulator>/usr/bin/kvm-spice</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/var/lib/libvirt/images/${node}.openstackci.local.img'/>
<target dev='vda' bus='virtio'/>
<boot order='1'/>
<alias name='virtio-disk0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
</disk>
<controller type='usb' index='0' model='ich9-ehci1'>
<alias name='usb0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x7'/>
</controller>
<controller type='usb' index='0' model='ich9-uhci1'>
<alias name='usb0'/>
<master startport='0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0' multifunction='on'/>
</controller>
<controller type='usb' index='0' model='ich9-uhci2'>
<alias name='usb0'/>
<master startport='2'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x1'/>
</controller>
<controller type='usb' index='0' model='ich9-uhci3'>
<alias name='usb0'/>
<master startport='4'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x2'/>
</controller>
<controller type='pci' index='0' model='pci-root'>
<alias name='pci.0'/>
</controller>
<controller type='virtio-serial' index='0'>
<alias name='virtio-serial0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
</controller>
<interface type='bridge'>
<mac address="52:54:00:bd:81:d${node_count}"/>
<source bridge='br-dhcp'/>
<target dev='vnet0'/>
<model type='virtio'/>
<boot order='2'/>
<alias name='net0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</interface>
<interface type='bridge'>
<mac address="52:54:00:fd:7f:4${node_count}"/>
<source bridge='br-mgmt'/>
<target dev='vnet1'/>
<model type='virtio'/>
<boot order='3'/>
<alias name='net1'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x0a' function='0x0'/>
</interface>
<interface type='bridge'>
<mac address="52:54:00:f5:44:8${node_count}"/>
<source bridge='br-storage'/>
<target dev='vnet2'/>
<model type='virtio'/>
<boot order='4'/>
<alias name='net2'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x0b' function='0x0'/>
</interface>
<interface type='bridge'>
<mac address="52:54:00:1f:96:c${node_count}"/>
<source bridge='br-vlan'/>
<target dev='vnet3'/>
<model type='virtio'/>
<boot order='5'/>
<alias name='net3'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x0c' function='0x0'/>
</interface>
<interface type='bridge'>
<mac address="52:54:00:50:03:8${node_count}"/>
<source bridge='br-vxlan'/>
<target dev='vnet4'/>
<model type='virtio'/>
<boot order='6'/>
<alias name='net4'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x0d' function='0x0'/>
</interface>
<serial type='pty'>
<source path='/dev/pts/4'/>
<target port='0'/>
<alias name='serial0'/>
</serial>
<console type='pty' tty='/dev/pts/4'>
<source path='/dev/pts/4'/>
<target type='serial' port='0'/>
<alias name='serial0'/>
</console>
<channel type='spicevmc'>
<target type='virtio' name='com.redhat.spice.0'/>
<alias name='channel0'/>
<address type='virtio-serial' controller='0' bus='0' port='1'/>
</channel>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<graphics type='spice' port='5900' autoport='yes' listen='127.0.0.1'>
<listen type='address' address='127.0.0.1'/>
</graphics>
<sound model='ich6'>
<alias name='sound0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
</sound>
<video>
<model type='vmvga' vram='9216' heads='1'/>
<alias name='video0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
</video>
<redirdev bus='usb' type='spicevmc'>
<alias name='redir0'/>
</redirdev>
<redirdev bus='usb' type='spicevmc'>
<alias name='redir1'/>
</redirdev>
<memballoon model='virtio'>
<alias name='balloon0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
</memballoon>
<rng model='virtio'>
<backend model='random'>/dev/random</backend>
<alias name='rng0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x09' function='0x0'/>
</rng>
</devices>
</domain>
EOF
virsh define /etc/libvirt/qemu/${node}.openstackci.local.xml
virsh create /etc/libvirt/qemu/${node}.openstackci.local.xml
cat > /opt/osa-${node}.openstackci.local-bridges.cfg <<EOF
auto br-mgmt
iface br-mgmt inet static
bridge_stp off
bridge_waitport 0
bridge_fd 0
# Notice the bridge port is the vlan tagged interface
bridge_ports eth1
address 172.29.236.10${node_count}
netmask 255.255.252.0
offload-sg off
auto br-storage
iface br-storage inet static
bridge_stp off
bridge_waitport 0
bridge_fd 0
bridge_ports eth2
address 172.29.244.10${node_count}
netmask 255.255.252.0
offload-sg off
auto br-vlan
iface br-vlan inet static
bridge_stp off
bridge_waitport 0
bridge_fd 0
address 172.29.248.10${node_count}
netmask 255.255.252.0
offload-sg off
# Create veth pair, don't bomb if already exists
pre-up ip link add br-vlan-veth type veth peer name eth12 || true
# Set both ends UP
pre-up ip link set br-vlan-veth up
pre-up ip link set eth12 up
# Delete veth pair on DOWN
post-down ip link del br-vlan-veth || true
bridge_ports br-vlan-veth eth3
auto br-vxlan
iface br-vxlan inet static
bridge_stp off
bridge_waitport 0
bridge_fd 0
bridge_ports eth4
address 172.29.240.10${node_count}
netmask 255.255.252.0
offload-sg off
# To ensure ssh checksum is correct
up /sbin/iptables -A POSTROUTING -t mangle -p tcp --dport 22 -j CHECKSUM --checksum-fill
down /sbin/iptables -D POSTROUTING -t mangle -p tcp --dport 22 -j CHECKSUM --checksum-fill
# To provide internet connectivity to instances
up /sbin/iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
down /sbin/iptables -t nat -D POSTROUTING -o eth0 -j MASQUERADE
EOF
done
function wait_ssh() {
count=0
echo "Waiting for all nodes to become available. This can take around 10 min"
for node in infra1 infra2 infra3 logging1 compute1 compute2 swift1 swift2 storage1; do
node_count="$((count++))"
echo "Waiting for node: ${node}"
ssh -q -o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10 10.0.0.10${node_count} exit > /dev/null
while test $? -gt 0; do
sleep 15
ssh -q -o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10 10.0.0.10${node_count} exit > /dev/null
done
done
}
wait_ssh
count=0
for node in infra1 infra2 infra3 logging1 compute1 compute2 swift1 swift2 storage1; do
node_count="$((count++))"
echo "Setup node bridge configs: ${node}"
scp -o StrictHostKeyChecking=no /opt/osa-${node}.openstackci.local-bridges.cfg 10.0.0.10${node_count}:/etc/network/interfaces.d/osa-${node}.openstackci.local-bridges.cfg
ssh -q -o StrictHostKeyChecking=no 10.0.0.10${node_count} <<EOF
if ! grep "^source.*cfg$" /etc/network/interfaces; then
echo 'source /etc/network/interfaces.d/*.cfg' | tee -a /etc/network/interfaces
fi
umount /deleteme || true
echo y | lvremove /dev/lxc/deleteme00 || true
sed -i 's/^\/dev\/mapper\/lxc-deleteme00.*//g' /etc/fstab
shutdown -r now
EOF
done
wait_ssh
# Infra storage setup
for node in 10.0.0.100 10.0.0.101 10.0.0.102; do
ssh -q -o StrictHostKeyChecking=no ${node} <<EOF
umount /var/lib/nova
echo y | lvremove /dev/lxc/nova00 || true
sed -i 's/^\/dev\/mapper\/lxc-nova00.*//g' /etc/fstab
lvresize -r -l+100%FREE /dev/lxc/root00
EOF
done
# Logging storage setup
for node in 10.0.0.103; do
ssh -q -o StrictHostKeyChecking=no ${node} <<EOF
umount /var/lib/nova
echo y | lvremove /dev/lxc/nova00 || true
sed -i 's/^\/dev\/mapper\/lxc-nova00.*//g' /etc/fstab
lvresize -r -l+100%FREE /dev/lxc/openstack00
EOF
done
# swift storage setup
for node in 10.0.0.106 10.0.0.107; do
ssh -q -o StrictHostKeyChecking=no ${node} <<EOF
umount /var/lib/nova
echo y | lvremove /dev/lxc/nova00 || true
sed -i 's/^\/dev\/mapper\/lxc-nova00.*//g' /etc/fstab
# apt-get update && apt-get -y install xfsprogs
for disk in disk1 disk2 disk3; do
lvcreate --name \${disk} -L 30G lxc
mkfs.xfs /dev/lxc/\${disk}
mkdir -p /src/\${disk}
mount /dev/lxc/\${disk} /srv/\${disk}
echo "/dev/mapper/lxc-\${disk} /srv/\${disk} xfs defaults 0 0" | tee -a /etc/fstab
done
EOF
done
# Storage storage setup
for node in 10.0.0.108; do
ssh -q -o StrictHostKeyChecking=no ${node} <<EOF
umount /var/lib/nova
echo y | lvremove /dev/lxc/nova00 || true
sed -i 's/^\/dev\/mapper\/lxc-nova00.*//g' /etc/fstab
lvcreate --name cinder -l 100%FREE lxc
vgcreate cinder-volumes /dev/lxc/cinder
EOF
done
apt-get install -y git tmux
pushd /opt
git clone https://github.com/openstack/openstack-ansible
cp -R openstack-ansible/etc/openstack_deploy /etc/openstack_deploy
popd
cat > /etc/openstack_deploy/conf.d/swift.yml <<EOF
---
global_overrides:
swift:
part_power: 8
storage_network: 'br-storage'
replication_network: 'br-storage'
drives:
- name: disk1
- name: disk2
- name: disk3
mount_point: /srv
storage_policies:
- policy:
name: default
index: 0
default: True
swift-proxy_hosts:
swift1:
ip: 172.29.236.106
container_vars:
swift_proxy_vars:
limit_container_types: swift_proxy
read_affinity: "r1=100"
write_affinity: "r1"
write_affinity_node_count: "1 * replicas"
swift2:
ip: 172.29.236.107
container_vars:
swift_proxy_vars:
limit_container_types: swift_proxy
read_affinity: "r1=100"
write_affinity: "r1"
write_affinity_node_count: "1 * replicas"
swift_hosts:
swift1:
ip: 10.0.0.106
container_vars:
swift_vars:
limit_container_types: swift
zone: 0
region: 1
swift2:
ip: 10.0.0.107
container_vars:
swift_vars:
limit_container_types: swift
zone: 0
region: 1
EOF
cat > /etc/openstack_deploy/openstack_user_config.yml<<EOF
---
cidr_networks:
container: 172.29.236.0/22
tunnel: 172.29.240.0/22
storage: 172.29.244.0/22
used_ips:
- "172.29.236.100,172.29.236.108"
- "172.29.240.100,172.29.240.108"
- "172.29.244.100,172.29.244.108"
- "172.29.248.100,172.29.248.108"
- "172.29.236.200"
- "172.29.240.200"
- "172.29.244.200"
- "172.29.248.200"
global_overrides:
internal_lb_vip_address: 172.29.236.200
# The external IP is quoted simply to ensure that the .aio file can be used as input
# dynamic inventory testing.
external_lb_vip_address: "$(ip route get 1 | awk '{print $NF;exit}')"
tunnel_bridge: "br-vxlan"
management_bridge: "br-mgmt"
provider_networks:
- network:
container_bridge: "br-mgmt"
container_type: "veth"
container_interface: "eth1"
ip_from_q: "container"
type: "raw"
group_binds:
- all_containers
- hosts
is_container_address: true
is_ssh_address: true
- network:
container_bridge: "br-vxlan"
container_type: "veth"
container_interface: "eth10"
ip_from_q: "tunnel"
type: "vxlan"
range: "1:1000"
net_name: "vxlan"
group_binds:
- neutron_linuxbridge_agent
- network:
container_bridge: "br-vlan"
container_type: "veth"
container_interface: "eth12"
host_bind_override: "eth12"
type: "flat"
net_name: "flat"
group_binds:
- neutron_linuxbridge_agent
- network:
container_bridge: "br-vlan"
container_type: "veth"
container_interface: "eth11"
type: "vlan"
range: "1:1"
net_name: "vlan"
group_binds:
- neutron_linuxbridge_agent
- network:
container_bridge: "br-storage"
container_type: "veth"
container_interface: "eth2"
ip_from_q: "storage"
type: "raw"
group_binds:
- glance_api
- cinder_api
- cinder_volume
- nova_compute
- swift_proxy
shared-infra_hosts:
infra1:
ip: 172.29.236.100
infra2:
ip: 172.29.236.101
infra3:
ip: 172.29.236.102
os-infra_hosts:
infra1:
ip: 172.29.236.100
infra2:
ip: 172.29.236.101
infra3:
ip: 172.29.236.102
storage-infra_hosts:
storage1:
ip: 172.29.236.108
repo-infra_hosts:
infra1:
ip: 172.29.236.100
infra2:
ip: 172.29.236.101
infra3:
ip: 172.29.236.102
identity_hosts:
infra1:
ip: 172.29.236.100
infra2:
ip: 172.29.236.101
infra3:
ip: 172.29.236.102
compute_hosts:
compute1:
ip: 172.29.236.104
compute2:
ip: 172.29.236.105
storage_hosts:
storage1:
ip: 172.29.236.108
container_vars:
cinder_backends:
limit_container_types: cinder_volume
lvm:
volume_group: cinder-volumes
volume_driver: cinder.volume.drivers.lvm.LVMVolumeDriver
volume_backend_name: LVM_iSCSI
iscsi_ip_address: "172.29.236.108"
log_hosts:
logging1:
ip: 172.29.236.103
network_hosts:
infra1:
ip: 172.29.236.100
infra2:
ip: 172.29.236.101
infra3:
ip: 172.29.236.102
haproxy_hosts:
loadbalancer1:
ip: 172.29.236.200
EOF
pushd /opt/openstack-ansible/
bash ./scripts/bootstrap-ansible.sh
python ./scripts/pw-token-gen.py --file /etc/openstack_deploy/user_secrets.yml
# This is happening so the VMs running the infra use less storage
echo 'lxc_container_backing_store: dir' | tee -a /etc/openstack_deploy/user_variables.yml
popd
pushd /opt/openstack-ansible/playbooks
# Running the HAP play is done because it "may" be needed. Note: In Master its not.
openstack-ansible haproxy-install.yml
openstack-ansible setup-everything.yml
popd
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment