Deploy Ceph Quincy & Openstack Zed Multinode on Ubuntu 22.04

README.txt

=========================
ENVIRONMENT
=========================
- 2x Ubuntu 22.04
- Vda-Vdd Disk
- ens3-ens6 Network Interface
  - ens6 == no-IP

=========================
CEPH-ANSIBLE PREPARE
=========================
apt-get update -y
apt-get install python3-venv -y
apt-get install python3-pip -y

python3 -m venv ~/ceph/ceph-venv
source ~/ceph/ceph-venv/bin/activate
cd ~/ceph

git clone https://github.com/ceph/ceph-ansible.git
cd ceph-ansible
git checkout stable-7.0

pip3 install -U pip
pip3 install -r requirements.txt

sudo add-apt-repository ppa:ansible/ansible
sudo apt update
sudo apt install ansible
apt install --no-install-recommends python3-netaddr
ansible-galaxy install -r requirements.yml

cp site.yml.sample site.yml
cd group_vars/
cp all.yml.sample all.yml
cp mons.yml.sample mons.yml
cp osds.yml.sample osds.yml
cp mgrs.yml.sample mgrs.yml

vi all.yml
=========================
CEPH-ANSIBLE ALLS.YML
=========================
ceph_origin: distro
ceph_repository: community
ceph_stable_release: quincy

#Interface Public Network Ceph
monitor_interface: ens4
osd_objectstore: bluestore
public_network: 192.168.30.0/24
cluster_network: 192.168.40.0/24
dashboard_enabled: True
dashboard_admin_user: admin
dashboard_admin_password: 6lC5IZfOEcvMRRQb7CZ+7XB4FFOLHj1wiHramvzwuM
ntp_daemon_type: timesyncd
grafana_plugins: {} # temporary
grafana_admin_user: admin # temporary
grafana_admin_password: admin # temporary

vi osds.yml
=========================
CEPH-ANSIBLE OSDS
=========================
devices:
  - /dev/vdb
  - /dev/vdc
  - /dev/vdd  
osd_auto_discovery: false


cd ~/ceph/ceph-ansible
vi ansible.cfg
---
[defaults]
inventory=hosts

vi hosts
---
[mons]
node1

[osds]
node1
node2

[mgrs]
node1

[grafana-server]
node1

ansible -m ping all
ansible-playbook site.yml
ceph -s

ceph osd pool create images 64
ceph osd pool set images size 2

ceph osd pool create volumes 64
ceph osd pool set volumes size 2

ceph osd pool create vms 64
ceph osd pool set vms size 2

ceph osd pool application enable volumes rbd
ceph osd pool application enable images rbd
ceph osd pool application enable vms rbd

ceph auth get-or-create client.glance mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=images' -o /etc/ceph/ceph.client.glance.keyring
ceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=images' -o /etc/ceph/ceph.client.cinder.keyring
ceph auth get-or-create client.nova mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=vms, allow rx pool=images' -o /etc/ceph/ceph.client.nova.keyring

mkdir /etc/kolla/config
mkdir /etc/kolla/config/nova
mkdir /etc/kolla/config/glance
mkdir -p /etc/kolla/config/cinder/cinder-volume
mkdir /etc/kolla/config/cinder/cinder-backup

cp /etc/ceph/ceph.conf /etc/kolla/config/cinder/
cp /etc/ceph/ceph.conf /etc/kolla/config/nova/
cp /etc/ceph/ceph.conf /etc/kolla/config/glance/
cp /etc/ceph/ceph.client.glance.keyring /etc/kolla/config/glance/
cp /etc/ceph/ceph.client.nova.keyring /etc/kolla/config/nova/
cp /etc/ceph/ceph.client.cinder.keyring /etc/kolla/config/nova/
cp /etc/ceph/ceph.client.cinder.keyring /etc/kolla/config/cinder/cinder-volume/

ceph osd pool ls detail # detail pool
ceph config dump # list ceph config
ceph auth ls # ceph client config
ceph -s # ceph status
ceph df # ceph cluster usage
ceph osd tree # ceph osd list

===============================================================================
KOLLA ANSIBLE PREPARE
===============================================================================

apt-get install python3-dev libffi-dev gcc libssl-dev python3-selinux python3-setuptools python3-venv -y
python3 -m venv ~/openstack/os-venv
source ~/openstack/os-venv/bin/activate
cd ~/openstack

pip3 install -U pip
pip3 install ansible-core==2.12.0
pip3 install kolla-ansible==15.0.0
pip3 install -U pip
kolla-ansible install-deps

sudo mkdir -p /etc/kolla
sudo chown $USER:$USER /etc/kolla

cp -r ~/openstack/os-venv/share/kolla-ansible/etc_examples/kolla/* /etc/kolla
cp ~/openstack/os-venv/share/kolla-ansible/ansible/inventory/* .

vi /etc/ansible/ansible.cfg
---
[defaults]
host_key_checking=False
pipelining=True
forks=100
interpreter_python=/usr/bin/python3

vi multinode
---
[control]
node1

[network]
node1

[compute]
node1
node2

[monitoring]
node1

[storage]
node1
node2

[deployment]
localhost ansible_connection=local

ansible -i multinode all -m ping
kolla-genpwd

nano /etc/kolla/globals.yml
============================
KOLLA-ANSIBLE GLOBALS.YML
============================

## GENERAL
kolla_base_distro: "ubuntu"
kolla_install_type: "source"
openstack_release: "zed"
kolla_internal_vip_address: "172.20.2.155"
kolla_external_vip_address: "172.20.2.156"
network_interface: "ens3"
storage_interface: "ens4"
neutron_external_interface: "ens6"
enable_openstack_core: "yes"
enable_neutron_provider_networks: "yes"
nova_compute_virt_type: "kvm"
enable_fluentd: "no"
neutron_plugin_agent: "ovn"

## CINDER
enable_cinder: "yes"
enable_cinder_backup: "no"

## FQDN
kolla_external_fqdn: "gl.adaptivenetworklab.org"

## monitoring
enable_prometheus: "yes"
enable_prometheus_openstack_exporter: "yes"
enable_grafana: "yes"

## tls 
kolla_enable_tls_external: "yes"

## ceph
ceph_cinder_keyring: "ceph.client.cinder.keyring"
ceph_glance_keyring: "ceph.client.glance.keyring"
ceph_nova_keyring: "ceph.client.nova.keyring"
ceph_cinder_backup_keyring: "ceph.client.cinder-backup.keyring"
glance_backend_ceph: "yes"
cinder_backend_ceph: "yes"
nova_backend_ceph: "yes"

ceph_cinder_user: "cinder"
ceph_nova_user: "nova"
ceph_glance_user: "glance"

kolla-ansible -i multinode certificates
mkdir /etc/kolla/config
mkdir /etc/kolla/config/nova
mkdir /etc/kolla/config/glance
mkdir -p /etc/kolla/config/cinder/cinder-volume

cp /etc/ceph/ceph.conf /etc/kolla/config/cinder/
cp /etc/ceph/ceph.conf /etc/kolla/config/nova/
cp /etc/ceph/ceph.conf /etc/kolla/config/glance/
cp /etc/ceph/ceph.client.glance.keyring /etc/kolla/config/glance/
cp /etc/ceph/ceph.client.cinder.keyring
/etc/kolla/config/cinder/cinder-volume/
cp /etc/ceph/ceph.client.nova.keyring /etc/kolla/config/nova/
cp /etc/ceph/ceph.client.cinder.keyring /etc/kolla/config/nova/

kolla-ansible -i ./multinode bootstrap-servers
kolla-ansible -i ./multinode prechecks
kolla-ansible -i ./multinode deploy
kolla-ansible -i ./multinode post-deploy

pip3 install python-openstackclient
source /etc/kolla/admin-openrc.sh
openstack service list
./kolla-venv/share/kolla-ansible/init-runonce

==========
REFERECE
==========
    - https://docs.ceph.com/projects/ceph-ansible/en/stable-7.0/
    - https://docs.openstack.org/project-deploy-guide/kolla-ansible/latest/quickstart.html
    - https://releases.openstack.org/teams/kolla.html