Devstack Multi Node setup

Devstack Multi Node

Preparation

First, make sure kernel is willing to forward packets

# Edit /etc/sysctl.conf file, make sure ip forwarding is enabled
vim /etc/sysctl.conf
  net.ipv4.ip_forward=1
  net.ipv4.conf.all.rp_filter=0
  net.ipv4.conf.default.rp_filter=0

# Make the changes take effect
sysctl -p
get the devstack code from git

cd ~/workspace/
git clone https://github.com/openstack-dev/devstack.git
cd devstack
git checkout stable/kilo
git checkout 24cc0023ddca09bbbf44b70f9b4f61b500a1c0fd
To install devstack multi-node (cluster), first run devstack on your controller node with its local.conf, then run devstack on each of the compute node. I use 1 controller node and leave other nodes to be compute nodes.

Controller node devstack/local.conf

[[local|localrc]]
GIT_BASE=${GIT_BASE:-https://git.openstack.org}

ADMIN_PASSWORD=secrete
DATABASE_PASSWORD=$ADMIN_PASSWORD
RABBIT_PASSWORD=$ADMIN_PASSWORD
SERVICE_PASSWORD=$ADMIN_PASSWORD
SERVICE_TOKEN=secrete
RECLONE=no

# public interface eth0 will be added to ovs br-ex. eth0 loses its ip, br-ex gets the ip.
# if the process is not finished correctly, host is left ssh unconnectable.
PUBLIC_INTERFACE=eth0

FLAT_INTERFACE=eth0
FIXED_RANGE=192.168.120.0/24
NETWORK_GATEWAY=192.168.120.1
FLOATING_RANGE=10.12.181.224/27
PUBLIC_NETWORK_GATEWAY=10.12.181.225
HOST_IP=10.12.181.120

# misc
API_RATE_LIMIT=False
LIBS_FROM_GIT=python-openstackclient

# log
DEBUG=True
VERBOSE=True
DEST=/opt/stack
LOGFILE=$DEST/logs/stack.sh.log
SCREEN_LOGDIR=$DEST/logs/screen

SYSLOG=False
LOG_COLOR=False
LOGDAYS=7

# If don't set ENABLED_SERVICES, devstack will install its default set of services
ENABLED_SERVICES=key,dstat,rabbit,mysql,tempest

# keystone
KEYSTONE_CATALOG_BACKEND=sql

# enable ceph
enable_service ceph
CEPH_LOOPBACK_DISK_SIZE=10G
CEPH_CONF=/etc/ceph/ceph.conf
CEPH_REPLICAS=3
GLANCE_CEPH_USER=glance
GLANCE_CEPH_POOL=glance
CINDER_DRIVER=ceph
CINDER_CEPH_USER=cinder
CINDER_CEPH_POOL=cinder
CINDER_CEPH_UUID=65B06887-B0EA-427F-B8BD-829AC2E18FF6
CINDER_BAK_CEPH_POOL=cinder_bak
CINDER_BAK_CEPH_USER=cind_bak
CINDER_ENABLED_BACKENDS=ceph,lvm
NOVA_CEPH_POOL=nova

# adjust scheduler to balance VMs (followed devstack official doc, but nova doesn't actually have this SimpleScheduler)
#SCHEDULER=nova.scheduler.simple.SimpleScheduler

# enable nova
enable_service n-api
enable_service n-sch
enable_service n-cond
enable_service n-api-meta
enable_service n-novnc
enable_service n-crt
enable_service n-net
enable_service n-cauth

# enable glance
enable_service g-api
enable_service g-reg

# enable swift (I find no way to just install s-proxy on controller node)
enable_service swift3
enable_service s-proxy s-object s-container s-account
SWIFT_HASH=E75834B828A54832B8AF2294FD8F5C5D
SWIFT_REPLICAS=3
SWIFT_DATA_DIR=$DEST/data/swift

# cinder
enable_service c-api
enable_service c-sch
enable_service c-bak

VOLUME_GROUP="stack-volumes"
VOLUME_NAME_PREFIX="volume-"
VOLUME_BACKING_FILE_SIZE=10250M

# enable neutron
enable_service q-svc

# VLAN configuration
Q_PLUGIN=ml2
ENABLE_TENANT_VLANS=True

# GRE tunnel configuration
Q_PLUGIN=ml2
ENABLE_TENANT_TUNNELS=True

# VXLAN tunnel configuration
Q_PLUGIN=ml2
Q_ML2_TENANT_NETWORK_TYPE=vxlan

# enable ceilometer
enable_service ceilometer-acentral
enable_service ceilometer-collector
enable_service ceilometer-alarm-singleton
enable_service ceilometer-alarm-notifier
enable_service ceilometer-alarm-evaluator
enable_service ceilometer-api
CEILOMETER_BACKEND=mongodb

# enable heat
enable_service heat
enable_service h-api
enable_service h-api-cfn
enable_service h-api-cw
enable_service h-eng

# enable horizon
enable_service horizon
Compute node devstack/local.conf. For multi-node guide, see [1][2].

[[local|localrc]]
GIT_BASE=${GIT_BASE:-https://git.openstack.org}

SERVICE_HOST=10.12.181.120
MYSQL_HOST=$SERVICE_HOST
RABBIT_HOST=$SERVICE_HOST
Q_HOST=$SERVICE_HOST
MATCHMAKER_REDIS_HOST=$SERVICE_HOST
DATABASE_TYPE=mysql

ADMIN_PASSWORD=secrete
DATABASE_PASSWORD=$ADMIN_PASSWORD
RABBIT_PASSWORD=$ADMIN_PASSWORD
SERVICE_PASSWORD=$ADMIN_PASSWORD
MYSQL_PASSWORD=$ADMIN_PASSWORD
SERVICE_TOKEN=secrete
RECLONE=no

# public interface eth0 will be added to ovs br-ex. eth0 loses its ip, br-ex gets the ip.
# if the process is not finished correctly, host is left ssh unconnectable.
PUBLIC_INTERFACE=eth0

FLAT_INTERFACE=eth0
FIXED_RANGE=192.168.120.0/24
NETWORK_GATEWAY=192.168.120.1
FLOATING_RANGE=10.12.181.224/27
PUBLIC_NETWORK_GATEWAY=10.12.181.225

# NEED TO CHANGE
HOST_IP=10.12.181.121

# misc
API_RATE_LIMIT=False
LIBS_FROM_GIT=python-openstackclient

# log
DEBUG=True
VERBOSE=True
DEST=/opt/stack
LOGFILE=$DEST/logs/stack.sh.log
SCREEN_LOGDIR=$DEST/logs/screen

SYSLOG=False
LOG_COLOR=False
LOGDAYS=7

# If don't set ENABLED_SERVICES, devstack will install its default set of services
ENABLED_SERVICES=dstat,rabbit,tempest

# Nova
enable_service n-cpu
NOVA_VNC_ENABLED=True
NOVNCPROXY_URL="http://${SERVICE_HOST}:6080/vnc_auto.html"
VNCSERVER_LISTEN=$HOST_IP
VNCSERVER_PROXYCLIENT_ADDRESS=$VNCSERVER_LISTEN

# Cinder
enable_service c-vol
GLANCE_CEPH_USER=glance
GLANCE_CEPH_POOL=glance
CINDER_DRIVER=ceph
CINDER_CEPH_USER=cinder
CINDER_CEPH_POOL=cinder
CINDER_CEPH_UUID=65B06887-B0EA-427F-B8BD-829AC2E18FF6
CINDER_BAK_CEPH_POOL=cinder_bak
CINDER_BAK_CEPH_USER=cind_bak
CINDER_ENABLED_BACKENDS=ceph,lvm
NOVA_CEPH_POOL=nova

VOLUME_GROUP="stack-volumes"
VOLUME_NAME_PREFIX="volume-"
VOLUME_BACKING_FILE_SIZE=10250M

# Swift
# I find no way to install multi-node swift with devstack, so I just use standalone swift on controller node
#enable_service s-proxy s-object s-container s-account
SWIFT_HASH=E75834B828A54832B8AF2294FD8F5C5D
SWIFT_REPLICAS=3
SWIFT_DATA_DIR=$DEST/data/swift

# Neutron
enable_service q-metering
enable_service q-agt
enable_service q-dhcp
enable_service q-l3
enable_service q-meta
enable_service q-fwaas
enable_service q-lbaas
#enable_service q-vpn

# VLAN configuration
Q_PLUGIN=ml2
ENABLE_TENANT_VLANS=True

# GRE tunnel configuration
Q_PLUGIN=ml2
ENABLE_TENANT_TUNNELS=True

# VXLAN tunnel configuration
Q_PLUGIN=ml2
Q_ML2_TENANT_NETWORK_TYPE=vxlan

# Ceilometer
enable_service ceilometer-acompute
CEILOMETER_BACKEND=mongodb
To run devstack. PS4 usage thanks to Charles. This will output debug message with file name and line number

PS4=':${0}:${LINENO}+' ./stack.sh
After installation finished, grep the error using below command

#rm /opt/stack/logs/*2015*    # delete all the old logs (change the year 2015 if you need)
grep -ir 'error\|fail\|usage\|not found' /opt/stack/logs/stack.sh.log
grep -ir 'error\|fail\|usage\|not found' /opt/stack/logs/screen/
If you met error, before reinstall devstack, you need to cleanup. The screen part thanks to Milind. Here is my myclean.sh to cleanup devstack remainings to prevent some error.

set -x
# clean rabbitmq-server
sudo kill -9 $(ps aux | grep rabbit | awk '{print $2}')
sudo kill -9 $(ps aux | grep epmd | awk '{print $2}')
sudo kill -9 $(ps aux | grep erl | awk '{print $2}')
sudo apt-get remove -y rabbitmq-server

# clean up ceph
sudo kill $(ps -ef| grep ceph | awk '{print $2}')
sudo pkill ceph
sudo apt-get remove -y ceph
sudo umount /var/lib/ceph
sudo rm -rf /var/lib/ceph
sudo rm -rf /etc/ceph

# stop swift
sudo swift-init stop all

# stop devstack and openstack
sudo pkill -9 -f python
sudo pkill -9 -f devstack
screen -ls | grep stack | cut -d. -f1 | awk '{print $1}' | xargs kill

# remove swift data
sudo umount /opt/stack/data/swift/drives/sdb1
sudo rm -rf /opt/stack/data

# clean vm net route
sudo route delete -net 192.168.120.0/24 gw 10.12.181.226
I recommend run myclean.sh this way because some services may delay a few seconds before they truely stop

./myclean.sh; sleep 5; ./myclean.sh
Installation

Step 1: Patch the devstack code on each node, according to "Troubleshooting: Misplaced ceph command"

Step 2: Patch the devstack code on each node, according to "Troubleshooting: Error: openstack role list raises unrecognized arguments: --group"

Step 3: Install dependencies, according to "Troubleshooting: Error: virsh: command not found"

Step 4: Install the controller node.

Step 5: On controller node, manually add swift user, service and endpoint. See "Troubleshooting: Cannot authenticate through swift client"

Step 6: Copy devstack/accrc folder from controller node to each compute node. See "Troubleshooting: Error: euca-bundle-image: error: certificate file does not exist"

Step 7: On compute node, source openstack login environment variables source accrc/admin/admin. See "Troubleshooting: ERROR: openstack Missing parameter(s)"

Step 8: Change HOST_IP in compute node local.conf to your real host ip.

Step 9: Run stack.sh on each of the compute nodes. After it finishes, Openstack should be available.

Step 10: To be able to access external network. Login Horizon, manually config external network & router to have correct Gateway, DNS and Provider Network Type. (For example, the gateway should be 10.12.181.1. The PUBLIC_NETWORK_GATEWAY in local.conf is wrong. So as is FLOATING_RANGE.) You might need to delete them and create new ones.

BTW, Murano can be installed by devstack very easily. Checkout here. Same works for Magnum, checkout here.

Troubleshooting

A useful tip: when you see "error", scroll the log up until you find the root one.

Error install rabbitmq-server

If you meet this

Errors were encountered while processing:
  rabbitmq-server
Try manually install rabbitmq-server

$ sudo apt-get install rabbitmq-server
Setting up rabbitmq-server (3.2.4-1) ...
 * Starting message broker rabbitmq-server
 * FAILED - check /var/log/rabbitmq/startup_\{log, _err\}]

invoke-rc.d: initscript rabbitmq-server, action "start" failed.
dpkg: error processing package rabbitmq-server (--configure):
 subprocess installed post-installation script returned error exit status 1
E: Sub-process /usr/bin/dpkg returned an error code (1)
Solve by my cleanup script. Thanks to Jordan.

If not solved, stop all docker containers. Finally I found it is the docker services who prevents me from killing rabbitmq. Docker is running Kolla, which contains privileged container running rabbitmq.

Ceph file exists, Ceph connect PermissionError

Symptom

mkdir: cannot create directory '/var/lib/ceph/mon/ceph-sclg120': File exists
Cause: you need to clean ceph totally. Solution: run my cleanup script.

Git clone openstack requirements fail

Symptom

git clone git://git.openstack.org/openstack/requirements.git /opt/stack/requirements
 Cloning into '/opt/stack/requirements'...
 fatal: unable to connect to git.openstack.org:
Solution: add below to local.conf. Thanks to Jacek and Mark.

GIT_BASE=${GIT_BASE:-https://git.openstack.org}
Misplaced ceph command

Symptom. It looks like sudo (ceph) -c missed the ceph command in between

sudo ceph -c /etc/ceph/ceph.conf osd pool set nova size 3
set pool 5 size to 3
[[ 3 -ne 1 ]]
sudo -c /etc/ceph/ceph.conf ceph osd pool set nova crush_ruleset 1
usage: sudo -h | -K | -k | -V
usage: sudo -v [-AknS] [-g group] [-h host] [-p prompt] [-u user]
usage: sudo -l [-AknS] [-g group] [-h host] [-p prompt] [-U user] [-u user]
Check the line of code which triggers this error. There is a bug report about this. You need to modify code to fix this error:

$ git diff
diff --git a/lib/ceph b/lib/ceph
index 76747cc..4068e26 100644
--- a/lib/ceph
+++ b/lib/ceph
@@ -279,7 +279,7 @@ function configure_ceph_embedded_nova {
     # configure Nova service options, ceph pool, ceph user and ceph key
     sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${NOVA_CEPH_POOL} size ${CEPH_REPLICAS}
     if [[ $CEPH_REPLICAS -ne 1 ]]; then
-        sudo -c ${CEPH_CONF_FILE} ceph osd pool set ${NOVA_CEPH_POOL} crush_ruleset ${RULE_ID}
+        sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${NOVA_CEPH_POOL} crush_ruleset ${RULE_ID}
     fi
 }
Related devstack bug: #1453055

openstack role list raises unrecognized arguments: --group

Symptom

::./stack.sh:780+openstack role list --group 3c65c1a8d12f40a2a9949d5b2922beae --project 18ab3a46314442b183db43bc13b175b4 --column ID --column Name
usage: openstack role list [-h] [-f {csv,html,json,table,yaml}] [-c COLUMN]
                           [--max-width <integer>]
                           [--quote {all,minimal,none,nonnumeric}]
                           [--project <project>] [--user <user>]
openstack role list: error: unrecognized arguments: --group 3c65c1a8d12f40a2a9949d5b2922beae
Code location at lib/keystone:418, invoked by functions-common:773.

The first reason is that the python-openstackclient version is too old (openstack --version), upgrade it

sudo pip install --upgrade python-openstackclient
You need to add python-openstackclient to LIBS_FROM_GIT in local.conf, to make sure devstack uses the newest version of python-openstackclient. Note that, devstack will use master branch of python-openstackclient instead of stable/kilo.

# Add python-openstackclient to your LIBS_FROM_GIT
LIBS_FROM_GIT=python-openstackclient
The next step, since keystone v2.0 doesn't even have the concept "group", you need to force here to use keystone V3 api.

$ git diff
diff --git a/functions-common b/functions-common
index d3e93ed..bd55d7e 100644
--- a/functions-common
+++ b/functions-common
@@ -773,12 +773,15 @@ function get_or_add_user_project_role {
 # Gets or adds group role to project
 # Usage: get_or_add_group_project_role <role> <group> <project>
 function get_or_add_group_project_role {
+    local os_url="$KEYSTONE_SERVICE_URI_V3"
     # Gets group role id
     local group_role_id=$(openstack role list \
         --group $2 \
         --project $3 \
         --column "ID" \
         --column "Name" \
+        --os-identity-api-version=3 \
+        --os-url=$os_url \
         | grep " $1 " | get_field 1)
     if [[ -z "$group_role_id" ]]; then
         # Adds role to group
@@ -786,6 +789,8 @@ function get_or_add_group_project_role {
             $1 \
             --group $2 \
             --project $3 \
+            --os-identity-api-version=3 \
+            --os-url=$os_url \
             | grep " id " | get_field 2)
     fi
     echo $group_role_id
Related devstack bug: #1441010

virsh: command not found

Symptom

sudo virsh secret-define --file secret.xml
sudo: virsh: command not found
Solution: install kvm and libvirt manually

sudo apt-get install qemu-kvm libvirt-bin ubuntu-vm-builder bridge-utils
sudo adduser `id -un` libvirtd
sudo adduser `id -un` kvm

# You should see kvm and libvirtd in your groups
groups

# logout and login so that groups take effect
exit

# to verify
virsh -c qemu:///system list
Module version does not exist!

Symptom

:./stack.sh:68+sudo a2enmod version
ERROR: Module version does not exist!
The code location is in lib/apache:68. See the comment

Ensure mod_version enabled for . This is built-in statically on anything recent, but precise (2.2) doesn't have it enabled
So, feel free to ignore this error.

ERROR: openstack No role with a name or ID of '...' exists

Symptom

::./stack.sh:378+get_or_create_role admin
:::./stack.sh:748+openstack role create admin --or-show -f value -c id
::./stack.sh:746+local role_id=40d9db7cef7840c1a85d4339e1f12979
::./stack.sh:747+echo 40d9db7cef7840c1a85d4339e1f12979
:./stack.sh:378+local admin_role=40d9db7cef7840c1a85d4339e1f12979
:./stack.sh:379+get_or_add_user_project_role 40d9db7cef7840c1a85d4339e1f12979 127f875b4cfb4cfea1d3bf4c55f21fca f4376e357c12439db40d01933ef648b2
::./stack.sh:760+get_field 1
::./stack.sh:760+grep ' 40d9db7cef7840c1a85d4339e1f12979 '
::./stack.sh:598+local data field
::./stack.sh:599+read data
::./stack.sh:760+openstack role list --user 127f875b4cfb4cfea1d3bf4c55f21fca --project f4376e357c12439db40d01933ef648b2 --column ID --column Name --os-identity-api-version=3
ERROR: openstack No project with a name or ID of 'f4376e357c12439db40d01933ef648b2' exists.
:./stack.sh:760+local user_role_id=
:./stack.sh:761+[[ -z '' ]]
::./stack.sh:768+get_field 2
::./stack.sh:598+local data field
::./stack.sh:768+grep ' id '
::./stack.sh:599+read data
::./stack.sh:768+openstack role add 40d9db7cef7840c1a85d4339e1f12979 --user 127f875b4cfb4cfea1d3bf4c55f21fca --project f4376e357c12439db40d01933ef648b2 --os-identity-api-version=3
ERROR: openstack No role with a name or ID of '40d9db7cef7840c1a85d4339e1f12979' exists.
Cause: The first "ERROR" is ok if you see the code. Looks like the newly create role 40d9db7cef7840c1a85d4339e1f12979 doesn't get enough time to write into db. But a manual role list says it is already added.

$ openstack role list --os-token=secrete --os-url=http://10.12.181.120:5000/v3 --os-identity-api-version=3
+----------------------------------+-----------------+
| ID                               | Name            |
+----------------------------------+-----------------+
| 122db501c75d467084c2569628257f77 | anotherrole     |
| 40d9db7cef7840c1a85d4339e1f12979 | admin           |
| 4e889676b5b245b99a2d613cf2332088 | ResellerAdmin   |
| 7c6f1796f0b744d4aecdb78fe18ed833 | Member          |
| 8cf25f2a9c034adf91677147fc438bef | heat_stack_user |
| df0d13a639f8437d827cf7fcc5e4e72d | service         |
+----------------------------------+-----------------+
Solution: re-install devstack again.

n-sch fails to start: no module named simple

Symptom, in /opt/stack/logs/screen/screen-n-sch.log

/usr/local/bin/nova-scheduler --config-file /etc/nova/nova.conf
2015-05-08 00:44:04.648 4677 DEBUG nova.servicegroup.api [-] ServiceGroup driver defined as an instance of db __init__ /opt/stack/nova/nova/servicegroup/api.py:68
2015-05-08 00:44:04.863 4677 INFO nova.openstack.common.periodic_task [-] Skipping periodic task _periodic_update_dns because its interval is negative
2015-05-08 00:44:04.890 4677 CRITICAL nova [-] ImportError: No module named simple
2015-05-08 00:44:04.890 4677 TRACE nova Traceback (most recent call last):
2015-05-08 00:44:04.890 4677 TRACE nova   File "/usr/local/bin/nova-scheduler", line 10, in <module>
2015-05-08 00:44:04.890 4677 TRACE nova     sys.exit(main())
2015-05-08 00:44:04.890 4677 TRACE nova   File "/opt/stack/nova/nova/cmd/scheduler.py", line 44, in main
2015-05-08 00:44:04.890 4677 TRACE nova     topic=CONF.scheduler_topic)
2015-05-08 00:44:04.890 4677 TRACE nova   File "/opt/stack/nova/nova/service.py", line 277, in create
2015-05-08 00:44:04.890 4677 TRACE nova     db_allowed=db_allowed)
2015-05-08 00:44:04.890 4677 TRACE nova   File "/opt/stack/nova/nova/service.py", line 148, in __init__
2015-05-08 00:44:04.890 4677 TRACE nova     self.manager = manager_class(host=self.host, *args, **kwargs)
2015-05-08 00:44:04.890 4677 TRACE nova   File "/opt/stack/nova/nova/scheduler/manager.py", line 63, in __init__
2015-05-08 00:44:04.890 4677 TRACE nova     self.driver = importutils.import_object(scheduler_driver)
2015-05-08 00:44:04.890 4677 TRACE nova   File "/usr/local/lib/python2.7/dist-packages/oslo_utils/importutils.py", line 38, in import_object
2015-05-08 00:44:04.890 4677 TRACE nova     return import_class(import_str)(*args, **kwargs)
2015-05-08 00:44:04.890 4677 TRACE nova   File "/usr/local/lib/python2.7/dist-packages/oslo_utils/importutils.py", line 27, in import_class
2015-05-08 00:44:04.890 4677 TRACE nova     __import__(mod_str)
2015-05-08 00:44:04.890 4677 TRACE nova ImportError: No module named simple
2015-05-08 00:44:04.890 4677 TRACE nova
n-sch failed to start
Cause: you must be following the devstack official guide

SCHEDULER=nova.scheduler.simple.SimpleScheduler
However, nova.scheduler.simple.SimpleScheduler doesn't exist actually. Solution: don't add this SimpleScheduler line.

#SCHEDULER=nova.scheduler.simple.SimpleScheduler
route add -net ... raises "SIOCADDRT: File exists"

Symptom

sudo route add -net 192.168.120.0/24 gw 10.12.181.226
SIOCADDRT: File exists
Cause: the route is added multiple times. Solution: remove the route 192.168.120.0 before reinstall devstack.

sudo route delete -net 192.168.120.0/24 gw 10.12.181.226
Added to my cleanup script.

Warning: pvs: Ignoring duplicate config node: global_filter

Symptom

sudo pvs --noheadings -o name
WARNING: Ignoring duplicate config node: global_filter (seeking global_filter)
WARNING: Ignoring duplicate config node: global_filter (seeking global_filter)
WARNING: Ignoring duplicate config node: global_filter (seeking global_filter)
WARNING: Ignoring duplicate config node: global_filter (seeking global_filter)
Cause: you have multiple line of 'filter = ' in lvm config file. For example, here is what I found in my /etc/lvm/lvm.conf

 # global_filter = []
    global_filter = [ "a|loop2|", "a|loop3|", "a|loop4|", "r|.*|" ]
    global_filter = [ "a|loop2|", "a|loop3|", "a|loop4|", "r|.*|" ]
    global_filter = [ "a|loop2|", "a|loop3|", "a|loop4|", "r|.*|" ]
    global_filter = [ "a|loop2|", "a|loop3|", "a|loop4|", "r|.*|" ]
Solution: delete duplicated lines.

ERROR: openstack No group with a name or ID of '425c537ceb8b4a5391692452ef2f64bb' exists

Symptom

openstack role list --group 425c537ceb8b4a5391692452ef2f64bb --project df5b59544ea84147addf09237a7fe0b0 --column ID --column Name --os-identity-api-version=3
ERROR: openstack No group with a name or ID of '425c537ceb8b4a5391692452ef2f64bb' exists.
Cause: use --verbose --debug to find out what's happening. Look at below

$ openstack role list --group 5bd604db63d34b158cec7ede42ff27a1 --project f636a88775b749d5aa71c7c585540d3e --column ID --column Name --os-identity-api-version=3 --verbose --debug
...
service_provider_endpoint='', timing=False, token='secrete', trust_id='', url='http://10.12.181.120:35357/v2.0', user_domain_id='', user_domain_name='', user_id='', username='', verbose_level=3, verify=False)
...
DEBUG: openstackclient.shell cloud cfg: {'auth_type': 'token_endpoint', 'compute_api_version': '2', 'region_name': '', 'volume_api_version': '1', 'insecure': False, 'auth': {'url': 'http://10.12.181.120:35357/v2.0', 'token': 'secrete'}, 'default_domain': 'default', 'timing': False, 'network_api_version': '2', 'object_api_version': '1', 'image_api_version': '1', 'verify': False, 'identity_api_version': '3', 'verbose_level': 3, 'deferred_help': False, 'debug': True}
...
DEBUG: keystoneclient.session REQ: curl -g -i -X GET http://10.12.181.120:35357/v2.0/groups/5bd604db63d34b158cec7ede42ff27a1 -H "User-Agent: python-keystoneclient" -H "Accept: application/json" -H "X-Auth-Token: {SHA1}e78608b0aa589f6a36e3e6fb9a720cece3163496"
...
CommandError: No group with a name or ID of '5bd604db63d34b158cec7ede42ff27a1' exists.
Note that I'm trying to force python-openstackclient to use keystone api V3, by --os-identity-api-version=3. However, the auth url http://10.12.181.120:35357/v2.0 is still v2. This makes python-openstackclient throws a "No group ..." error.

Solution: when you want to use keystone V3 api, you should always one of the pair

# Switch to V3 by environment variable
export OS_URL=http://10.12.181.120:5000/v3
export OS_IDENTITY_API_VERSION=3

# Switch to V3 by command options
--os-url=http://10.12.181.120:5000/v3
--os-identity-api-version=3
ERROR: openstack No group with a name or ID of 'XDG_SESSION_ID=54' exists

Symptom

get_or_add_group_project_role bdc3408336f647f5bf858e01ca2d4bd3 XDG_SESSION_ID=54 WHEELHOUSE=/opt/stack/.wheelhouse ...
...
openstack role list --group XDG_SESSION_ID=54 --project WHEELHOUSE=/opt/stack/.wheelhouse --column ID --column Name --os-identity-api-version=3 --os-url=http://10.12.181.120:5000/v3
ERROR: openstack No group with a name or ID of 'XDG_SESSION_ID=54' exists.
Cause: the get_or_add_group_project_role is taking don't know what argument. This is because I put env in functions to dump environment variables. But the output is treated as return value, then taken by get_or_add_group_project_role as arguments. For example:

function get_or_create_group {
    local domain=${2:+--domain ${2}}
    local desc="${3:-}"
    local os_url="$KEYSTONE_SERVICE_URI_V3"
    env    # debug
    # Gets group id
    local group_id=$(
        # Creates new group with --or-show
        openstack --os-token=$OS_TOKEN --os-url=$os_url \
            --os-identity-api-version=3 group create $1 \
            $domain --description "$desc" --or-show \
            -f value -c id
    )
    echo $group_id
}
Solution: remove the debug commands in function body.

ERROR: openstack No service with a type, name or ID of 's3' exists

Symptom

2015-05-14 23:39:30.253 | ::~/devstack/tools/create_userrc.sh:137+openstack endpoint show -f value -c publicurl s3
2015-05-14 23:39:33.340 | ERROR: openstack No service with a type, name or ID of 's3' exists.
Cause: This is because S3 service is not enabled. Take a look at lib/nova:375.

Solutoin: enable swift3 service.

enable_service swift3
Added to my local.conf.

ERROR (EndpointNotFound): publicURL endpoint for compute service in RegionOne region not found

Symptom: my controller node installation succeeds. However compute node reports

:::./stack.sh:197+nova flavor-list
ERROR (EndpointNotFound): publicURL endpoint for compute service in RegionOne region not found
Just after controller is installed, and compute node not installed. Run nova flavor-list on controller node is OK. After compute node installs, both controller node and comupte node reports above error.

Cause: compute node installed keystone again. Note that devstack on default install a servies of default services, see devstack/stackrc line 48

# If you don't set ENABLED_SERVICES, devstack installs a series of default services
if ! isset ENABLED_SERVICES ; then
    # Keystone - nothing works without keystone
    ENABLED_SERVICES=key
    # Nova - services to support libvirt based openstack clouds
    ENABLED_SERVICES+=,n-api,n-cpu,n-net,n-cond,n-sch,n-novnc,n-crt
    # Glance services needed for Nova
    ENABLED_SERVICES+=,g-api,g-reg
    # Cinder
    ENABLED_SERVICES+=,c-sch,c-api,c-vol
    # Dashboard
    ENABLED_SERVICES+=,horizon
    # Additional services
    ENABLED_SERVICES+=,rabbit,tempest,mysql,dstat
fi
Usually I use enable_service to enable a service, what is its relation with ENABLED_SERVICES? The enable_service adds service to ENABLED_SERVICES. See devstack/functions-common line 1667

function enable_service {
    local tmpsvcs="${ENABLED_SERVICES}"
    local service
    for service in $@; do
        if ! is_service_enabled $service; then
            tmpsvcs+=",$service"
        fi
    done
    ENABLED_SERVICES=$(_cleanup_service_list "$tmpsvcs")
    disable_negated_services
}
So if you don't write ENABLED_SERVICES explicitly in local.conf, devstack will install default services.

Solution: explicitly specify ENABLED_SERVICE. Merged in my local.conf above.

Cannot authenticate through swift client

Symptom: After controller node installed, cannot authenticate through swift client

$ source accrc/admin/admin
$ swift list
Endpoint for object-store not found - have you specified a region?
Cause: check swift config I find it uses user swift and tenant service

$ cat /etc/swift/proxy-server.conf
[filter:keystoneauth]
...
admin_user = swift
admin_tenant_name = service
admin_password = secrete
...
However, keystone doesn't has swift user. Keystone doesn't have object-store service, either.

$ source accrc/admin/admin
$ keystone user-list
+----------------------------------+----------------------------------+---------+----------------------------------------------+
|                id                |               name               | enabled |                    email                     |
+----------------------------------+----------------------------------+---------+----------------------------------------------+
| c40195e8f7bf4b96b6d96497b712f14e |              admin               |   True  |                                              |
| eaa5205fd7b54a23a73c0c71b5af01de |             alt_demo             |   True  |             [email protected]             |
| 93276360fe014b33b8d41fe6ed5bf7c6 |            ceilometer            |   True  |                                              |
| a7ae0298ed054d30a0e249837d053a3e |              cinder              |   True  |                                              |
| 5beb0900ae6e4d9b8bd8deb1976c3799 |               demo               |   True  |               [email protected]               |
| 268dbc01b733425fb01bf0dc9bb8cc48 |              glance              |   True  |                                              |
| 914a2945c2604cf0a888af2c7ab75cf8 |           glance-swift           |   True  |           [email protected]           |
| 65fba8c2200b4bd597b4f5013a963641 |               heat               |   True  |                                              |
| 7b53803da68e42dbb1c3ca4b0c3ce8fd |             neutron              |   True  |                                              |
| 03de71cfaed24fefbba070c0fbdb42f8 |               nova               |   True  |                                              |
| cb0577eecfd14b30b865b09b89d48e01 | verify_tempest_config-1054526126 |   True  | [email protected] |
+----------------------------------+----------------------------------+---------+----------------------------------------------+
$ keystone service-list
+----------------------------------+------------+----------------+-----------------------------+
|                id                |    name    |      type      |         description         |
+----------------------------------+------------+----------------+-----------------------------+
| 6dba83f0ce15479b865d555856c2ae6b | ceilometer |    metering    | OpenStack Telemetry Service |
| d7065bd18b6a48ee93ded600ba46f24d |   cinder   |     volume     |    Cinder Volume Service    |
| 51e8c16252824f308057804fef5af887 |  cinderv2  |    volumev2    |   Cinder Volume Service V2  |
| 3f11529fb0664b9ea402c24357283c96 |    ec2     |      ec2       |   EC2 Compatibility Layer   |
| 6454affd1c7445eaaf24da37a0fbeb65 |   glance   |     image      |     Glance Image Service    |
| 1d95168b965540379a2052e8d87364f4 |    heat    | orchestration  |  Heat Orchestration Service |
| c0c77fc0e0334aeda4c0253f8dfede99 |  heat-cfn  | cloudformation | Heat CloudFormation Service |
| f66252ab764040aebb9845b020a342c6 |  keystone  |    identity    |  Keystone Identity Service  |
| c2dd7ae47c394d34b1e1a92fade588a9 |  neutron   |    network     |       Neutron Service       |
| 9f07331af66e4ba28bda3da908b2e6d2 |    nova    |    compute     |     Nova Compute Service    |
| 8cf19cea035f40df9c170c677b26ccef |  novav21   |   computev21   |  Nova Compute Service V2.1  |
| 64c4225d886b4e70aa2cee06f561db92 |     s3     |       s3       |              S3             |
+----------------------------------+------------+----------------+-----------------------------+
Solution: thanks to Danlzack's answer, let's manually add swift endpoint and service.

# On controller node, after devstack installation finished, before you install compute node
source accrc/admin/admin
keystone user-create --name=swift --pass=secrete [email protected]
keystone user-role-add --user=swift --tenant=service --role=admin
keystone service-create --name=swift --type=object-store --description="Object Storage Service"
keystone endpoint-create \
  --service-id=$(keystone service-list | awk '/ object-store / {print $2}') \
  --publicurl='http://10.12.181.120:8080/v1/AUTH_%(tenant_id)s' \
  --internalurl='http://10.12.181.120:8080/v1/AUTH_%(tenant_id)s' \
  --adminurl=http://10.12.181.120:8080 \
  --region RegionOne
ERROR: openstack Missing parameter(s)

Symptom: on compute node

2015-05-21 04:29:24.147 | ::./stack.sh:729+local os_cmd=openstack
2015-05-21 04:29:24.147 | ::./stack.sh:730+local domain=
2015-05-21 04:29:24.147 | ::./stack.sh:731+[[ ! -z '' ]]
2015-05-21 04:29:24.147 | :::./stack.sh:740+openstack project create alt_demo --or-show -f value -c id
2015-05-21 04:29:25.470 | ERROR: openstack Missing parameter(s):
2015-05-21 04:29:25.470 | Set a username with --os-username, OS_USERNAME, or auth.username
2015-05-21 04:29:25.470 | Set an authentication URL, with --os-auth-url, OS_AUTH_URL or auth.auth_url
2015-05-21 04:29:25.470 | Set a scope, such as a project or domain, with --os-project-name, OS_PROJECT_NAME or auth.project_name
2015-05-21 04:29:25.499 | ::./stack.sh:738+local project_id=
2015-05-21 04:29:25.500 | ::./stack.sh:739+echo
Cause: I dumped the environment variables when executing the openstack project create, found missing

OS_PASSWORD=secrete
OS_AUTH_URL=http://10.12.181.120:35357/v2.0
OS_TENANT_NAME=admin
OS_USERNAME=admin
Compare this compute node with controller node, I found these environment variables should be imported at stack.sh line 1010

if is_service_enabled keystone; then
    ...
    export OS_AUTH_URL=$SERVICE_ENDPOINT
    export OS_TENANT_NAME=admin
    export OS_USERNAME=admin
    export OS_PASSWORD=$ADMIN_PASSWORD
    export OS_REGION_NAME=$REGION_NAME
fi
So because I won't install keystone again on compute node, so devstack didn't export the necessary environment variables.

Solution: before install the compute node, source accrc/admin/admin which copied from controller node.

euca-bundle-image: error: certificate file does not exist

Symptom: On compute node, devstack reports below error

2015-05-21 05:34:37.566 | ::./stack.sh:1062+real_install_package euca2ools
2015-05-21 05:34:37.567 | ::./stack.sh:1047+is_ubuntu
2015-05-21 05:34:37.567 | ::./stack.sh:380+[[ -z deb ]]
2015-05-21 05:34:37.567 | ::./stack.sh:383+'[' deb = deb ']'
2015-05-21 05:34:37.567 | ::./stack.sh:1048+apt_get install euca2ools
2015-05-21 05:34:37.572 | ::./stack.sh:877+sudo DEBIAN_FRONTEND=noninteractive http_proxy= https_proxy= no_proxy= apt-get --option Dpkg::Options::=--force-confold --assume-yes install
 euca2ools
2015-05-21 05:34:37.587 | Reading package lists...
2015-05-21 05:34:37.893 | Building dependency tree...
2015-05-21 05:34:37.894 | Reading state information...
2015-05-21 05:34:38.144 | euca2ools is already the newest version.
2015-05-21 05:34:38.145 | 0 upgraded, 0 newly installed, 0 to remove and 84 not upgraded.
2015-05-21 05:34:38.146 | ::./stack.sh:581+source /home/labadmin/workspace/devstack/accrc/demo/demo
2015-05-21 05:34:38.166 | :::./stack.sh:2+export EC2_ACCESS_KEY=b73139469b9e45c9bb7c51439ec59bf0
2015-05-21 05:34:38.166 | :::./stack.sh:2+EC2_ACCESS_KEY=b73139469b9e45c9bb7c51439ec59bf0
2015-05-21 05:34:38.166 | :::./stack.sh:3+export EC2_SECRET_KEY=03822bf48f0d4773a64f7d2d1c0a4f03
2015-05-21 05:34:38.167 | :::./stack.sh:3+EC2_SECRET_KEY=03822bf48f0d4773a64f7d2d1c0a4f03
2015-05-21 05:34:38.167 | :::./stack.sh:4+export EC2_URL=http://localhost:8773/
2015-05-21 05:34:38.167 | :::./stack.sh:4+EC2_URL=http://localhost:8773/
2015-05-21 05:34:38.167 | :::./stack.sh:5+export S3_URL=http://localhost:3333
2015-05-21 05:34:38.168 | :::./stack.sh:5+S3_URL=http://localhost:3333
2015-05-21 05:34:38.168 | :::./stack.sh:7+export OS_USERNAME=demo
2015-05-21 05:34:38.168 | :::./stack.sh:7+OS_USERNAME=demo
2015-05-21 05:34:38.168 | :::./stack.sh:9+export OS_TENANT_NAME=demo
2015-05-21 05:34:38.168 | :::./stack.sh:9+OS_TENANT_NAME=demo
2015-05-21 05:34:38.168 | :::./stack.sh:10+export OS_AUTH_URL=http://10.12.181.120:35357/v2.0
2015-05-21 05:34:38.168 | :::./stack.sh:10+OS_AUTH_URL=http://10.12.181.120:35357/v2.0
2015-05-21 05:34:38.169 | :::./stack.sh:11+export OS_CACERT=
2015-05-21 05:34:38.169 | :::./stack.sh:11+OS_CACERT=
2015-05-21 05:34:38.169 | :::./stack.sh:12+export EC2_CERT=/home/labadmin/workspace/devstack/accrc/demo/demo-cert.pem
2015-05-21 05:34:38.169 | :::./stack.sh:12+EC2_CERT=/home/labadmin/workspace/devstack/accrc/demo/demo-cert.pem
2015-05-21 05:34:38.170 | :::./stack.sh:13+export EC2_PRIVATE_KEY=/home/labadmin/workspace/devstack/accrc/demo/demo-pk.pem
2015-05-21 05:34:38.170 | :::./stack.sh:13+EC2_PRIVATE_KEY=/home/labadmin/workspace/devstack/accrc/demo/demo-pk.pem
2015-05-21 05:34:38.170 | :::./stack.sh:14+export EC2_USER_ID=42
2015-05-21 05:34:38.171 | :::./stack.sh:14+EC2_USER_ID=42
2015-05-21 05:34:38.171 | :::./stack.sh:15+export EUCALYPTUS_CERT=/home/labadmin/workspace/devstack/accrc/cacert.pem
2015-05-21 05:34:38.171 | :::./stack.sh:15+EUCALYPTUS_CERT=/home/labadmin/workspace/devstack/accrc/cacert.pem
2015-05-21 05:34:38.171 | :::./stack.sh:16+export NOVA_CERT=/home/labadmin/workspace/devstack/accrc/cacert.pem
2015-05-21 05:34:38.171 | :::./stack.sh:16+NOVA_CERT=/home/labadmin/workspace/devstack/accrc/cacert.pem
2015-05-21 05:34:38.171 | :::./stack.sh:17+export OS_PASSWORD=secrete
2015-05-21 05:34:38.171 | :::./stack.sh:17+OS_PASSWORD=secrete
2015-05-21 05:34:38.172 | ::./stack.sh:582+euca-bundle-image -r x86_64 -i /home/labadmin/workspace/devstack/files/images/cirros-0.3.2-x86_64-uec/cirros-0.3.2-x86_64-vmlinuz --kernel t
rue -d /home/labadmin/workspace/devstack/files/images/s3-materials/cirros-0.3.2
2015-05-21 05:34:38.460 | usage: euca-bundle-image -i FILE [-p PREFIX] -r {i386,x86_64,armhf} [-c FILE]
2015-05-21 05:34:38.460 |                          [-k FILE] [-u ACCOUNT] [--region USER@REGION]
2015-05-21 05:34:38.461 |                          [--ec2cert FILE] [--kernel IMAGE] [--ramdisk IMAGE]
2015-05-21 05:34:38.461 |                          [-B VIRTUAL1=DEVICE1,VIRTUAL2=DEVICE2,...] [-d DIR]
2015-05-21 05:34:38.461 |                          [--productcodes CODE1,CODE2,...] [--debug]
2015-05-21 05:34:38.461 |                          [--debugger] [--version] [-h]
2015-05-21 05:34:38.461 | euca-bundle-image: error: certificate file '/home/labadmin/workspace/devstack/accrc/demo/demo-cert.pem' does not exist
Cause: The controller node generates the certificate file, but compute didn't. Looks like it is because some services are not installed on compute node, so the generation is not triggered.

Solution: Manually copy accrc folder to each compute node.

c-vol fails to start: ArgumentError: Could not parse rfc1738 URL from string ''

Symptom: On compute node, stach.sh reports "Service c-vol is not running", /opt/stack/logs/screen/screen-c-vol.log shows

/usr/local/bin/cinder-volume --config-file /etc/cinder/cinder.conf
/opt/stack/cinder/cinder/openstack/common/service.py:38: DeprecationWarning: The oslo namespace package is deprecated. Please use oslo_config instead.
  from oslo.config import cfg
2015-05-20 22:51:43.073 2849 DEBUG oslo_db.api [-] Loading backend 'sqlalchemy' from 'cinder.db.sqlalchemy.api' _load_backend /usr/local/lib/python2.7/dist-packages/oslo_db/api.py:214
2015-05-20 22:51:43.350 2849 CRITICAL cinder [-] ArgumentError: Could not parse rfc1738 URL from string ''
2015-05-20 22:51:43.350 2849 TRACE cinder Traceback (most recent call last):
2015-05-20 22:51:43.350 2849 TRACE cinder   File "/usr/local/bin/cinder-volume", line 10, in <module>
2015-05-20 22:51:43.350 2849 TRACE cinder     sys.exit(main())
2015-05-20 22:51:43.350 2849 TRACE cinder   File "/opt/stack/cinder/cinder/cmd/volume.py", line 72, in main
2015-05-20 22:51:43.350 2849 TRACE cinder     binary='cinder-volume')
2015-05-20 22:51:43.350 2849 TRACE cinder   File "/opt/stack/cinder/cinder/service.py", line 249, in create
2015-05-20 22:51:43.350 2849 TRACE cinder     service_name=service_name)
2015-05-20 22:51:43.350 2849 TRACE cinder   File "/opt/stack/cinder/cinder/service.py", line 129, in __init__
2015-05-20 22:51:43.350 2849 TRACE cinder     *args, **kwargs)
2015-05-20 22:51:43.350 2849 TRACE cinder   File "/opt/stack/cinder/cinder/volume/manager.py", line 195, in __init__
2015-05-20 22:51:43.350 2849 TRACE cinder     *args, **kwargs)
2015-05-20 22:51:43.350 2849 TRACE cinder   File "/opt/stack/cinder/cinder/manager.py", line 130, in __init__
2015-05-20 22:51:43.350 2849 TRACE cinder     super(SchedulerDependentManager, self).__init__(host, db_driver)
2015-05-20 22:51:43.350 2849 TRACE cinder   File "/opt/stack/cinder/cinder/manager.py", line 80, in __init__
2015-05-20 22:51:43.350 2849 TRACE cinder     super(Manager, self).__init__(db_driver)
2015-05-20 22:51:43.350 2849 TRACE cinder   File "/opt/stack/cinder/cinder/db/base.py", line 42, in __init__
2015-05-20 22:51:43.350 2849 TRACE cinder     self.db.dispose_engine()
2015-05-20 22:51:43.350 2849 TRACE cinder   File "/opt/stack/cinder/cinder/db/api.py", line 80, in dispose_engine
2015-05-20 22:51:43.350 2849 TRACE cinder     if 'sqlite' not in IMPL.get_engine().name:
2015-05-20 22:51:43.350 2849 TRACE cinder   File "/opt/stack/cinder/cinder/db/sqlalchemy/api.py", line 85, in get_engine
2015-05-20 22:51:43.350 2849 TRACE cinder     facade = _create_facade_lazily()
2015-05-20 22:51:43.350 2849 TRACE cinder   File "/opt/stack/cinder/cinder/db/sqlalchemy/api.py", line 72, in _create_facade_lazily
2015-05-20 22:51:43.350 2849 TRACE cinder     **dict(CONF.database.iteritems())
2015-05-20 22:51:43.350 2849 TRACE cinder   File "/usr/local/lib/python2.7/dist-packages/oslo_db/sqlalchemy/session.py", line 796, in __init__
2015-05-20 22:51:43.350 2849 TRACE cinder     **engine_kwargs)
2015-05-20 22:51:43.350 2849 TRACE cinder   File "/usr/local/lib/python2.7/dist-packages/oslo_db/sqlalchemy/session.py", line 376, in create_engine
2015-05-20 22:51:43.350 2849 TRACE cinder     url = sqlalchemy.engine.url.make_url(sql_connection)
2015-05-20 22:51:43.350 2849 TRACE cinder   File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/url.py", line 176, in make_url
2015-05-20 22:51:43.350 2849 TRACE cinder     return _parse_rfc1738_args(name_or_url)
2015-05-20 22:51:43.350 2849 TRACE cinder   File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/url.py", line 225, in _parse_rfc1738_args
2015-05-20 22:51:43.350 2849 TRACE cinder     "Could not parse rfc1738 URL from string '%s'" % name)
2015-05-20 22:51:43.350 2849 TRACE cinder ArgumentError: Could not parse rfc1738 URL from string ''
2015-05-20 22:51:43.350 2849 TRACE cinder
c-vol failed to start
Check out the cinder conf file /etc/cinder/cinder.conf

...
[database]
connection =
...
The db connection string is empty. That's why cinder fails to start. Same issue found in nova.conf. The db connection string should be generated by

# lib/cinder
iniset $CINDER_CONF database connection `database_connection_url cinder`

# lib/database
function database_connection_url {
    local db=$1
    database_connection_url_$DATABASE_TYPE $db
}
Looks like the DATABASE_TYPE is not assigned since compute node doesn't install mysql. Also found below in log.

2015-05-21 05:49:42.876 | ::./stack.sh:205+database_connection_url cinder
2015-05-21 05:49:42.876 | ::./stack.sh:125+local db=cinder
2015-05-21 05:49:42.876 | ::./stack.sh:126+database_connection_url_ cinder
2015-05-21 05:49:42.876 | /home/labadmin/workspace/devstack/lib/database: line 126: database_connection_url_: command not found
Solution: Add DATABASE_TYPE=mysql into local.conf. Merged to my local.conf above. There is a doc mentioned adding DATABASE_TYPE into compute local.conf.