OpenStack keystone benchmarking with nginx,apache and default wsgi

apache-keystone.conf

Listen 5000
Listen 35357

<VirtualHost *:5000>
    WSGIDaemonProcess keystone processes=20 threads=20 user=keystone display-name=%{GROUP}
    WSGIProcessGroup keystone
    WSGIScriptAlias / /usr/bin/keystone-main
    #WSGIApplicationGroup %{GLOBAL}
    #WSGIPassAuthorization On
    <IfVersion >= 2.4>
      ErrorLogFormat "%{cu}t %M"
    </IfVersion>
    ErrorLog /var/log/apache2/keystone.log
    CustomLog /var/log/apache2/keystone_access.log combined
</VirtualHost>

<VirtualHost *:35357>
    WSGIDaemonProcess keystone-admin processes=20 threads=20 user=keystone display-name=%{GROUP}
    WSGIProcessGroup keystone-admin
    WSGIScriptAlias / /usr/bin/keystone-main
    #WSGIApplicationGroup %{GLOBAL}
    #WSGIPassAuthorization On
    <IfVersion >= 2.4>
      ErrorLogFormat "%{cu}t %M"
    </IfVersion>
    ErrorLog /var/log/apache2/keystone.log
    CustomLog /var/log/apache2/keystone_access.log combined
</VirtualHost>

Architecture brief

We are using keystone as a global service and hence its centre of universe .
Any api calls internal or via customer for the OpenStack cloud has to go via keystone for auth which internally calls ldap and ,etc.
Also keystone is running on a seperate box , seperate from other OpenStack components.
Need some approvals to post architecture related data and hence just briefed as above to give some insight.

keystone-admin.ini

[uwsgi]
master = true
processes = 10
threads = 2
chmod-socket = 666

socket = /run/uwsgi/keystone-admin.socket
pidfile = /run/uwsgi/keystone-admin.pid
log-syslog = '[keystone-admin]'

name = keystone
uid = keystone
gid = www-data

chdir = /usr/bin/
wsgi-file = /usr/bin/keystone-admin

keystone-admin.py

import logging
import os
import socket
import sys

from oslo import i18n
import pbr.version


# If ../keystone/__init__.py exists, add ../ to Python search path, so that
# it will override what happens to be installed in /usr/(local/)lib/python...
possible_topdir = os.path.normpath(os.path.join(os.path.abspath(__file__),
                                   os.pardir,
                                   os.pardir))
if os.path.exists(os.path.join(possible_topdir,
                               'keystone',
                               '__init__.py')):
    sys.path.insert(0, possible_topdir)

# NOTE(dstanek): i18n.enable_lazy() must be called before
# keystone.i18n._() is called to ensure it has the desired lazy lookup
# behavior. This includes cases, like keystone.exceptions, where
# keystone.i18n._() is called at import time.
i18n.enable_lazy()


from keystone import backends
from keystone.common import dependency
from keystone.common import environment
from keystone.common import sql
from keystone import config
from keystone.openstack.common import log
from keystone import service


CONF = config.CONF

config.configure()
sql.initialize()
config.set_default_for_default_log_levels()

CONF(project='keystone')
config.setup_logging()

environment.use_stdlib()
name = os.path.basename(__file__)

if CONF.debug:
    CONF.log_opt_values(log.getLogger(CONF.prog), logging.DEBUG)


drivers = backends.load_backends()

# NOTE(ldbragst): 'application' is required in this context by WSGI spec.
# The following is a reference to Python Paste Deploy documentation
# http://pythonpaste.org/deploy/
application = service.loadapp('config:%s' % config.find_paste_config(), name)

dependency.resolve_future_dependencies()

keystone-main.ini

[uwsgi]
master = true
processes = 100
threads = 100
chmod-socket = 666

socket = /run/uwsgi/keystone-main.socket
pidfile = /run/uwsgi/keystone-main.pid
log-syslog = '[keystone-main]'

name = keystone
uid = keystone
gid = www-data

chdir = /usr/bin/
wsgi-file = /usr/bin/keystone-main

keystone-main.py

import logging
import os
import socket
import sys

from oslo import i18n
import pbr.version


# If ../keystone/__init__.py exists, add ../ to Python search path, so that
# it will override what happens to be installed in /usr/(local/)lib/python...
possible_topdir = os.path.normpath(os.path.join(os.path.abspath(__file__),
                                   os.pardir,
                                   os.pardir))
if os.path.exists(os.path.join(possible_topdir,
                               'keystone',
                               '__init__.py')):
    sys.path.insert(0, possible_topdir)

# NOTE(dstanek): i18n.enable_lazy() must be called before
# keystone.i18n._() is called to ensure it has the desired lazy lookup
# behavior. This includes cases, like keystone.exceptions, where
# keystone.i18n._() is called at import time.
i18n.enable_lazy()


from keystone import backends
from keystone.common import dependency
from keystone.common import environment
from keystone.common import sql
from keystone import config
from keystone.openstack.common import log
from keystone import service


CONF = config.CONF

config.configure()
sql.initialize()
config.set_default_for_default_log_levels()

CONF(project='keystone')
config.setup_logging()

environment.use_stdlib()
name = os.path.basename(__file__)

if CONF.debug:
    CONF.log_opt_values(log.getLogger(CONF.prog), logging.DEBUG)


drivers = backends.load_backends()

# NOTE(ldbragst): 'application' is required in this context by WSGI spec.
# The following is a reference to Python Paste Deploy documentation
# http://pythonpaste.org/deploy/
application = service.loadapp('config:%s' % config.find_paste_config(), name)

dependency.resolve_future_dependencies()

keystone.conf

[ldap]
user_name_attribute=xyz
user=xyz
user_objectclass=user
timeout=30
user_id_attribute=xyz
tls_req_cert=never
query_scope=sub
url=xyz:636
password=xyz
page_size=1000
user_tree_dn=xyz
user_additional_attribute_mapping=mail:email,company:company

[signing]
keyfile=/etc/keystone/pki/signing_key.pem
ca_certs=/etc/keystone/pki/ca.pem
certfile=/etc/keystone/pki/signing_cert.pem



[identity]
driver=keystone.identity.backends.sql.xyz
default_domain_id=default

[tfa]
enabled=true
radius_port=1645
project_tfa_key=tfa
radius_server=xyz
radius_secret=xyz

[token]
provider=keystone_plugins.token.providers.pkiz.Provider
driver=keystone.token.persistence.backends.sql.Token
expiration=86400

[DEFAULT]
log_file=keystone.log
admin_endpoint=https://xyz:443/
public_port=5000
public_workers=20
use_syslog=False
compute_port=8774
debug=True
admin_token=xyz
log_dir=/var/log/keystone
bind_host=0.0.0.0
member_role_id=xyz
public_endpoint=https://xyz:5443/
admin_port=35357
log_config=/etc/keystone/logging.conf
member_role_name=Member
admin_workers=20
verbose=True

[paste_deploy]
config_file=keystone-paste.ini

[policy]
driver=keystone.policy.backends.sql.Policy

[database]
mysql_sql_mode=ANSI
connection=mysql://keystone:xyz@xyz/keystone
idle_timeout=3600
max_overflow=20
max_pool_size=10

locustIO for testing

we did testing on  Ubuntu 12.04.2 LTS , 3.2.0-38-generic

We used locust IO to run the benchmarking, Just point it to the server running keystone and post.
e.g. foor create token, use the payload to create token with user and password and mention 5000 port when running the test
Refer to http://locust.io/ for setting up test locust 

spawning 100 users per sec for create token, below are the results:

Using nginx with uwsgi: 
rps 32    —(reqests/sec)
median time ~ 3.3 sec 
no of processes 20 

using apache 
rps 75 
median time ~ 1.3 sec 
avg time - 1.5 sec 
no of processes 20 

using wsgi 
rps 28 
median time ~ 3.4 
avg 3.5 
no of processes 20 


One can definitely change # of processes in the apache,uwsgi or nginx config

nginx-keystone.conf

server {
        listen          5000;
        access_log /var/log/nginx/keystone_access.log;
        error_log /var/log/nginx/keystone_error.log;

        location / {
            uwsgi_pass      unix:///run/uwsgi/keystone-main.socket;
            include         uwsgi_params;
            uwsgi_param      SCRIPT_NAME   main;
         }
}
server {
        listen          35357;
        access_log /var/log/nginx/keystone_access.log;
        error_log /var/log/nginx/keystone_error.log;

        location / {
            uwsgi_pass      unix:///run/uwsgi/keystone-admin.socket;
            include         uwsgi_params;
            uwsgi_param      SCRIPT_NAME   admin;

       }

Outline

Mentioned files are used to run keystone on either apache2 or nginx with uwsgi. 
Ofcourse do follow the basic configuration settings like file and folder permissions to keystone user for the above files.
Do take care of softlinks for sites-enable and sites-available including soft links fir keystone-main and admin.py files

uwsgi.conf for upstart

description "uwsgi for nginx keystone admin"


start on runlevel [2345]
stop on runlevel [!2345]

respawn

pre-start script
  if [ ! -d /run/uwsgi ]; then
      mkdir /run/uwsgi/
      chown keystone:keystone /run/uwsgi
      chmod 775 /run/uwsgi
  fi
end script

post-stop script
  if [ -d /run/uwsgi ]; then
     rm -r /run/uwsgi
  fi
end script
exec /usr/local/bin/uwsgi --master --emperor /etc/uwsgi