Operating the Chef Server at Scale - ChefConf Workshop 2016

1important_links.md

Chef Server

• The Chef Server runbook: https://docs.chef.io/runbook.html
• Tuning the Chef Server for scale: http://irvingpop.github.io/blog/2015/04/20/tuning-the-chef-server-for-scale
• pgAdmin 3: https://www.pgadmin.org/

Linux

• Brendan Gregg - Linux Performance: http://www.brendangregg.com/linuxperf.html
• Brendan Gregg - USE method: http://www.brendangregg.com/usemethod.html

Postgres

• Introduction to Postgres for Sysadmins: http://www.slideshare.net/jkshah/introduction-to-postgresql-for-system-administrators
• EXPLAIN visualizer: https://explain.depesz.com/
• Pgtune: http://pgfoundry.org/projects/pgtune
• Pgbadger: http://dalibo.github.io/pgbadger/

Solr

• Solr performance tuning: https://wiki.apache.org/solr/SolrPerformanceProblems
• GCViewer: https://github.com/chewiebug/GCViewer/wiki

Fixie and REPLs

• https://github.com/chef/fixie/blob/master/doc/GETTING_STARTED.md
• REPL driven development with pry: https://www.youtube.com/watch?v=D9j_Mf91M0I
• Using chef/rest as a REPL: https://gist.github.com/irvingpop/a8188a111e1d641d4476

Erlang tracing

• Collecting erchef and bifrost metrics via rpc: https://gist.github.com/irvingpop/679e9ce3eaeda2602f6b
• https://github.com/massemanet/eper/

2postgres_better_size_report.sql

SELECT nspname || '.' || relname AS "relation",
    pg_size_pretty(pg_relation_size(C.oid)) AS "size"
  FROM pg_class C
  LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace)
  WHERE nspname NOT IN ('pg_catalog', 'information_schema')
  ORDER BY pg_relation_size(C.oid) DESC
  LIMIT 20;

backendless-chef-server.rb

# DNS name for load balancer VIP
api_fqdn "chef.example.com"

# Specify that postgresql is an external database, and provide the
# VIP of this cluster.  This prevents the chef-server instance
# from creating it's own local postgresql instance.
postgresql['external'] = true
# Point vip at a VIP/load-balancer that automatically routes to the master node
# NOTE: for AWS RDS use the RDS hostname
postgresql['vip'] = '172.28.128.3'
postgresql['db_superuser'] = 'chef_pgsql'
postgresql['db_superuser_password'] = '1351494fc1d3806e688536781ae9fdc044eda4c932c0da848436832e0813a9c961ccd2f7b7191f0c4bd2c4d841c0150005da'

# These settings ensure that we use remote elasticsearch
# instead of local solr for search.  This also
# set search_queue_mode to 'batch' to remove the indexing
# dependency on rabbitmq, which is not supported in this HA configuration.
opscode_solr4['external'] = true
# Point external_url at a load balancer that evenly distributes requests to all active ElasticsSearch nodes
opscode_solr4['external_url'] = 'http://172.28.128.3:9200'
opscode_erchef['search_provider'] = 'elasticsearch'
opscode_erchef['search_queue_mode'] = 'batch'

# HA mode requires sql-backed storage for bookshelf.
bookshelf['storage_type'] = :sql

# RabbitMQ settings

# At this time we are not providing a rabbit backend. Note that this makes
# this incompatible with reporting and analytics unless you're bringing in
# an external rabbitmq.
rabbitmq['enable'] = false
rabbitmq['management_enabled'] = false
rabbitmq['queue_length_monitor_enabled'] = false

# Opscode Expander
#
# opscode-expander isn't used when the search_queue_mode is batch.  It
# also doesn't support the elasticsearch backend.
opscode_expander['enable'] = false

# Prevent startup failures due to missing rabbit host
dark_launch['actions'] = false

# Cookbook Caching
opscode_erchef['nginx_bookshelf_caching'] = :on
opscode_erchef['s3_url_expiry_window_size'] = '50%'

gameshow1-ec-metal-config.json

{
  "provider": "ec2",
  "ec2_options": {
    "region": "us-west-2",
    "vpc_subnet": "subnet-b2bb82f4",
    "ami_id": "ami-775e4f16",
    "ssh_username": "ec2-user",
    "backend_storage_type": "ephemeral",
    "ebs_disk_size": "100",
    "use_private_ip_for_ssh": false,
    "elb": true,
    "tags": {
      "X-Project": "CSE"
    }
  },
  "default_package": "https://packages.chef.io/stable/el/7/chef-server-core-12.7.0-1.el7.x86_64.rpm",
  "manage_package":    "https://packages.chef.io/stable/el/7/chef-manage-2.4.0-1.el7.x86_64.rpm",
  "reporting_package": "https://packages.chef.io/stable/el/7/opscode-reporting-1.6.0-1.el7.x86_64.rpm",
  "analytics_package": "https://packages.chef.io/stable/el/7/opscode-analytics-1.4.0-1.el7.x86_64.rpm",
  "loadtesters": {
    "num_loadtesters": 6,
    "num_groups": 1,
    "num_containers": 1000
  },
  "packages": {
  },
  "layout": {
    "topology": "tier",
    "api_fqdn": "api.scenario1.ws",
    "manage_fqdn": "manage.scenario1.ws",
    "analytics_fqdn": "analytics.scenario1.ws",
    "default_orgname": "ponyville",
    "configuration": {
      "postgresql": {
        "max_connections": 512,
        "checkpoint_segments": 64,
        "checkpoint_completion_target": 0.9,
        "log_min_duration_statement": 500
      },
      "rabbitmq": {
        "analytics_max_length": 100000
      },
      "opscode_solr4": {
        "heap_size": 4096,
        "new_size": 256
      },
      "opscode_erchef": {
        "depsolver_worker_count": 5,
        "depsolver_timeout": 20000,
        "depsolver_pool_queue_max": 10,
        "depsolver_pooler_timeout": 100000,
        "db_pool_size": 32,
        "db_pool_queue_max": 64,
        "db_pooler_timeout": 2000,
        "authz_pooler_timeout": 2000,
        "keygen_cache_size": 1000,
        "nginx_bookshelf_caching": ":on",
        "s3_url_expiry_window_size": "100%"
      },
      "oc_chef_authz": {
        "http_init_count": 100,
        "http_max_count": 100,
        "http_queue_max": 200
        },
      "oc_bifrost": {
        "db_pool_size": 32,
        "db_pooler_timeout": 2000,
        "db_pool_queue_max": 64
      },
      "nginx": {
        "enable_non_ssl": true
      },
      "oc_id": {
        "applications": {
          "analytics": {
            "redirect_uri": "https://analytics.scenario1.ws"
          },
          "supermarket": {
            "redirect_uri": "https://default-centos-66/auth/chef_oauth2/callback"
          }
        }
      },
      "license": {
        "nodes": 100000
      }
    },
    "analytics_standalones": {
      "ws1-analytics1": {
        "hostname": "ws1-analytics1.scenario1.ws",
        "ebs_optimized": true,
        "instance_type": "m3.2xlarge",
        "bootstrap": true
      }
    },
    "frontends": {
      "ws1-frontend1": {
        "hostname": "ws1-frontend1.scenario1.ws",
        "ebs_optimized": true,
        "instance_type": "c4.xlarge"
      },
      "ws1-frontend2": {
        "hostname": "ws1-frontend2.scenario1.ws",
        "ebs_optimized": true,
        "instance_type": "c4.xlarge"
      }
    },
    "backends": {
      "ws1-backend1": {
        "hostname": "ws1-backend1.scenario1.ws",
        "ebs_optimized": true,
        "instance_type": "c3.2xlarge",
        "bootstrap": true
      }
    },
    "loadtesters": {
      "ws1-loadtester-uswest-2a": {
        "hostname": "loadtester1.scenario1.ws",
        "ami_id": "ami-6206e002",
        "ssh_username": "ubuntu",
        "vpc_subnet": "subnet-f28f7797",
        "instance_type": "m3.xlarge"
      }
    }
  }
}

gameshow2-ec-metal-config.json

{
  "provider": "ec2",
  "ec2_options": {
    "region": "us-west-2",
    "vpc_subnet": "subnet-b2bb82f4",
    "ami_id": "ami-775e4f16",
    "ssh_username": "ec2-user",
    "backend_storage_type": "ephemeral",
    "ebs_disk_size": "100",
    "use_private_ip_for_ssh": false,
    "elb": true,
    "tags": {
      "X-Project": "CSE"
    }
  },
  "default_package": "https://packages.chef.io/stable/el/7/chef-server-core-12.7.0-1.el7.x86_64.rpm",
  "manage_package":    "https://packages.chef.io/stable/el/7/chef-manage-2.4.0-1.el7.x86_64.rpm",
  "reporting_package": "https://packages.chef.io/stable/el/7/opscode-reporting-1.6.0-1.el7.x86_64.rpm",
  "analytics_package": "https://packages.chef.io/stable/el/7/opscode-analytics-1.4.0-1.el7.x86_64.rpm",
  "loadtesters": {
    "num_loadtesters": 3,
    "num_groups": 3,
    "num_containers": 1000
  },
  "packages": {
  },
  "layout": {
    "topology": "tier",
    "api_fqdn": "api.scenario2.ws",
    "manage_fqdn": "manage.scenario2.ws",
    "analytics_fqdn": "analytics.scenario2.ws",
    "default_orgname": "ponyville",
    "configuration": {
      "postgresql": {
        "max_connections": 512,
        "checkpoint_segments": 64,
        "checkpoint_completion_target": 0.9,
        "log_min_duration_statement": 500
      },
      "opscode_solr4": {
        "heap_size": 4096,
        "new_size": 256
      },
      "opscode_erchef": {
        "depsolver_worker_count": 5,
        "depsolver_timeout": 20000,
        "depsolver_pool_queue_max": 10,
        "depsolver_pooler_timeout": 100000,
        "db_pool_size": 32,
        "db_pool_queue_max": 64,
        "db_pooler_timeout": 2000,
        "authz_pooler_timeout": 2000,
        "keygen_cache_size": 1000,
        "nginx_bookshelf_caching": ":on",
        "s3_url_expiry_window_size": "100%"
      },
      "oc_chef_authz": {
        "http_init_count": 100,
        "http_max_count": 100,
        "http_queue_max": 200
        },
      "oc_bifrost": {
        "db_pool_size": 32,
        "db_pooler_timeout": 2000,
        "db_pool_queue_max": 64
      },
      "nginx": {
        "enable_non_ssl": true
      },
      "oc_id": {
        "applications": {
          "analytics": {
            "redirect_uri": "https://analytics.scenario2.ws"
          },
          "supermarket": {
            "redirect_uri": "https://default-centos-66/auth/chef_oauth2/callback"
          }
        }
      },
      "license": {
        "nodes": 100000
      }
    },
    "analytics_standalones": {
      "ws2-analytics1": {
        "hostname": "ws2-analytics1.scenario2.workshop",
        "ebs_optimized": true,
        "instance_type": "m3.2xlarge",
        "bootstrap": true
      }
    },
    "frontends": {
      "ws2-frontend1": {
        "hostname": "ws2-frontend1.scenario2.workshop",
        "ebs_optimized": true,
        "instance_type": "c4.xlarge"
      },
      "ws2-frontend2": {
        "hostname": "ws2-frontend2.scenario2.workshop",
        "ebs_optimized": true,
        "instance_type": "c4.xlarge"
      }
    },
    "backends": {
      "ws2-backend1": {
        "hostname": "ws2-backend1.scenario2.workshop",
        "ebs_optimized": true,
        "instance_type": "c3.2xlarge",
        "bootstrap": true
      }
    },
    "loadtesters": {
      "ws2-loadtester-uswest-2a": {
        "hostname": "loadtester1.scenario2.ws",
        "ami_id": "ami-6206e002",
        "ssh_username": "ubuntu",
        "vpc_subnet": "subnet-f28f7797",
        "instance_type": "m3.2xlarge"
      },
      "ws2-loadtester-uswest-2b": {
        "hostname": "loadtester1.scenario2.ws",
        "ami_id": "ami-6206e002",
        "ssh_username": "ubuntu",
        "vpc_subnet": "subnet-c1372eb5",
        "instance_type": "m3.2xlarge"
      },
      "ws2-loadtester-uswest-2c": {
        "hostname": "loadtester1.scenario2.ws",
        "ami_id": "ami-6206e002",
        "ssh_username": "ubuntu",
        "vpc_subnet": "subnet-b2bb82f4",
        "instance_type": "m3.2xlarge"
      }
    }
  }
}

gameshow3-ec-metal-config.json

{
  "provider": "ec2",
  "ec2_options": {
    "region": "us-west-2",
    "vpc_subnet": "subnet-b2bb82f4",
    "ami_id": "ami-775e4f16",
    "ssh_username": "ec2-user",
    "backend_storage_type": "ephemeral",
    "ebs_disk_size": "100",
    "use_private_ip_for_ssh": false,
    "elb": true,
    "tags": {
      "X-Project": "CSE"
    }
  },
  "default_package": "https://packages.chef.io/stable/el/7/chef-server-core-12.7.0-1.el7.x86_64.rpm",
  "manage_package":    "https://packages.chef.io/stable/el/7/chef-manage-2.4.0-1.el7.x86_64.rpm",
  "reporting_package": "https://packages.chef.io/stable/el/7/opscode-reporting-1.6.0-1.el7.x86_64.rpm",
  "analytics_package": "https://packages.chef.io/stable/el/7/opscode-analytics-1.4.0-1.el7.x86_64.rpm",
  "loadtesters": {
    "num_loadtesters": 15,
    "num_groups": 3,
    "num_containers": 1000
  },
  "packages": {
  },
  "layout": {
    "topology": "tier",
    "api_fqdn": "api.scenario3.ws",
    "manage_fqdn": "manage.scenario3.ws",
    "analytics_fqdn": "analytics.scenario3.ws",
    "default_orgname": "ponyville",
    "configuration": {
      "opscode_erchef": {
        "keygen_cache_size": 1000,
        "nginx_bookshelf_caching": ":on",
        "s3_url_expiry_window_size": "100%"
      },
      "nginx": {
        "enable_non_ssl": true
      },
      "oc_id": {
        "applications": {
          "analytics": {
            "redirect_uri": "https://analytics.scenario3.ws"
          },
          "supermarket": {
            "redirect_uri": "https://default-centos-66/auth/chef_oauth2/callback"
          }
        }
      },
      "license": {
        "nodes": 100000
      }
    },
    "x_configuration": {
      "postgresql": {
        "max_connections": 512,
        "checkpoint_segments": 64,
        "checkpoint_completion_target": 0.9,
        "log_min_duration_statement": 500
      },
      "opscode_solr4": {
        "heap_size": 4096,
        "new_size": 256
      },
      "opscode_erchef": {
        "depsolver_worker_count": 5,
        "depsolver_timeout": 20000,
        "depsolver_pool_queue_max": 10,
        "depsolver_pooler_timeout": 100000,
        "db_pool_size": 32,
        "db_pool_queue_max": 64,
        "db_pooler_timeout": 2000,
        "authz_pooler_timeout": 2000,
        "keygen_cache_size": 1000,
        "nginx_bookshelf_caching": ":on",
        "s3_url_expiry_window_size": "100%"
      },
      "oc_chef_authz": {
        "http_init_count": 100,
        "http_max_count": 100,
        "http_queue_max": 200
        },
      "oc_bifrost": {
        "db_pool_size": 32,
        "db_pooler_timeout": 2000,
        "db_pool_queue_max": 64
      },
      "nginx": {
        "enable_non_ssl": true
      },
      "oc_id": {
        "applications": {
          "analytics": {
            "redirect_uri": "https://analytics.scenario3.ws"
          },
          "supermarket": {
            "redirect_uri": "https://default-centos-66/auth/chef_oauth2/callback"
          }
        }
      },
      "license": {
        "nodes": 100000
      }
    },
    "analytics_standalones": {
      "ws3-analytics1": {
        "hostname": "ws3-analytics1.scenario3.ws",
        "ebs_optimized": true,
        "instance_type": "m3.2xlarge",
        "bootstrap": true
      }
    },
    "frontends": {
      "ws3-frontend1": {
        "hostname": "ws3-frontend1.scenario3.ws",
        "ebs_optimized": true,
        "instance_type": "c4.2xlarge"
      },
      "ws3-frontend2": {
        "hostname": "ws3-frontend2.scenario3.ws",
        "ebs_optimized": true,
        "instance_type": "c4.2xlarge"
      }
    },
    "backends": {
      "ws3-backend1": {
        "hostname": "ws3-backend1.scenario3.ws",
        "ebs_optimized": true,
        "instance_type": "c3.2xlarge",
        "bootstrap": true
      }
    },
    "loadtesters": {
      "ws3-loadtester-uswest-2a": {
        "hostname": "loadtester1.scenario3.ws",
        "ami_id": "ami-6206e002",
        "ssh_username": "ubuntu",
        "vpc_subnet": "subnet-f28f7797",
        "instance_type": "m3.2xlarge"
      },
      "ws3-loadtester-uswest-2b": {
        "hostname": "loadtester1.scenario3.ws",
        "ami_id": "ami-6206e002",
        "ssh_username": "ubuntu",
        "vpc_subnet": "subnet-c1372eb5",
        "instance_type": "m3.2xlarge"
      },
      "ws3-loadtester-uswest-2c": {
        "hostname": "loadtester1.scenario3.ws",
        "ami_id": "ami-6206e002",
        "ssh_username": "ubuntu",
        "vpc_subnet": "subnet-b2bb82f4",
        "instance_type": "m3.2xlarge"
      }
    }
  }
}