SeleniumConf 2017 Amplify Code Snippets

cleanup_browsers.sh

#!/bin/bash
set -euo pipefail
IFS=$'\n\t'
# Bash unofficial strict mode: http://redsymbol.net/articles/unofficial-bash-strict-mode/

# This is a fairly simple script for terminating old browsers that should have been cleaned up automatically

# Grab the current number of sessions running on the selenium grid. This uses jq to parse the response from the
# selenium grid and return the number of sessions
num_active_browsers="$(/usr/bin/curl --silent http://localhost:5555/wd/hub/sessions | \
    /usr/bin/jq --raw-output '.value | length')"

if [[ "$num_active_browsers" -eq "0" ]]; then
    # If we shouldn't have any active sessions, make it so. We do this because sometimes chrome/firefox don't close
    # properly when the grid closes them.
    /usr/bin/pkill -9 firefox chrome chrome-sandbox chromedriver
else
    # Get all of the chrome and firefox processes, kill them if they've been running for more than 30 minutes
    /bin/ps -eo pid,comm,etime | \
    /usr/bin/grep -e 'chrome\|firefox' | \
    /usr/bin/sed -nr 's|([0-9]+) .*[3-5][0-9]:[0-9]{2}$|\1|p' | \
    /usr/bin/xargs --no-run-if-empty /bin/kill
fi

install_browsers.sh

#######################################
# Install_firefox: Installs the version of firefox specified
# Arguments:
#   desired_version:
#           * latest_esr:       Gets the latest extended support release
#           * latest_beta:      Gets the latest beta release
#           * latest:           Gets the latest release
#           * version number:   Gets the version specified
#           * default:          Chooses latest_esr as default
#######################################
install_firefox() {
  ## Install firefox
  local desired_version=${1:-latest_esr}

  case "$desired_version" in
  latest_esr)
    local firefox_dl="https://download.mozilla.org/?product=firefox-esr-latest&os=linux64&lang=en-US"
    ;;
  latest_beta)
    local firefox_dl="https://download.mozilla.org/?product=firefox-esr-latest&os=linux64&lang=en-US"
    ;;
  latest)
    local firefox_dl="https://download.mozilla.org/?product=firefox-esr-latest&os=linux64&lang=en-US"
    ;;
  *)
    local firefox_dl="https://ftp.mozilla.org/pub/firefox/releases/${desired_version}/linux-x86_64/en-US/firefox-${desired_version}.tar.bz2"
    ;;
  esac

  curl --silent --show-error --location --output firefox.tar.bz2 "$firefox_dl"

  mkdir -p /opt/wgen/firefox && tar -xjf firefox.tar.bz2 --directory /opt/wgen/firefox --strip 1

  chmod -R 0755 /opt/wgen/firefox

  ln -s /opt/wgen/firefox/firefox /usr/bin/firefox

  # Install requirements for firefox without installing firefox, because we installed a specific version above due to selenium compatibility problems.
  yum install -y $(repoquery --requires firefox)
}

#######################################
# Install_geckodriver: Installs the version of gecko driver specified
# Arguments:
#   desired_version:
#           * latest:           Gets the latest stable release
#           * version number:   Gets the version specified
#           * default:          Chooses latest as default
#######################################
install_geckodriver(){

  local desired_version=${1:-latest}

  case "$desired_version" in
  latest)
    # Get the latest version of the geckodriver
    local latest_geckodriver_release="$(curl --silent --show-error 'https://api.github.com/repos/mozilla/geckodriver/releases/latest')"
    ;;
  *)
    local geckodriver_release="$(curl --silent --show-error 'https://api.github.com/repos/mozilla/geckodriver/releases/${desired_version}')"
    ;;
  esac

  local geckodriver_dl="$(echo "$latest_geckodriver_release" | jq --raw-output '.assets[] | select(.name | contains("linux64")) | .browser_download_url')"

  # Download it, extract it, and install it
  curl --silent --show-error --location --output geckodriver.tar.gz "$geckodriver_dl"
  tar -xzvf geckodriver.tar.gz

  mv geckodriver /opt/wgen/firefox/geckodriver
  chmod 0755 /opt/wgen/firefox/geckodriver
  ln -s /opt/wgen/firefox/geckodriver /usr/bin/geckodriver
}

#######################################
# Install_chrome: Installs the version of chrome specified
# Arguments:
#   desired_version:
#           * stable:           Gets the latest stable release
#           * beta:             Gets the latest beta release
#           * version number:   Gets the version specified
#           * default:          Chooses stable as default
#######################################
install_chrome() {

  local desired_version=${1:-stable}

  # Install chrome
  yum install -y --enablerepo='google-chrome' google-chrome-${desired_version}

  # Install libOSMesa needed for enabling WebGL in Chrome
  yum_install mesa-libOSMesa
  # Copy osmesa library to chrome directory to able to reference it with --use-gl flag
  cp /usr/lib64/libOSMesa.so.? /opt/google/chrome/libosmesa.so
}

#######################################
# Install_chrome_driver: Installs the version of chrome driver specified
# Arguments:
#   desired_version:
#           * LATEST_RELEASE:   Gets the latest stable release
#           * version number:   Gets the version specified
#           * default:          Chooses LATEST_RELEASE as default
#######################################
install_chrome_driver(){

  yum_install wget unzip

  local desired_version=${1:-LATEST_RELEASE}

  local chromedriver_download_site="http://chromedriver.storage.googleapis.com"

  case "$desired_version" in
  LATEST_RELEASE)
    # Install chromedriver
    local chromedriver_latest_release="$(curl --silent --show-error ${chromedriver_download_site}/${desired_version})"
    local chromedriver_dl="${chromedriver_download_site}/${chromedriver_latest_release}/chromedriver_linux64.zip"
    ;;
  *)
    local chromedriver_dl="${chromedriver_download_site}/${desired_version}/chromedriver_linux64.zip"
    ;;
  esac

  curl --silent --show-error -o chromedriver.zip "$chromedriver_dl"

  unzip -o chromedriver.zip -d /opt/selenium-grid
  chmod 755 /opt/selenium-grid/chromedriver

  ln -s /opt/selenium-grid/chromedriver /usr/bin/chromedriver
}

Install_selenium.sh

install_selenium_server_standalone() {
  local desired_version=${1:-latest}

  # Create our shell user
  init_shell_user "selenium" "*"

  local selenium_download_site="http://selenium-release.storage.googleapis.com"
  local selenium_releases=$(curl --silent --show-error "$selenium_download_site" | grep --only-matching --perl-regexp '[0-9\.-\w]+?/selenium-server-standalone.*?\.jar' | sort --reverse)

  if [ "$desired_version" == "latest" ]; then
    local selenium_release=$(echo "$selenium_releases" | head -1)
  else
    local selenium_release=$(echo "$selenium_releases" | grep -m 1 "$desired_version")
  fi

  curl -o /opt/selenium-grid/selenium-server-standalone.jar --silent --location --show-error "$selenium_download_site/$selenium_release"
}

LICENSE

Copyright 2017 Amplify Education, Inc.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

selenium_grid_datadog_integration.py

import json
import re

import requests

from collections import defaultdict, Counter
from checks import AgentCheck


class SeleniumGridHubUsageCheck(AgentCheck):
    """Class for generating Datadog metrics against a Selenium Grid Hub"""

    def get_grid_hub_sessions_capacity(self, url):
        """Get max sessions capacity of the grid"""
        hub_max_capacity = 0

        # get all max sessions info from each nodes configuration
        for node in self.get_all_registered_nodes_ip(url):
            data = {"id": node}
            response = requests.get(url + "/grid/api/proxy/", data=json.dumps(data)).json()
            hub_max_capacity += response["request"]["configuration"]["maxSession"]

        return hub_max_capacity

    def get_grid_hub_queue_length(self, url):
        """
        Gets the length of the queue for the given selenium hub
        :param url: Url to reach the Selenium Grid Hub at.
        :return: Integer indicating the length of the hub's queue.
        """
        data = {"configuration": ["newSessionRequestCount"]}
        response = requests.get(url + "/grid/api/hub", data=json.dumps(data)).json()
        return response.get("newSessionRequestCount", 0)


    def get_all_registered_nodes_ip(self, url):
        """Gets all of the ips of nodes registered to the selenium hub"""
        html_content = requests.get(url + '/grid/console').content
        # get all ips from the html of the grid console
        nodes_ip_list = re.findall('id : (http://.+:[0-9]+)', html_content)
        return nodes_ip_list

    def get_usage_per_browser_type(self, url):
        """
        Gets browser usage information for the given selenium hub
        :param url: Url to reach the Selenium Grid Hub at.
        :return: A dictionary of browser usage information. See below.

        {
            "total": 10,
            "breakdown": {
                "firefox": {
                    "version": {
                        "35.6": "10"
                    }
                }
            }
        }
        """
        nodes = []
        browser_stats_dict = {}
        browser_nums = defaultdict(int)
        browser_versions = defaultdict(list)

        # Get all sessions info for each node in nodes list
        for node_ip in self.get_all_registered_nodes_ip(url):
            nodes.append(requests.get(node_ip + "/wd/hub/sessions/").json())
        # parse dictionary to get total browser type on each node
        for node in nodes:
            for browser in node['value']:
                browser_name = browser['capabilities']['browserName']
                browser_version = browser['capabilities']['version']
                browser_nums[browser_name] += 1
                browser_versions[browser_name].append(browser_version)

        browser_stats_dict['total'] = sum(browser_nums.values())
        browser_stats_dict["breakdown"] = {}
        for browser_type in browser_versions:
            browser_stats_dict["breakdown"][browser_type] = {
                'version': {versions_number: number for versions_number, number in
                             Counter(browser_versions[browser_type]).iteritems()}
                }
        return browser_stats_dict

    def check(self, instance):
        url = instance['url']
        browser_stats_dict = self.get_usage_per_browser_type(url)
        queue_length = self.get_grid_hub_queue_length(url)
        hub_capacity = self.get_grid_hub_sessions_capacity(url)
        self.gauge('selenium_grid.hub.queue_length', queue_length)
        self.gauge('selenium_grid.hub.capacity', hub_capacity)

        num_total = float(browser_stats_dict['total']) * 100

        for browser_name in browser_stats_dict['breakdown']:
            for version in browser_stats_dict['breakdown'][browser_name]['version']:
                num_version = browser_stats_dict['breakdown'][browser_name]['version'][version]
                percentage = round(num_version / num_total)

                count = browser_stats_dict['breakdown'][browser_name]['version'][version]

                browser_tag = "browser:{0}".format(browser_name)
                version_tag = "version:{0}".format(version)

                self.gauge('selenium_grid.hub.per', percentage, tags=[browser_tag, version_tag])
                self.gauge('selenium_grid.hub.num', count, tags=[browser_tag, version_tag])

selenium_grid_datadog_integration.yaml

init_config:

instances:
  - url: http://localhost:4444