Script that (re)starts a Plone instance, loads its main page and all links within the Plone instance that are on the main page (some filters provided)

warmup_plone.py

# -*- coding: utf-8 -*-
# ----------------------------------------------------------------------------
# Script to warm up the instance.
#
# The script starts/restarts the instance and measures the time it takes the
# instance to become responsive. Afterwards it crawls the front-page for links
# and warms up these resources.
# ----------------------------------------------------------------------------

from datetime import datetime
from email.mime.text import MIMEText
from urllib2 import URLError

import lxml.html
import os
import smtplib
import subprocess
import sys
import time
import urllib2


# Constants
EMAIL_HOST = 'mailhost'
EMAIL_FROM = '[email protected]'
EMAIL_TO = '[email protected]'
EMAIL_SUBJECT = '[Freitag Server restart][{0}]'

PLONE_LOCAL_PORT = '8080'
PLONE_SUPERVISOR_PORT = '8081'
PLONE_INSTANCE = 'website'

SLEEP_SHORT = 10
SLEEP_LONG = 60
FAILED = '\033[91mFAILED\033[0m'
OK = '\033[92mOK\033[0m'
CHECK_EXISTS = [
    'der Freitag',
    'Salon',
    'Buch der Woche',
    'Filmothek',
    'Kollektion',
    'der Freitag Mediengesellschaft mbH & Co. KG',
]
CHECK_NOT_EXISTS = ['Fehler 404', ]
IGNORE_MIDDLE = [
    '@@',
    '++theme++',
    '#',
    '/front-page/',
    '?',
]
IGNORE_END = [
    '.css',
    '.js',
    '.png',
    '.jpg',
    '.jpeg',
    '.gif',
    '#',
    '/kiosk-finder',
]


class Startup():

    start_instance_time = None
    warmup_front_page_time = None
    warmup_time = None

    def __init__(self, base_url, has_supervisor=False):
        self.email_text = ''
        self.base_url = base_url
        self.has_supervisor = has_supervisor

        start = datetime.now()
        self.start_instance()
        self.start_instance_time = datetime.now() - start
        self.log('Start took: {0}.'.format(self.start_instance_time))

        start = datetime.now()
        links = self.warmup_front_page()
        self.warmup_front_page_time = datetime.now() - start
        self.log(
            'Front-page warmup took: {0}.'.format(self.warmup_front_page_time)
        )

        #self.ab(self.base_url)

        start = datetime.now()
        self.warmup(links)
        self.warmup_time = datetime.now() - start
        self.log('Warmup took: {0}'.format(self.warmup_time))

        self.elapsed_time()

        self.send_email()

    def _headline(self, headline):
        self.log('----------------------------------------\n')
        self.log(headline)
        self.log('----------------------------------------\n')

    def log(self, text):
        print text
        # remove the colorings, on a plain text mail are not seen at all
        # and just clutter the text
        for markup in ((OK, 'OK'), (FAILED, 'FAILED')):
            if markup[0] in text:
                text = text.replace(markup[0], markup[1])
        self.email_text += '\n{0}'.format(text)

    def _probing(self, url, max_attempts=10,
                 check_exists=CHECK_EXISTS,
                 check_not_exists=CHECK_NOT_EXISTS):
        i = 0
        start = datetime.now()
        while True:
            try:
                output = urllib2.urlopen(url).read()
                if [x for x in check_exists if x in output] and \
                   [x for x in check_not_exists if x not in output]:
                    elapsed = datetime.now() - start
                    self.log('Warmup: {0} [ {1} sec. ] [ {2} ]'.format(
                        url,
                        elapsed.seconds,
                        OK))
                    return output
            except URLError:
                self.log('.')
            time.sleep(SLEEP_SHORT)
            if i >= max_attempts:
                elapsed = datetime.now() - start
                self.log('Warmup: {0} [ {1} sec. ] [ {2} ]'.format(
                    url,
                    elapsed.seconds,
                    FAILED))
                break
            i += 1

    def start_instance(self):
        if self.has_supervisor:
            self._headline('Restart Supervisor')
            p = subprocess.Popen(['./bin/supervisorctl', 'status'],
                                 stdout=subprocess.PIPE)
            out, err = p.communicate()
            if 'refused connection' in out:
                self.log(
                    'Supervisor daemon not running, starting the daemon...'
                )
                time.sleep(SLEEP_LONG)
                subprocess.call(['bin/supervisord', ])
            elif 'RUNNING' in out:
                self.log('Supervisor: instance is running, restarting...')
                subprocess.call(['bin/supervisorctl', 'shutdown'])
                time.sleep(SLEEP_LONG)
                subprocess.call(['bin/supervisord', ])
            elif 'STOPPED' in out:
                self.log('Supervisor: instance not running, starting...')
                time.sleep(SLEEP_LONG)
                subprocess.call(['bin/supervisorctl', 'start', 'all'])
            else:
                self.log(
                    "Don't know what state supervisor is in, exiting "
                    'start_instance!'
                )
                return
        elif os.path.exists('var/instance.pid'):
            self._headline('Restart Zope Instance')
            self.restart = True
            subprocess.call(['bin/instance', 'restart'])
        else:
            self._headline('Start Zope Instance')
            self.restart = False
            subprocess.call(['bin/instance', 'start'])
        self._probing(
            self.base_url + '/kontakt',
            check_exists=['Kontakt', 'Empfänger'])

    def warmup_front_page(self):
        self._headline('Warmup Front-Page')
        output = self._probing(
            self.base_url,
            check_exists=['Das Meinungsmedium', 'section-front-page'])
        tree = lxml.html.fromstring(output)
        links = []
        for link in tree.iterlinks():
            link = link[2]
            ignore = False
            if not link.startswith(self.base_url):
                continue
            if link == self.base_url:
                continue
            for middle in IGNORE_MIDDLE:
                if not ignore and middle in link:
                    ignore = True
            if ignore:
                continue
            for end in IGNORE_END:
                if not ignore and link.endswith(end):
                    ignore = True
            if ignore:
                continue
            if link not in links:
                links.append(link)
        self.log('{0} links found on the front-page.'.format(len(links)))
        return links

    def warmup(self, urls):
        self._headline('Warmup')
        for url in urls:
            self._probing(url)

    def elapsed_time(self):
        # Compute elapsed time
        self._headline('Elapsed Time')
        self.log('Start took: {0}.'.format(self.start_instance_time))
        self.log(
            'Front-page warmup took: {0}.'.format(self.warmup_front_page_time)
        )
        self.log('Warmup took: {0}'.format(self.warmup_time))
        self.log(
            'Full script took: {0}'.format(
                self.start_instance_time +
                self.warmup_front_page_time +
                self.warmup_time)
        )

    def send_email(self):
        msg = MIMEText(self.email_text)
        msg['Subject'] = EMAIL_SUBJECT.format(os.uname()[1])
        msg['From'] = EMAIL_FROM
        msg['To'] = EMAIL_TO
        smtp = smtplib.SMTP(EMAIL_HOST)
        smtp.sendmail(EMAIL_FROM, [EMAIL_TO, ], msg.as_string())
        smtp.quit()


if __name__ == '__main__':
    arg = True
    try:
        instance = sys.argv[1]
    except IndexError:
        arg = False
        pass

    if arg and instance == 'local':
        url = 'http://localhost:{0}/{1}'.format(PLONE_LOCAL_PORT,
                                                PLONE_INSTANCE)
        Startup(url, has_supervisor=False)
    else:
        url = 'http://localhost:{0}/{1}'.format(PLONE_SUPERVISOR_PORT,
                                                PLONE_INSTANCE)
        Startup(url, has_supervisor=True)