Experimental CacheWarmer which increases performance by re-using the shopware kernel

WarmUpHttpCacheFasterCommand.php

<?php
/**
 * Shopware 5
 * Copyright (c) shopware AG
 *
 * According to our dual licensing model, this program can be used either
 * under the terms of the GNU Affero General Public License, version 3,
 * or under a proprietary license.
 *
 * The texts of the GNU Affero General Public License with an additional
 * permission and of our proprietary license can be found at and
 * in the LICENSE file you have received along with this program.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 *
 * "Shopware" is a registered trademark of shopware AG.
 * The licensing of the program under the AGPLv3 does not imply a
 * trademark license. Therefore any rights, title and interest in
 * our trademarks remain entirely with us.
 */

namespace Shopware\Commands;

use Shopware\Components\HttpCache\AppCache;
use Shopware\Kernel;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\HttpFoundation\Request;

class WarmUpHttpCacheFasterCommand extends ShopwareCommand
{
    protected $shops;
    /** @var  Kernel */
    protected $kernel;
    protected $front;
    protected $requestReflection;
    protected $responseReflection;
    /** @var  OutputInterface */
    protected $output;
    /** @var  InputInterface  */
    protected $input;

    /**
     * {@inheritdoc}
     */
    protected function configure()
    {
        $this
            ->setName('sw:warm:http:cache:fast')
            ->setDescription('warm up http cache')
            ->addArgument('shopId', InputArgument::OPTIONAL, 'The Id of the shop')
            ->addOption('docroot', null, InputOption::VALUE_REQUIRED, 'The document root of your shopware installation, e.g. /var/www/html')
            ->setHelp('The <info>%command.name%</info> warms up the http cache faster by re-using the kernel');
    }

    /**
     * Create a sf request object with a given URL and the given cookies (shopId!)
     *
     * @param $url
     * @param array $cookies
     * @return Request
     */
    private function createRequestForURL($url, $cookies = [])
    {
        $docRoot = $this->input->getOption('docroot');
        $server = [];
        $server['SCRIPT_FILENAME'] = realpath(str_replace('bin/console', 'shopware.php', $_SERVER['SCRIPT_FILENAME']));
        $server['SCRIPT_NAME'] = str_replace($docRoot, '', $server['SCRIPT_FILENAME']);
        $server['PHP_SELF'] = $server['SCRIPT_NAME'];

        $request = Request::create($url, 'GET', [], $cookies, [], $server, null);
        return $request;
    }

    /**
     * Do call a given url
     *
     * @param $url
     * @param $shopId
     */
    private function callUrl($url, $shopId)
    {
        
        $cookies = [];
        if (!empty($this->shops[$shopId]["main_id"])) {
            $cookies = ['shop' => $shopId];
        }

        $request = $this->createRequestForURL($url, $cookies);

        $this->resetRequest();

        try {
            $result = $this->kernel->handle($request);
        } catch (\Throwable $e) {
            error_log("error:" . $e->getMessage());
        }
    }

    /**
     * Prepare everything for cache warming. Will create a kernel decorated by the appcache
     * and makes some protected properties visible as a temp hack
     */
    private function setUp()
    {
        // preload shops for faster access
        $shopData = $this->getContainer()->get('dbal_connection')->fetchAll(
            'SELECT * FROM s_core_shops WHERE active = 1'
        );

        foreach ($shopData as $shop) {
            $this->shops[$shop['id']] = $shop;
        }


        // create cached kernel
        $kernel = new Kernel('production', false);
        $kernel->boot();

        $cache = new AppCache($kernel, $kernel->getHttpCacheConfig());

        // will need to reset the request of the front controller after each request
        // should be fixed in core later on
        $this->front = Shopware()->Container()->get('front');

        $class = new \ReflectionClass($this->front);
        $property = $class->getProperty("request");
        $property->setAccessible(true);
        $this->requestReflection = $property;
        $property = $class->getProperty("response");
        $property->setAccessible(true);
        $this->responseReflection = $property;

        $this->kernel = $cache;
    }


    /**
     * {@inheritdoc}
     */
    protected function execute(InputInterface $input, OutputInterface $output)
    {
        $this->setUp();
        $this->output = $output;
        $this->input = $input;


        $shopIds = $this->getShopsFromInput();

        foreach ($shopIds as $shopId) {
            $this->warmShopUrls($shopId);
        }


        $output->writeln("\n The HttpCache is now warmed up");
    }

    private function resetRequest()
    {
        $this->requestReflection->setValue($this->front, null);
        $this->responseReflection->setValue($this->front, null);
    }


    /**
     * @param $shopId
     */
    protected function warmShopUrls($shopId)
    {
        $output = $this->output;
        /** @var \Shopware\Components\HttpCache\CacheWarmer $cacheWarmer */
        $cacheWarmer = $this->container->get('http_cache_warmer');

        $totalUrlCount = $cacheWarmer->getAllSEOUrlCount($shopId);
        $output->writeln("\n Calling URLs for shop with id " . $shopId);

        $progressBar = new ProgressBar($output, $totalUrlCount);
        $progressBar->setBarWidth(100);
        $progressBar->start();

        $offset = 0;
        while ($offset < $totalUrlCount) {
            $urls = $cacheWarmer->getAllSEOUrls($shopId, 10, $offset);

            foreach ($urls as $url) {
                $this->callUrl($url, $shopId);
            }
            $progressBar->advance(count($urls));
            $offset += count($urls);
        }
        $progressBar->finish();
    }

    /**
     * @return array
     */
    protected function getShopsFromInput()
    {
        $shopId = $this->input->getArgument('shopId');

        if (!empty($shopId)) {
            return [$shopId];
        }

        return $this->container->get('db')->fetchCol('SELECT id FROM s_core_shops WHERE active = 1');

    }
}

WarmUpHttpCacheFasterCommand

Highly experimental cache warmer, which is able to re-use parts of the shopware stack during cache warming - and therefore will be faster than the default cache warmer

Setting it up

• quick and dirty: Content goes to engine/Shopware/Commands/WarmUpHttpCacheFasterCommand.php.
• better: Write a plugin around it

Running it

This command has a required option "docroot", as the script cannot figure it out automatically:

sw:warm:http:cache:fast --docroot=/var/www/html

Issues

This is experimental - there might be several issues, e.g.

• Path resolving might fail. In the frontend this would show as missing images or failing ajax request.
• Shared state: By default the "shared nothing" architecture of PHP requests will make sure, that independent requests will not share any state. When re-using parts of the stack (as done here), requests will share state. Double check the generated pages!
• By its nature this will only work with the shopware reverse proxy