Deployment with deployerphp and gitlabci

.gitlab-ci.yml

cache:
  paths:
    - $CI_PROJECT_DIR/cache
    - $CI_PROJECT_DIR/vendor
    - $CI_PROJECT_DIR/.Build/vendor

stages:
  - composer
  - deploy

build:
  stage: composer
  image: composer:1
  before_script:
    - composer config -g cache-dir $CI_PROJECT_DIR/cache/composer
  script:
    - cd .Build && composer install --no-dev
  only:
    - develop
    - tags

.deploy: &deploy_template
  stage: deploy
  image: composer:1
  before_script:
    - apk add --no-cache rsync
    - mkdir -p $CI_PROJECT_DIR/cache/composer
    - mkdir -p $CI_PROJECT_DIR/cache/deployment/staging
    - mkdir -p $CI_PROJECT_DIR/cache/deployment/shared/fileadmin
    - composer config -g cache-dir $CI_PROJECT_DIR/cache/composer
    - eval $(ssh-agent -s)
    - mkdir -p ~/.ssh
    - echo "$GIT_PRIVATE_KEY" | ssh-add -
    - echo "$RSYNC_PRIVATE_KEY" | ssh-add -
    - ssh-keyscan -H gitlab.cp-compartner.de >> ~/.ssh/known_hosts

  artifacts:
    expire_in: 4 weeks
    paths:
      - $CI_PROJECT_DIR/vendor
      - $CI_PROJECT_DIR/.Build/vendor

deploy_staging:
  <<: *deploy_template
  environment:
    name: Deploy to staging
  script:
    - ssh-keyscan -H staging.project.de >> ~/.ssh/known_hosts
    - cd $CI_PROJECT_DIR/.Build
    - composer install --no-dev
    - vendor/bin/dep --file=deploy.php deploy:unlock staging
    - vendor/bin/dep --file=deploy.php deploy staging --branch=develop
  only:
    - develop

deploy_production:
  <<: *deploy_template
  environment:
    name: Deploy to production
  script:
    - ssh-keyscan -H www.project.de >> ~/.ssh/known_hosts
    - cd $CI_PROJECT_DIR/.Build
    - composer install --no-dev
    - vendor/bin/dep --file=deploy.php deploy production --tag=${CI_COMMIT_TAG}
  only:
    - tags

composer.json

{
	"name": "my/deployment",
	"description": "deployphp of TYPO3 CMS installation",
	"type": "Project",

	"repositories": [
		{
			"type": "composer",
			"url": "https://composer.typo3.org/"
		}
	],

	"require": {
		"deployer/deployer": "^6.0.5",
		"deployer/recipes": "^6.0.2"
	},

	"scripts": {
		"staging": [
			"vendor/bin/dep deploy staging --branch=develop"
		]
	}
}

deploy.php

<?php

namespace Deployer;

require 'vendor/deployer/deployer/recipe/common.php';
require 'vendor/deployer/recipes/recipe/rsync.php';

set('CI_PROJECT_DIR', getEnv('CI_PROJECT_DIR'));

// read configuration
inventory('hosts.yaml');

task('cms:vendors', function () {
    if (!commandExist('unzip')) {
        writeln(
            '<comment>To speed up composer installation setup "unzip" command' .
            ' with PHP zip extension https://goo.gl/sxzFcD</comment>'
        );
    }

    // if composer.json exists
    run(
        'if [ -f "{{release_path}}/composer.json" ]; then ' .
        'cd {{release_path}} && {{bin/composer}} {{composer_options}}; fi;'
    );
})->desc('Installing vendors');

task('general:shared', function () {
    $sharedPath = "{{deploy_path}}/shared";

    // Validate shared_dir, find duplicates
    foreach (get('shared_dirs') as $a) {
        foreach (get('shared_dirs') as $b) {
            if ($a !== $b && strpos(rtrim($a, '/') . '/', rtrim($b, '/') . '/') === 0) {
                throw new \Deployer\Exception\Exception("Can not share same dirs `$a` and `$b`.");
            }
        }
    }

    foreach (get('shared_dirs') as $dir) {
        // Check if shared dir does not exists.
        if (!test("[ -d $sharedPath/$dir ]")) {
            // Create shared dir if it does not exist.
            run("mkdir -p $sharedPath/$dir");

            // If release contains shared dir, copy that dir from release to shared.
            if (test("[ -d $(echo {{release_path}}/private/$dir) ]")) {
                run("cp -rv {{release_path}}/private/$dir $sharedPath/" . dirname($dir));
            }
        }

        // Remove from source.
        run("rm -rf {{release_path}}/private/$dir");

        // Create path to shared dir in release dir if it does not exist.
        // Symlink will not create the path and will fail otherwise.
        run("mkdir -p `dirname {{release_path}}/private/$dir`");

        // Symlink shared dir to release dir
        run("cd {{deploy_path}} && {{bin/symlink}} ../../../shared/$dir {{release_path}}/private/$dir");
    }

    foreach (get('shared_files') as $file) {
        $dirname = dirname($file);

        // Create dir of shared file
        run("mkdir -p $sharedPath/" . $dirname);

        // Check if shared file does not exists in shared.
        // and file exist in release
        if (!test("[ -f $sharedPath/$file ]") && test("[ -f {{release_path}}/private/$file ]")) {
            // Copy file in shared dir if not present
            run("cp -rv {{release_path}}/private/$file $sharedPath/$file");
        }

        // Remove from source.
        run("if [ -f $(echo {{release_path}}/private/$file) ]; then rm -rf {{release_path}}/private/$file; fi");

        // Ensure dir is available in release
        run("if [ ! -d $(echo {{release_path}}/private/$dirname) ];"
            . " then mkdir -p {{release_path}}/private/$dirname;fi");

        // Touch shared
        run("touch $sharedPath/$file");

        // Symlink shared dir to release dir
        run("cd {{deploy_path}} && {{bin/symlink}} ../../../shared/$file {{release_path}}/private/$file");
    }
})->desc('Creating symlinks for shared files and dirs in cms private folder');

task('general:writable_files', function () {
    $sudo = get('clear_use_sudo') ? 'sudo' : '';
    $paths = get('writable_files');

    foreach ($paths as $path) {
        run("if [ -e \"{{release_path}}/$path\" ]; then $sudo chmod g+w {{release_path}}/$path; fi");
    }
})->desc('Set writable bit on files in "writable_files"');

task('general:executable_files', function () {
    if (!has('executable_files')) {
        return;
    }
    $sudo = get('clear_use_sudo') ? 'sudo' : '';
    $paths = get('executable_files');

    foreach ($paths as $path) {
        run('if [ -e "{{release_path}}/' . $path . '" ]; '
            . 'then ' . $sudo . ' chmod g+x {{release_path}}/' . $path . '; '
            . $sudo . ' chmod u+x {{release_path}}/' . $path . '; fi');
    }
})->desc('Set executable bit on files in "executable_files"');

task('cms:create_folder', function () {
    $sudo = get('clear_use_sudo') ? 'sudo' : '';

    if (!test("[ -d {{release_path}}/private/typo3temp ]")) {
        run("mkdir {{release_path}}/private/typo3temp");
    }
    run("$sudo chmod -R 775 {{release_path}}/private/typo3temp");

    run("$sudo {{release_path}}/vendor/bin/typo3cms install:fixfolderstructure");
})->desc('Creating folder via typo3cms install:fixfolderstructure');

task('cms:symlink', function () {
    if (test("[ -d {{deploy_path}}/current ]")) {
        run("rm {{deploy_path}}/current"); // Remove previous current
    }

    run("cd {{deploy_path}} && {{bin/symlink}} releases/{{release_name}} current"); // Atomic override symlink.
    run("cd {{deploy_path}} && rm release"); // Remove release link.
})->desc('Creating symlink to release');

task('cms:clear_cache', function () {
    $config = \Deployer\Task\Context::get()->getHost()->getConfig();
    if ($config->get('local') && $config->get('remote_path')) {
        // backup
        $contextBackup = \Deployer\Task\Context::pop();
        $config->set('local', false);

        // create temporary host
        $hostname = $contextBackup->getHost()->getHostname();
        $host = new \Deployer\Host\Host($hostname);
        $methods = [
            'hostname',
            'user',
            'port',
            'configFile',
            'identityFile',
            'forwardAgent',
            'multiplexing',
            'sshOptions',
            'sshFlags',
            'shellCommand',
        ];
        foreach ($methods as $method) {
            if ($config->has($method)) {
                $host->$method($config->get($method));
            }
        }
        foreach ($config->getCollection()->getIterator() as $name => $value) {
            $host->set($name, $value);
        }

        // create new context with host
        $context = new \Deployer\Task\Context($host, $contextBackup->getInput(), $contextBackup->getOutput());
        \Deployer\Task\Context::push($context);

        $sudo = get('clear_use_sudo') ? 'sudo' : '';
        $result = run("$sudo {{remote_path}}/releases/{{release_name}}/vendor/bin/typo3cms cache:flush --force");
        writeln($result);

        // restore
        \Deployer\Task\Context::pop();
        $config->set('local', true);
        \Deployer\Task\Context::push($contextBackup);
    }
})->desc('Clear cache via typo3cms cache:flush');

task('cms:clear_opcache', function () {
    $webDomain = rtrim(get('web_domain'), '/');
    $htaccess = has('htaccess') ? '--user ' . get('htaccess') : '';
    run("curl $htaccess -sk $webDomain/cache.php");
})->desc('Clear opcache via curl on /cache.php');

task('cms:db_compare', function () {
    $config = \Deployer\Task\Context::get()->getHost()->getConfig();
    if ($config->get('local') && $config->get('remote_path')) {
        // backup
        $contextBackup = \Deployer\Task\Context::pop();
        $config->set('local', false);

        // create temporary host
        $hostname = $contextBackup->getHost()->getHostname();
        $host = new \Deployer\Host\Host($hostname);
        $methods = [
            'hostname',
            'user',
            'port',
            'configFile',
            'identityFile',
            'forwardAgent',
            'multiplexing',
            'sshOptions',
            'sshFlags',
            'shellCommand',
        ];
        foreach ($methods as $method) {
            if ($config->has($method)) {
                $host->$method($config->get($method));
            }
        }
        foreach ($config->getCollection()->getIterator() as $name => $value) {
            $host->set($name, $value);
        }

        // create new context with host
        $context = new \Deployer\Task\Context($host, $contextBackup->getInput(), $contextBackup->getOutput());
        \Deployer\Task\Context::push($context);

        $sudo = get('clear_use_sudo') ? 'sudo' : '';
        $result = run($sudo . ' {{remote_path}}/releases/{{release_name}}'
            . '/vendor/bin/typo3cms database:updateschema "*.add,*.change"');
        writeln($result);

        // restore
        \Deployer\Task\Context::pop();
        $config->set('local', true);
        \Deployer\Task\Context::push($contextBackup);
    }
})->desc('DB compare via typo3cms database:updateschema');

task('rsync:warmup', function () {
    $config = get('rsync');

    $source = "{{rsync_dest}}";
    $destination = "{{deploy_path}}";

    if (test("[ -d $(echo $destination) ]")) {
        run("rsync -{$config['flags']} {{rsync_options}}{{rsync_excludes}}"
            . "{{rsync_includes}}{{rsync_filter}} $source/ $destination/");
    } else {
        writeln("<comment>No way to warmup rsync.</comment>");
    }
});

task('rsync:switch_current', function () {
    $config = get('rsync');

    $src = get('rsync_src');
    while (is_callable($src)) {
        $src = $src();
    }

    if (!trim($src)) {
        // if $src is not set here rsync is going to do a directory listing
        // exiting with code 0, since only doing a directory listing clearly
        // is not what we want to achieve we need to throw an exception
        throw new \RuntimeException('You need to specify a source path.');
    }

    $dst = get('rsync_dest');
    while (is_callable($dst)) {
        $dst = $dst();
    }

    if (!trim($dst)) {
        // if $dst is not set here we are going to sync to root
        // and even worse - depending on rsync flags and permission -
        // might end up deleting everything we have write permission to
        throw new \RuntimeException('You need to specify a destination path.');
    }

    $server = \Deployer\Task\Context::get()->getHost();
    if ($server instanceof \Deployer\Host\Localhost) {
        runLocally("rsync -{$config['flags']} {{rsync_options}} '$src/current' '$dst/'", $config);
        return;
    }

    $host = $server->getRealHostname();
    $port = $server->getPort() ? ' -p' . $server->getPort() : '';
    $sshArguments = $server->getSshArguments();
    $user = !$server->getUser() ? '' : $server->getUser() . '@';

    runLocally(
        "rsync -{$config['flags']} -e 'ssh$port $sshArguments' {{rsync_options}} '$src/current' '$user$host:$dst/'",
        $config
    );
})->desc('Sync current after release was uploaded')->onHosts('staging', 'production');

task('remote:writable', function () {
    $config = \Deployer\Task\Context::get()->getHost()->getConfig();
    if ($config->get('local') && $config->get('remote_path')) {
        // backup
        $contextBackup = \Deployer\Task\Context::pop();
        $config->set('local', false);

        // create temporary host
        $hostname = $contextBackup->getHost()->getHostname();
        $host = new \Deployer\Host\Host($hostname);
        $methods = [
            'hostname',
            'user',
            'port',
            'configFile',
            'identityFile',
            'forwardAgent',
            'multiplexing',
            'sshOptions',
            'sshFlags',
            'shellCommand',
        ];
        foreach ($methods as $method) {
            if ($config->has($method)) {
                $host->$method($config->get($method));
            }
        }
        foreach ($config->getCollection()->getIterator() as $name => $value) {
            $host->set($name, $value);
        }

        // create new context with host
        $context = new \Deployer\Task\Context($host, $contextBackup->getInput(), $contextBackup->getOutput());
        \Deployer\Task\Context::push($context);

        $sudo = get('writable_use_sudo') ? 'sudo' : '';
        $writeableDirs = get('writable_dirs') ?? [];
        foreach ($writeableDirs as $dir) {
            run("$sudo chmod -R 2775 {{remote_path}}/releases/{{release_name}}/$dir");
            //run("$sudo chgrp -R www-data {{remote_path}}/releases/{{release_name}}/$dir");
        }

        $paths = get('writable_files') ?? [];
        foreach ($paths as $path) {
            run("if [ -e \"{{remote_path}}/releases/{{release_name}}/$path\" ];"
                . " then $sudo chmod g+w {{remote_path}}/releases/{{release_name}}/$path; fi");
        }

        // restore
        \Deployer\Task\Context::pop();
        $config->set('local', true);
        \Deployer\Task\Context::push($contextBackup);
    }
})->onHosts('staging', 'production');

// Main deployment task
task('deploy', [
    'deploy:prepare',
    'deploy:lock',
    'rsync:warmup',
    'deploy:release',
    'deploy:update_code',
    'cms:vendors',
    'general:shared',
    'general:writable_files',
    'general:executable_files',
    'deploy:clear_paths',
    'cms:create_folder',
    'cms:symlink',

    'cleanup',
    'rsync',
    'remote:writable',
    'cms:db_compare',
    'rsync:switch_current',
    'cms:clear_cache',
    'cms:clear_opcache',

    'deploy:unlock',
])->desc('Deploy your project');


// Display success message on completion
after('deploy', 'success');


// [Optional] if deploy fails automatically unlock.
after('deploy:failed', 'deploy:unlock');

host.yaml

# for more settings look in https://github.com/deployphp/deployer/recipe/common.php
.base: &base
  user: root

  repository: [email protected]:customer/project.git
  ssh_type: native

  # needs to be set because we remove .git in clear_paths
  git_cache: false

  shared_dirs:
    - uploads
    - fileadmin
    - .well-known

  writable_dirs:
    - private/typo3temp
  writable_files:
    - private/typo3conf/PackageStates.php

  clear_paths:
    - .Build
    - .editorconfig
    - .git
    - .gitignore
    - .gitlab-ci.yml
    - composer.json
    - composer.lock
    - private/typo3conf/deprecation_*

local:
  <<: *base
  stage: local
  local: true

  deploy_path: /var/www/project/www/archive
  web_domain: https://www.project.local.cp.lan/

staging:
  <<: *base
  local: true
  stage: staging

  bin/symlink: ln -s
  deploy_path: '{{CI_PROJECT_DIR}}/cache/deployment/staging'
  web_domain: https://staging.project.de/

  user: cp1
  hostname: staging.project.de
  remote_path: /var/www/staging.project.de/var/www/staging.project.de/deployment
  htaccess: project:password

  rsync_src: '{{CI_PROJECT_DIR}}/cache/deployment/staging'
  rsync_dest: '{{user}}@{{hostname}}:{{remote_path}}'
  rsync:
    flags: rlzc --delete-after
    exclude:
      - shared
      - current
      - "-,p private/typo3temp/var/Cache"

    exclude-file:
    include: []
    include-file:
    filter: []
    filter-file: false
    filter-perdir: false
    timeout: 600
    options: []

production:
  <<: *base
  local: true
  stage: production

  bin/symlink: ln -s
  deploy_path: '{{CI_PROJECT_DIR}}/cache/deployment/production'
  web_domain: https://www.project.de/

  user: cp1
  hostname: www.project.de
  remote_path: /var/www/project.de/var/www/project.de/deployment

  rsync_src: '{{CI_PROJECT_DIR}}/cache/deployment/production'
  rsync_dest: '{{user}}@{{hostname}}:{{remote_path}}'
  rsync:
    flags: rlzc --delete-after
    exclude:
      - shared
      - current
      - "-,p private/typo3temp/var/Cache"

    exclude-file:
    include: []
    include-file:
    filter: []
    filter-file: false
    filter-perdir: false
    timeout: 600
    options: []