GitlabSyncCommand for custom packagist installations

GitlabSyncCommand

<?php

namespace Packagist\WebBundle\Command;

use Gitlab\Api\Projects;
use Gitlab\Api\Repositories;
use Packagist\WebBundle\Entity\Package;
use Packagist\WebBundle\Entity\User;
use sweikenb\Library\Threading\Thread;
use sweikenb\Library\Threading\ThreadRegistry;
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Form\Form;
use Symfony\Component\Form\FormTypeInterface;

/**
 * Class GitlabSyncCommand
 *
 * @package Packagist\WebBundle\Command
 */
class GitlabSyncCommand extends ContainerAwareCommand
{
    /**
     * @return \Doctrine\Bundle\DoctrineBundle\Registry
     */
    private function getDoctrine()
    {
        return $this->getContainer()->get('doctrine');
    }

    /**
     * @return \Symfony\Bundle\FrameworkBundle\Routing\Router
     */
    private function getRouter()
    {
        return $this->getContainer()->get('router');
    }

    /**
     * Setup the command
     */
    protected function configure()
    {
        $this
            ->setName('packagist:sync:gitlab')
            ->setDescription('Syncs all known repositories of the given GitLab repository into the packagist index.');
    }

    /**
     * @param InputInterface  $input
     * @param OutputInterface $output
     *
     * @throws \Exception
     * @return int|null|void
     */
    protected function execute(InputInterface $input, OutputInterface $output)
    {
        // get the configuration
        $config = $this->getContainer()->getParameter('vendor_sync');
        
        /*
         Content of $config:
         
         vendor_sync:
            gitlab:
                api_endpoint: https://YOUR_GITLAB_SERVER/api/v3/
                api_tokens:
                    - YourFirstTokenHere
                    - MoreTokensHere
            packagist:
                target_user: TargetUsernameOfYourPackagistInstallation
        */

        // info
        $output->writeln("\n\n-> Starging GitLab sync ...\n\n", OutputInterface::VERBOSITY_DEBUG);

        // process token
        $registry = new ThreadRegistry(5);
        foreach ($config['gitlab']['api_tokens'] as $token) {
            while ($registry->isLimitReached()) {
                $registry->waitForChildsToExit();
            }
            $thread = Thread::getInstance()->spawn(array($this, 'processToken'), array($output, $config, $token));
            $registry->registerThread($thread);
        }

        // wait for processes
        while ($registry->count() > 0) {
            $registry->waitForChildsToExit();
        }
        $output->writeln("\n\n-> GitLab sync done!\n\n", OutputInterface::VERBOSITY_DEBUG);
    }

    /**
     * @param Thread          $thread
     * @param OutputInterface $output
     * @param array           $config
     * @param string          $token
     *
     * @throws \Exception
     */
    public function processToken(Thread $thread, OutputInterface $output, array $config, $token)
    {
        $manager = $this->getContainer()->get('doctrine.orm.default_entity_manager');
        $manager->getConnection()->close();
        $manager->getConnection()->connect();

        // find the target user
        $user = $this->getContainer()->get('fos_user.user_manager')->findUserByUsername($config['packagist']['target_user']);
        if (!$user) {
            throw new \Exception(sprintf("<error>Target user not found: %s</error>", $user));
        }
        /* @var User $user */

        $client = new \Gitlab\Client($config['gitlab']['api_endpoint']); // change here
        $client->authenticate($token, \Gitlab\Client::AUTH_HTTP_TOKEN); // change here

        $projectApi = $client->api('projects');
        /* @var Projects $projectApi */

        $page = 1;
        $registry = new ThreadRegistry(5);
        while (($projects = $projectApi->accessible($page, 50))) {
            // we only need the repo URLS
            foreach ($projects as $project) {
                while ($registry->isLimitReached()) {
                    $registry->waitForChildsToExit();
                }
                $subThread = Thread::getInstance()->spawn(array($this, 'processProject'), array($output, $config, $token, $project, $user));
                $registry->registerThread($subThread);
            }

            // next page
            ++$page;
        };

        // wait for processes
        while ($registry->count() > 0) {
            $registry->waitForChildsToExit();
        }

        // terminate thread
        $thread->terminate();
    }

    /**
     * @param Thread          $thread
     * @param OutputInterface $output
     * @param array           $config
     * @param string          $token
     * @param array           $project
     * @param User            $user
     */
    public function processProject(Thread $thread, OutputInterface $output, array $config, $token, array $project, User $user)
    {
        $manager = $this->getContainer()->get('doctrine.orm.default_entity_manager');
        $manager->getConnection()->close();
        $manager->getConnection()->connect();

        $client = new \Gitlab\Client($config['gitlab']['api_endpoint']); // change here
        $client->authenticate($token, \Gitlab\Client::AUTH_HTTP_TOKEN); // change here

        $reposApi = $client->api('repositories');
        /* @var Repositories $reposApi */

        // read 'composer.json'
        try {
            $content = $reposApi->blob($project['id'], 'master', 'composer.json');
        } catch (\Exception $e) {
            // skipp this repo
            return;
        }

        // TODO: make this configurable
        $skippNames = array(
            'symfony/framework-standard-edition'
        );

        if (isset($content['name']) && !in_array($content['name'], $skippNames)) {
            $this->updatePackageForRepository(
                $output,
                $project['id'],
                array(
                    'name'          => $content['name'],
                    'package'       => $project['path_with_namespace'],
                    'git_path'      => $project['ssh_url_to_repo'],
                    'description'   => $project['description'],
                    'composer.json' => $content
                ),
                $user
            );
        }

        // terminate thread
        $thread->terminate();
    }

    /**
     * @param OutputInterface $output
     * @param int             $repoId
     * @param array           $repoMeta
     * @param User            $user
     *
     * @return Package
     */
    private function updatePackageForRepository(OutputInterface $output, $repoId, array $repoMeta, User $user)
    {
        $package = $this->getDoctrine()->getRepository('PackagistWebBundle:Package')->findOneBy(
            array(
                'repository' => $repoMeta['git_path']
            )
        );
        $new = !$package;
        if ($new) {
            $package = new Package();
            $package->addMaintainer($user);
        }

        $package->setRepository($repoMeta['git_path']);
        $package->setEntityRepository($this->getDoctrine()->getRepository('PackagistWebBundle:Package'));
        $package->setRouter($this->getRouter());

        $package->setName($repoMeta['name']);
        $package->setDescription($repoMeta['description']);

        $em = $this->getDoctrine()->getManager();
        $em->persist($package);

        try {
            $em->flush($package);
            $output->writeln(sprintf("%s-> <info>%s</info> done", ($new ? '[NEW] ' : '[UPD] '), $repoMeta['git_path']), OutputInterface::VERBOSITY_DEBUG);
        }
        catch(\Doctrine\DBAL\DBALException $e) {
            $output->writeln(sprintf("[ERR] -> <comment>%s</comment>", $e->getMessage()), OutputInterface::VERBOSITY_DEBUG);
        }
    }
}

The used Thread-stuff can be found here: https://github.com/sweikenb/php-library/tree/master/sweikenb/Library/Threading