DoctrineFixturesBundle for big data

ORMFixture.php

<?php

// Copyright (c) 2016 Andrey <[email protected]> Savitsky. All rights reserved.
// Site: http://qRoC.pro

namespace CommonBundle\DataFixtures;

use Doctrine\Common\DataFixtures\FixtureInterface;
use Doctrine\Common\DataFixtures\OrderedFixtureInterface;
use Symfony\Component\DependencyInjection\ContainerAwareInterface;
use Symfony\Component\DependencyInjection\ContainerInterface;
use Doctrine\Common\Persistence\ObjectManager;

abstract class ORMFixture implements FixtureInterface, ContainerAwareInterface, OrderedFixtureInterface
{
    const kMemoryLimit = 32;
    const kMemoryCanUsage = 0.3; // 70% резерв на flush() и clear(). Иногда нужно больше

    /**
     * @var ContainerInterface
     */
    protected $container;

    final public function setContainer(ContainerInterface $container = null)
    {
        $this->container = $container;
    }

    final public function load(ObjectManager $manager)
    {
        $manager->getConnection()->getConfiguration()->setSQLLogger(null);

        $this->loadStandart($manager);
        $this->innerLoadIteration($manager);

        $manager->flush();
        $manager->clear();
    }

    public function getOrder()
    {
        return 0;
    }

    private function innerLoadIteration(ObjectManager $manager)
    {
        $memory_limit = self::kMemoryLimit;

        if (preg_match('/^(\d+)(.)$/', ini_get('memory_limit'), $matches))
        {
            $memory_limit = round($matches[1] * self::kMemoryCanUsage);

            switch($matches[2])
            {
                case 'G':
                    $memory_limit *=1024;
                case 'M':
                    $memory_limit *=1024;
                case 'K':
                    $memory_limit *=1024;
            }
        }

        $iteration = function(array $work_objects = []) use($memory_limit, $manager)
        {
            $usage = memory_get_usage();

            if ($usage > $memory_limit)
            {
//                echo 'USAGE: ' . $usage . PHP_EOL;
//                echo 'MAX: ' . $memory_limit . PHP_EOL;

                $manager->flush();

                foreach($work_objects as $work_object)
                {
                    $manager->clear($work_object);
                }

                //echo 'AFTER CLEAR: ' . memory_get_usage() . PHP_EOL;
            }
        };

        $this->loadIteration($manager, $iteration);
    }

    /**
     * iteration {
     *
     *     // ....
     *
     *     $iteration();
     * }
     *
     * @param ObjectManager $manager
     * @param callable      $iteration
     */
    protected function loadIteration(ObjectManager $manager, callable $iteration)
    {
    }

    protected function loadStandart(ObjectManager $manager)
    {

    }
}

TestUsage.php

<?php

// Copyright (c) 2016 Andrey <[email protected]> Savitsky. All rights reserved.
// Site: http://qRoC.pro

namespace CommonBundle\DataFixtures\ORM;

use CommonBundle\DataFixtures\ORMFixture;
use Doctrine\Common\Persistence\ObjectManager;
use CommonBundle\Entity\Country;
use CommonBundle\Entity\State;
use CommonBundle\Entity\City;

class LoadLocationData extends ORMFixture
{
    public function loadIteration(ObjectManager $manager, callable $iteration)
    {
        $file = fopen(__DIR__ . '/data/weblocations.dat', 'r');
        
        while(!feof($file))
        {
            $line = fgets($file);
            
            // $entity = new City();
            $manager->persist($entity);
            
            // $iteration(); // Никогда не выполнять $manager->clear();
            $iteration([City::class]); // $manager->clear(City::class) - нужно если вы работаете с несколькими типами сущностей одновременно. 
        }
    }
}