Skip to content

Instantly share code, notes, and snippets.

@chillu

chillu/PaginatedFileMigrationHelper.php

Last active September 3, 2019 23:36
Show Gist options
  • Save chillu/45743f22cdaba223296c926475baab5b to your computer and use it in GitHub Desktop.
Save chillu/45743f22cdaba223296c926475baab5b to your computer and use it in GitHub Desktop.
Download ZIP
SS 4.x file migration performance tips
Raw
_README.md
Raw
PaginatedFileMigrationHelper.php
<?php
use Generator;
use LogicException;
use Psr\Log\LoggerInterface;
use SilverStripe\Assets\Dev\Tasks\FileMigrationHelper;
use SilverStripe\Assets\File;
use SilverStripe\Assets\FilenameParsing\FileIDHelperResolutionStrategy;
use SilverStripe\Assets\FilenameParsing\FileResolutionStrategy;
use SilverStripe\Assets\FilenameParsing\LegacyFileIDHelper;
use SilverStripe\Assets\Flysystem\FlysystemAssetStore;
use SilverStripe\Assets\Folder;
use SilverStripe\Assets\Storage\AssetStore;
use SilverStripe\Assets\Storage\FileHashingService;
use SilverStripe\Core\Config\Configurable;
use SilverStripe\Core\Environment;
use SilverStripe\Core\Injector\Injectable;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\ORM\DataList;
use SilverStripe\ORM\DataObject;
use SilverStripe\ORM\DB;
use SilverStripe\ORM\ValidationException;
use SilverStripe\Subsites\Extensions\FileSubsites;
use SilverStripe\Versioned\Versioned;
/**
* Service to help migrate File dataobjects to the new APL.
*
* This service does not alter these records in such a way that prevents downgrading back to 3.x
*/
class PaginatedFileMigrationHelper extends FileMigrationHelper
{
use Injectable;
use Configurable;
private static $dependencies = [
'logger' => '%$' . LoggerInterface::class,
];
/**
* @var LoggerInterface
*/
private $logger;
/**
* @var FlysystemAssetStore
*/
private $store;
/**
* If a file fails to validate during migration, delete it.
* If set to false, the record will exist but will not be attached to any filesystem
* item anymore.
*
* @config
* @var bool
*/
private static $delete_invalid_files = true;
protected $fromFileID = null;
protected $toFileID = null;
public function __construct()
{
parent::__construct();
}
/**
* @param LoggerInterface $logger
* @return $this
*/
public function setLogger(LoggerInterface $logger)
{
$this->logger = $logger;
return $this;
}
/**
* Perform migration
*
* @param string $base Absolute base path (parent of assets folder). Will default to PUBLIC_PATH
* @param null|int $fromFileID
* @param null|int $toFileID
*
* @return int Number of files successfully migrated
*/
public function run($base = null, $fromFileID = null, $toFileID = null)
{
$this->fromFileID = (int) $fromFileID;
$this->toFileID = (int) $toFileID;
$this->store = Injector::inst()->get(AssetStore::class);
if (!$this->store instanceof AssetStore || !method_exists($this->store, 'normalisePath')) {
throw new LogicException(
'FileMigrationHelper: Can not run if the default asset store does not have a `normalisePath` method.'
);
}
if (empty($base)) {
$base = PUBLIC_PATH;
}
// Set max time and memory limit
Environment::increaseTimeLimitTo();
Environment::setMemoryLimitMax(-1);
Environment::increaseMemoryLimitTo(-1);
ini_set('memory_limit', -1); //Actually set it to -1.
/** @var FileHashingService $hasher */
$hasher = Injector::inst()->get(FileHashingService::class);
$hasher::flush();
$this->logger->notice('Step 1/2: Migrate 3.x legacy files to new format');
$ss3Count = $this->ss3Migration($base);
$this->logger->notice('Step 2/2: Migrate files in <4.4 format to new format');
$ss4Count = 0;
if (class_exists(Versioned::class) && File::has_extension(Versioned::class)) {
Versioned::prepopulate_versionnumber_cache(File::class, Versioned::LIVE);
Versioned::prepopulate_versionnumber_cache(File::class, Versioned::DRAFT);
$stages = [Versioned::LIVE => 'live', Versioned::DRAFT => 'draft'];
foreach ($stages as $stageId => $stageName) {
$this->logger->info(sprintf('Migrating files in the "%s" stage', $stageName));
$count = Versioned::withVersionedMode(function () use ($stageId, $stageName) {
Versioned::set_stage($stageId);
return $this->normaliseAllFiles(sprintf('on the "%s" stage', $stageName));
});
if ($count) {
$this->logger->info(sprintf('Migrated %d files in the "%s" stage', $count, $stageName));
} else {
$this->logger->info(sprintf(
'No files required migrating in the "%s" stage',
$stageName
));
}
$ss4Count += $count;
}
} else {
$ss4Count = $this->normaliseAllFiles('');
$this->logger->info(sprintf('Migrated %d files', $ss4Count));
}
return $ss3Count + $ss4Count;
}
protected function ss3Migration($base)
{
// Check if the File dataobject has a "Filename" field.
// If not, cannot migrate
/** @skipUpgrade */
if (!DB::get_schema()->hasField('File', 'Filename')) {
return 0;
}
// Check if we have files to migrate
$totalCount = $this->getFileQuery()->count();
if (!$totalCount) {
$this->logger->info('No files required migrating');
return 0;
}
$this->logger->debug(sprintf('Migrating %d files', $totalCount));
// Create a temporary SS3 Legacy File Resolution strategy for migrating SS3 Files
$initialStrategy = $this->store->getPublicResolutionStrategy();
$ss3Strategy = $this->buildSS3MigrationStrategy($initialStrategy);
if (!$ss3Strategy) {
$this->logger->warning(
'Skipping this step because the asset store is using an unsupported public file ' .
'resolution strategy.'
);
return 0;
}
$this->store->setPublicResolutionStrategy($ss3Strategy);
// Force stage to draft
if (class_exists(Versioned::class) && File::has_extension(Versioned::class)) {
$originalState = Versioned::get_reading_mode();
Versioned::set_stage(Versioned::DRAFT);
}
// Set up things before going into the loop
$ss3Count = 0;
$originalState = null;
// Loop over the files to migrate
try {
//CHANGE to getPaginatedFileQuery
foreach ($this->getPaginatedFileQuery() as $file) {
// Bypass the accessor and the filename from the column
$filename = $file->getField('Filename');
$success = $this->migrateFile($base, $file, $filename);
if ($success) {
$ss3Count++;
}
}
} finally {
// Reset back to our initial state no matter what
if (class_exists(Versioned::class)) {
Versioned::set_reading_mode($originalState);
}
$this->store->setPublicResolutionStrategy($initialStrategy);
}
// Show summary of results
if ($ss3Count > 0) {
$this->logger->info(sprintf('%d 3.x legacy files have been migrated.', $ss3Count));
} else {
$this->logger->info(sprintf('No 3.x legacy files required migrating.'));
}
return $ss3Count;
}
/**
* Construct a temporary SS3 File Resolution Strategy based off the provided initial strategy.
* If `$initialStrategy` is not suitable for a migration, we return null.
* We're overriding the helpers to avoid doing unnecessary checks.
* @param FileResolutionStrategy $initialStrategy
* @return int|FileIDHelperResolutionStrategy
*/
private function buildSS3MigrationStrategy(FileResolutionStrategy $initialStrategy)
{
// If the project is using a custom FileResolutionStrategy, we can't be confident that our migration won't
// break stuff, so let's bail
if (!$initialStrategy instanceof FileIDHelperResolutionStrategy) {
return null;
}
// Let's make sure the initial strategy contains a LegacyFileIDHelper. If it doesn't, the owner of the project
// has explicitly disabled Legacy resolution, so there's no SS3 files to migrate
$foundLegacyHelper = false;
foreach ($initialStrategy->getResolutionFileIDHelpers() as $helper) {
if ($helper instanceof LegacyFileIDHelper) {
$foundLegacyHelper = true;
break;
}
}
if (!$foundLegacyHelper) {
return null;
}
// Build the migration strategy
$ss3Strategy = FileIDHelperResolutionStrategy::create();
$ss3Strategy->setDefaultFileIDHelper($initialStrategy->getDefaultFileIDHelper());
$ss3Strategy->setResolutionFileIDHelpers([new LegacyFileIDHelper(false)]);
return $ss3Strategy;
}
/**
* Migrate a single file
*
* @param string $base Absolute base path (parent of assets folder)
* @param File $file
* @param string $legacyFilename
* @return bool True if this file is imported successfully
*/
protected function migrateFile($base, File $file, $legacyFilename)
{
// Make sure this legacy file actually exists
$path = $this->findNativeFile($base, $file, $legacyFilename);
if ($path === false) {
return false;
}
// Fix file classname if it has a classname that's incompatible with its extention
$extension = $file->getExtension();
if (!$this->validateFileClassname($file, $extension)) {
// We disable validation (if it is enabled) so that we are able to write a corrected
// classname, once that is changed we re-enable it for subsequent writes
$validationEnabled = DataObject::Config()->get('validation_enabled');
if ($validationEnabled) {
DataObject::Config()->set('validation_enabled', false);
}
$destinationClass = $file->get_class_for_file_extension($extension);
$file = $file->newClassInstance($destinationClass);
$fileID = $file->write();
if ($validationEnabled) {
DataObject::Config()->set('validation_enabled', true);
}
$file = File::get_by_id($fileID);
}
// Remove invalid files
$validationResult = $file->validate();
if (!$validationResult->isValid()) {
if ($this->config()->get('delete_invalid_files')) {
$file->delete();
}
if ($this->logger) {
$messages = implode("\n\n", array_map(function ($msg) {
return $msg['message'];
}, $validationResult->getMessages()));
$this->logger->warning(
sprintf(
" %s was not migrated because the file is not valid. More information: %s",
$legacyFilename,
$messages
)
);
}
return false;
}
// Copy local file into this filesystem
$dbFilename = $file->generateFilename();
$fsFilename = $path === true ? $this->stripAssetsDir($legacyFilename) : $this->stripAssetsDir($path, $base);
if ($path === true && $fsFilename == $dbFilename) {
$results = $this->store->normalisePath($dbFilename);
} else {
$results = $this->store->setFromLocalFile(
$base . DIRECTORY_SEPARATOR . ASSETS_DIR . DIRECTORY_SEPARATOR . $fsFilename,
$dbFilename,
null,
null,
AssetStore::VISIBILITY_PROTECTED
);
$this->store->delete($fsFilename, $results['Hash']);
}
// Move file if the APL changes filename value
$file->File->Filename = $results['Filename'];
$file->File->Hash = $results['Hash'];
$file->setField('Filename', null);
// Save and publish
try {
if (class_exists(Versioned::class)) {
$file->writeToStage(Versioned::LIVE);
} else {
$file->write();
}
} catch (ValidationException $e) {
if ($this->logger) {
$this->logger->error(sprintf(
"File %s could not be migrated due to an error.
This problem likely existed before the migration began. Error: %s",
$legacyFilename,
$e->getMessage()
));
}
return false;
}
if ($dbFilename == $file->getFilename()) {
$this->logger->info(sprintf('* SS3 file %s converted to SS4 format', $file->getFilename()));
} else {
$this->logger->warning(sprintf(
'* SS3 file %s converted to SS4 format but was renamed to %s',
$dbFilename,
$file->getFilename()
));
}
if (!empty($results['Operations'])) {
foreach ($results['Operations'] as $origin => $destination) {
$this->logger->debug(sprintf(' related thumbnail %s moved to %s', $origin, $destination));
}
}
return true;
}
/**
* Look for a file by `$legacyFilename`. If an exact match is found return true. If no match is found return false.
* If a match using a different case is found use the path of that file.
* @param string $base Location of the assets folder, usually equals to ASSETS_PATH
* @param File $file
* @param string $legacyFilename SS3 filename prefix with ASSETS_DIR
* @return bool|string True if file is found as-is, false if file is missing or string to alternative file
*/
private function findNativeFile($base, File $file, $legacyFilename)
{
$path = $base . DIRECTORY_SEPARATOR . $legacyFilename;
if (file_exists($path)) {
return true;
}
$strippedLegacyFilename = $this->stripAssetsDir($legacyFilename);
// Try to find an alternative file
$legacyFilenameGlob = preg_replace_callback('/[a-z]/i', function ($matches) {
return sprintf('[%s%s]', strtolower($matches[0]), strtoupper($matches[0]));
}, $strippedLegacyFilename);
$files = glob($base . DIRECTORY_SEPARATOR . ASSETS_DIR . DIRECTORY_SEPARATOR . $legacyFilenameGlob);
switch (sizeof($files)) {
case 0:
$this->logger->error(sprintf(
'%s could not be migrated because no matching file was found.',
$legacyFilename
));
return false;
case 1:
$path = $files[0];
break;
default:
// We found multiple files with same casing, life's just too dam complicated
$this->logger->error(sprintf(
'%s could not be migrated because no matching file was found. ' .
'Other files with alternative casing exists: %s',
$legacyFilename,
implode(', ', $files)
));
return false;
}
// Make sure we do not have an unmigrated or migrated DB entry for our alternative file.
$strippedPath = $this->stripAssetsDir($path, $base);
$unmigratedFiles = $this->getFileQuery()
->filter('Filename:case', ASSETS_DIR . '/' . $strippedPath)
->exclude('ID', $file->ID);
$migratedFiles = File::get()->filter('FileFilename:case', $strippedPath)
->exclude('ID', $file->ID);
if ($unmigratedFiles->count() > 0 || $migratedFiles->count() > 0) {
$this->logger->error(sprintf(
'%s could not be migrated because no matching file was found.',
$legacyFilename
));
return false;
}
// We can use our alternative file
$this->logger->warning(sprintf(
'%s could not be found, but a file with an alternative casing was identified. %s will be used instead.',
$legacyFilename,
$strippedPath
));
return $path;
}
/**
* Given a path, remove the asset dir folder and the optional base path.
* @param $path
* @param string $base
* @return string
*/
private function stripAssetsDir($path, $base = '')
{
return preg_replace(
sprintf(
'#^%s%s%s?#',
$base ? $base . DIRECTORY_SEPARATOR : DIRECTORY_SEPARATOR . '?',
ASSETS_DIR,
DIRECTORY_SEPARATOR
),
'',
$path
);
}
/**
* Go through the list of files and make sure each one is at its default location
* @param string $stageString Complement of information to append to the confirmation message
* @param bool $skipIdenticalStages Whatever files that are already present on an other stage should be skipped
* @return int
*/
protected function normaliseAllFiles($stageString, $skipIdenticalStages = false)
{
$count = 0;
//CHANGE add from to filter
$fileQuery = File::get()->exclude('ClassName', [Folder::class, 'Folder']);
$files = $this->chunk($this->addFromToFilter($fileQuery));
/** @var File $file */
foreach ($files as $file) {
// There's no point doing those checks the live and draft file are the same
if ($skipIdenticalStages && !$file->stagesDiffer()) {
continue;
}
if (!$this->store->exists($file->File->Filename, $file->File->Hash)) {
$this->logger->warning(sprintf(
' Can not migrate %s / %s because it does not exist %s',
$file->File->Filename,
$file->File->Hash,
$stageString
));
continue;
}
$results = $this->store->normalise($file->File->Filename, $file->File->Hash);
if ($results && !empty($results['Operations'])) {
$this->logger->debug(
sprintf(' %s has been migrated %s', $file->getFilename(), $stageString)
);
foreach ($results['Operations'] as $origin => $destination) {
$this->logger->debug(sprintf(' related thumbnail %s moved to %s', $origin, $destination));
}
$count++;
}
}
return $count;
}
/**
* Split queries into smaller chunks to avoid using too much memory
* @param DataList $query
* @return Generator
*/
private function chunk(DataList $query)
{
$chunkSize = 100;
$greaterThanID = 0;
$query = $query->limit($chunkSize)->sort('ID');
while ($chunk = $query->filter('ID:GreaterThan', $greaterThanID)) {
foreach ($chunk as $file) {
yield $file;
}
if ($chunk->count() == 0) {
break;
}
$greaterThanID = $file->ID;
}
}
/**
* @return Generator
*/
protected function getPaginatedFileQuery()
{
$fileQuery = $this->getFileQuery();
return $this->chunk($this->addFromToFilter($fileQuery));
}
/**
* Get list of File dataobjects to import.
* Checks if the FileSubsite extension is applied and
* disables the subsite filter in that case.
*
* @return DataList
*/
protected function getFileQuery()
{
$fQuery = parent::getFileQuery();
if (class_exists(FileSubsites::class) &&
File::has_extension(FileSubsites::class)) {
$fQuery = $fQuery->setDataQueryParam('Subsite.filter', false);
}
return $fQuery;
}
/**
* @param DataList $query
*
* @return mixed
*/
protected function addFromToFilter($query)
{
if (!empty($this->fromFileID)) {
$query = $query->filter('ID:GreaterThan', $this->fromFileID);
}
if (!empty($this->toFileID)) {
$query = $query->filter('ID:LessThanOrEqual', $this->toFileID);
}
return $query;
}
}
Raw
PaginatedMigrateFileTask.php
<?php
use function ceil;
use function ini_set;
use Monolog\Logger;
use PaginatedFileMigrationHelper;
use function ob_clean;
use function ob_end_flush;
use function ob_start;
use Psr\Log\LoggerInterface;
use SilverStripe\AssetAdmin\Helper\ImageThumbnailHelper;
use SilverStripe\Assets\Dev\Tasks\LegacyThumbnailMigrationHelper;
use SilverStripe\Assets\File;
use SilverStripe\Assets\Storage\FileHashingService;
use SilverStripe\Control\HTTPRequest;
use SilverStripe\Core\Environment;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\Assets\Dev\Tasks\SecureAssetsMigrationHelper;
use SilverStripe\Dev\Tasks\FixFolderPermissionsHelper;
use SilverStripe\Dev\Tasks\MigrateFileTask;
use Symbiote\QueuedJobs\Controllers\QueuedTaskRunner;
use TractorCow\SilverStripeProxyDB\ProxyDBFactory;
/**
* Migrates all 3.x file dataobjects to use the new DBFile field.
* @todo This is copied from SilverStripe\Dev\Tasks\MigrateFileTask.
* Create pull request to push changes back to framework.
*/
class PaginatedMigrateFileTask extends MigrateFileTask
{
private static $segment = 'FilePaginated';
protected $title = 'Migrate File dataobjects paginated from 3.x and successive iterations in 4.x';
protected $defaultSubtasks = [
'move-files',
'move-thumbnails',
'generate-cms-thumbnails',
'fix-folder-permissions',
'fix-secureassets',
];
protected static $planQueueName = 'planQueue';
private static $dependencies = [
'logger' => '%$' . LoggerInterface::class,
];
/** @var Logger */
private $logger;
public function run($request)
{
$this->unloadExtensions();
$this->addLogHandlers();
$args = $request->getVars();
if (!empty($args[self::$planQueueName])) {
$this->queueSplitTasks($request);
return;
}
$this->validateArgs($args);
Injector::inst()->get(FileHashingService::class)->enableCache();
// Set max time and memory limit
Environment::increaseTimeLimitTo();
Environment::setMemoryLimitMax(-1);
Environment::increaseMemoryLimitTo(-1);
ini_set('memory_limit', -1); //Actually set it to -1.
$this->extend('preFileMigration');
$this->logger->warn(
'Please read https://docs.silverstripe.org/en/4/developer_guides/files/file_migration/ ' .
'before running this task.'
);
$subtasks = !empty($args['only']) ? explode(',', $args['only']) : $this->defaultSubtasks;
$subtask = 'move-files';
if (in_array($subtask, $subtasks)) {
//CHANGE FROM MigrateFileTask
if (!class_exists(PaginatedFileMigrationHelper::class)) {
$this->logger->error("No file migration helper detected");
} else {
$this->extend('preFileMigrationSubtask', $subtask);
$this->logger->notice("######################################################");
$this->logger->notice("Migrating filesystem and database records ({$subtask})");
$this->logger->notice("######################################################");
$from = $args['from'] ?: null;
$to = $args['to'] ?: null;
PaginatedFileMigrationHelper::singleton()
->setLogger($this->logger)
->run(null, $from, $to);
// TODO Split file migration helper into two tasks,
// and report back on their process counts consistently here
// if ($count) {
// $this->logger->info("{$count} File objects upgraded");
// } else {
// $this->logger->info("No File objects needed upgrading");
// }
$this->extend('postFileMigrationSubtask', $subtask);
}
}
$subtask = 'move-thumbnails';
if (in_array($subtask, $subtasks)) {
if (!class_exists(LegacyThumbnailMigrationHelper::class)) {
$this->logger->error("LegacyThumbnailMigrationHelper not found");
} else {
$this->extend('preFileMigrationSubtask', $subtask);
$this->logger->notice("#############################################################");
$this->logger->notice("Migrating existing thumbnails to new file format ({$subtask})");
$this->logger->notice("#############################################################");
$paths = LegacyThumbnailMigrationHelper::singleton()
->setLogger($this->logger)
->run($this->getStore());
if ($paths) {
$this->logger->info(sprintf("%d thumbnails moved", count($paths)));
} else {
$this->logger->info("No thumbnails needed to be moved");
}
$this->extend('postFileMigrationSubtask', $subtask);
}
}
$subtask = 'generate-cms-thumbnails';
if (in_array($subtask, $subtasks)) {
if (!class_exists(ImageThumbnailHelper::class)) {
$this->logger->error("ImageThumbnailHelper not found");
} else {
$this->extend('preFileMigrationSubtask', $subtask);
$this->logger->notice("#############################################");
$this->logger->notice("Generating new CMS UI thumbnails ({$subtask})");
$this->logger->notice("#############################################");
$count = ImageThumbnailHelper::singleton()
->setLogger($this->logger)
->run();
if ($count > 0) {
$this->logger->info("Created {$count} CMS UI thumbnails");
} else {
$this->logger->info("No CMS UI thumbnails needed to be created");
}
$this->extend('postFileMigrationSubtask', $subtask);
}
}
$subtask = 'fix-folder-permissions';
if (in_array($subtask, $subtasks)) {
if (!class_exists(FixFolderPermissionsHelper::class)) {
$this->logger->error("FixFolderPermissionsHelper not found");
} else {
$this->extend('preFileMigrationSubtask', $subtask);
$this->logger->notice("####################################################");
$this->logger->notice("Fixing secure-assets folder permissions ({$subtask})");
$this->logger->notice("####################################################");
$this->logger->debug('Only required if the 3.x project included silverstripe/secure-assets');
$count = FixFolderPermissionsHelper::singleton()
->setLogger($this->logger)
->run();
if ($count > 0) {
$this->logger->info("Repaired {$count} folders with broken CanViewType settings");
} else {
$this->logger->info("No folders required fixes");
}
$this->extend('postFileMigrationSubtask', $subtask);
}
}
$subtask = 'fix-secureassets';
if (in_array($subtask, $subtasks)) {
if (!class_exists(SecureAssetsMigrationHelper::class)) {
$this->logger->error("SecureAssetsMigrationHelper not found");
} else {
$this->extend('preFileMigrationSubtask', $subtask);
$this->logger->notice("#####################################################");
$this->logger->notice("Fixing secure-assets folder restrictions ({$subtask})");
$this->logger->notice("#####################################################");
$this->logger->debug('Only required if the 3.x project included silverstripe/secure-assets');
$paths = SecureAssetsMigrationHelper::singleton()
->setLogger($this->logger)
->run($this->getStore());
if (count($paths) > 0) {
$this->logger->info(sprintf("Repaired %d folders broken folder restrictions", count($paths)));
} else {
$this->logger->info("No folders required fixes");
}
$this->extend('postFileMigrationSubtask', $subtask);
}
}
$this->logger->info("Done!");
$this->extend('postFileMigration');
}
/**
* Remove extensions not desired for file migration
*/
protected function unloadExtensions()
{
ProxyDBFactory::remove_extension(\SilverStripe\FullTextSearch\Search\Extensions\ProxyDBExtension::class);
ProxyDBFactory::remove_extension(\SilverStripe\Auditor\Extensions\ProxyDBExtension::class);
}
/**
* @param LoggerInterface $logger
*/
public function setLogger(LoggerInterface $logger)
{
$this->logger = $logger;
return $this;
}
/**
* Plan and prepare tasks with a number of subtasks
*
* @param HTTPRequest $request
*/
public function queueSplitTasks($request)
{
/**
* @var QueuedTaskRunner $queueTaskRunner
*/
$queueTaskRunner = Injector::inst()->get(QueuedTaskRunner::class);
$getVars = $request->getVars();
$taskCount = (int) $getVars['subtasks'] ?: 2;
unset($getVars[self::$planQueueName]);
unset($getVars['subtasks']);
$maxFileID = (int) File::get()->max('ID');
$parts = ceil($maxFileID/$taskCount);
ob_end_flush();
$from = $to = 0;
for ($i = 0; $i < $taskCount; $i++) {
if ($i === $taskCount - 1) {
$to = $maxFileID;
} else {
$to = $parts * ($i + 1);
}
$getVars['from'] = $from;
$getVars['to'] = $to;
$newReq = new HTTPRequest(
$request->httpMethod(),
$request->getURL(false),
$getVars,
$request->postVars()
);
$newReq->setRouteParams(['TaskName' => 'FilePaginated']);
$queueTaskRunner->queueTask($newReq);
$from = $to + 1;
}
//Prevent messages from QueuedTaskRunner
ob_clean();
ob_start();
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment