Last active
November 9, 2018 22:31
-
-
Save abarriosr/7ba5b476e4c7f08c61337b94eac9753f to your computer and use it in GitHub Desktop.
Bulk exports all entities according to the selection on the Entity Configuration page. IMPORTANT: Use only with Export Queue Enabled.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?php | |
/** | |
* Process a subset of all the entities to be enqueued in a single request. | |
* | |
* @param $entity_type | |
* The entity type. | |
* @param $bundle | |
* The entity bundle. | |
* @param $bundle_key | |
* THe entity bundle key. | |
*/ | |
function export_enqueue_entities($entity_type, $bundle, $entity_ids, &$context) { | |
/** | |
* Number of entities per iteration. Decrease this number if your site has | |
* too many dependencies per node. | |
* | |
* @var int $entities_per_iteration | |
*/ | |
$entities_per_iteration = 5; | |
if (empty($context['sandbox'])) { | |
$context['sandbox']['progress'] = 0; | |
$context['sandbox']['max'] = count($entity_ids); | |
$context['results']['total'] = 0; | |
} | |
/** @var \Drupal\acquia_contenthub\EntityManager $entity_manager */ | |
$entity_manager = \Drupal::service('acquia_contenthub.entity_manager'); | |
/** @var \Drupal\acquia_contenthub\Controller\ContentHubEntityExportController $export_controller */ | |
$export_controller = \Drupal::service('acquia_contenthub.acquia_contenthub_export_entities'); | |
$slice_entity_ids = array_slice($entity_ids, $context['sandbox']['progress'], $entities_per_iteration); | |
$ids = array_values($slice_entity_ids); | |
if (!empty($ids)) { | |
$entities = \Drupal::entityTypeManager() | |
->getStorage($entity_type) | |
->loadMultiple($ids); | |
foreach ($entities as $entity) { | |
if ($entity_manager->isEligibleEntity($entity)) { | |
// Entity is eligible, then re-export. | |
$export_controller->exportEntities([$entity]); | |
} | |
} | |
} | |
$context['sandbox']['progress'] += count($ids); | |
$enqueued = implode(',', $ids); | |
$message = empty($enqueued) ? "Enqueuing '$entity_type' ($bundle) entities: No entities to queue." : "Enqueuing '$entity_type' ($bundle) entities with IDs: " . $enqueued . "\n"; | |
$context['results']['total'] += count($ids); | |
$context['message'] = $message; | |
if ($context['sandbox']['progress'] != $context['sandbox']['max']) { | |
$context['finished'] = $context['sandbox']['progress'] / $context['sandbox']['max']; | |
} | |
} | |
function export_enqueue_finished($success, $results, $operations) { | |
// The 'success' parameter means no fatal PHP errors were detected. All | |
// other error management should be handled using 'results'. | |
if ($success) { | |
$message = 'Total number of enqueued entities: ' . $results['total']; | |
} | |
else { | |
$message = t('Finished with an error.'); | |
} | |
drush_print($message); | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?php | |
/** | |
* @file | |
* Bulk Exports all entities to Content Hub. | |
* | |
* IMPORTANT: You should only use this script with the Export Queue Enabled. | |
* | |
* How to use: | |
* | |
* 1. Go to admin/config/services/acquia-contenthub/configuration and select | |
* all the entity types and bundles that you want to export to Content Hub. | |
* | |
* 2. Be gentle to the exporting site by increasing the export queue waiting | |
* time to 5 seconds and a queue batch size of 5 entities if your site is not | |
* packed with too many dependencies. You should decide according to your | |
* content structure how many entities per item and batch size would fit | |
* your situation better. | |
* | |
* 3. Execute this script in the following way: | |
* $drush scr ../scripts/ach-bulk-export.php | |
* | |
* (Assuming the script files were installed in the '/script' directory). | |
* | |
* This will queue all selected entity types and bundles in the export queue. | |
* | |
* 4. Go to admin/config/services/acquia-contenthub/export-queue and run the | |
* export queue. | |
*/ | |
use Drupal\acquia_contenthub\ContentHubEntityDependency; | |
use Drupal\acquia_contenthub\ContentHubEntitiesTracking; | |
const IGNORE_EXPORTED_ENTITIES = TRUE; | |
$entity_manager = \Drupal::getContainer()->get('acquia_contenthub.entity_manager'); | |
// Read all configured entity types and bundles from Content Hub. | |
/** @var Drupal\acquia_contenthub\ContentHubEntityTypeConfigInterface[] $contenthub_types */ | |
$contenthub_types = $entity_manager->getContentHubEntityTypeConfigurationEntities(); | |
// Move nodes to the end of the array. | |
// The reason is because we want first to export all entity types that a node | |
// might have as dependencies. Then when we export nodes, those entities that | |
// could potentially be dependencies of those nodes won't be exported anymore, | |
// thus decreasing the export load. | |
if (isset($contenthub_types['node'])) { | |
$node_types_config = $contenthub_types['node']; | |
unset($contenthub_types['node']); | |
} | |
$operations = []; | |
// Obtain list of dependent entity types. | |
$dependent_entity_type_ids = ContentHubEntityDependency::getPostDependencyEntityTypes(); | |
foreach ($contenthub_types as $entity_type => $entity_type_config) { | |
// If it is a dependent entity, do not process because they will be | |
// imported as dependencies. | |
if (in_array($entity_type, $dependent_entity_type_ids)) { | |
continue; | |
} | |
process_entity_type($entity_type, $entity_type_config, $operations); | |
} | |
// Finally process the node. | |
process_entity_type('node', $node_types_config, $operations); | |
$batch = [ | |
'title' => "Enqueue entities for export", | |
'file' => '../scripts/ach-bulk-export-batch-functions.php', | |
'operations' => $operations, | |
'finished' => 'export_enqueue_finished', | |
]; | |
// Unsetting some variables not needed anymore. | |
unset($dependent_entity_type_ids); | |
unset($entity_manager); | |
unset($contenthub_types); | |
// Batch processing. | |
batch_set($batch); | |
// Start the batch process. | |
drush_backend_batch_process(); | |
/** | |
* Exports all entities for a specific entity type. | |
* | |
* @param string $entity_type | |
* The entity type. | |
* @param \Drupal\acquia_contenthub\ContentHubEntityTypeConfigInterface $entity_type_config | |
* The Content Hub Entity Type Configuration Entity. | |
*/ | |
function process_entity_type($entity_type, $entity_type_config, &$operations) { | |
$bundles = array_keys($entity_type_config->getBundles()); | |
// $drupal_bundles = \Drupal::entityManager()->getBundleInfo($entity_type); | |
/** @var \Drupal\Core\Entity\EntityTypeManagerInterface $entity_type_manager */ | |
$entity_type_manager = \Drupal::entityTypeManager(); | |
$bundle_key = $entity_type_manager->getDefinition($entity_type)->getKey('bundle'); | |
foreach ($bundles as $bundle) { | |
if ($entity_type_config->isEnableIndex($bundle) === FALSE) { | |
// Do not process bundle if it is not enabled. | |
continue; | |
} | |
print("Checking type = $entity_type, bundle = $bundle..."); | |
// For all enabled bundles. | |
/** @var \Drupal\Core\Entity\Query\QueryInterface $query */ | |
$query = \Drupal::entityQuery($entity_type); | |
if ($entity_type === 'node') { | |
$query->condition('status', 1); | |
} | |
// If we do not have file bundles but files have been marked as "exportable" | |
// then do not include a condition for bundle. | |
if (!empty($bundle_key)) { | |
$query->condition($bundle_key, $bundle); | |
} | |
$entity_ids = array_values($query->execute()); | |
$filtered_entity_ids = (!empty($entity_ids) && IGNORE_EXPORTED_ENTITIES) ? get_entities_not_exported($entity_ids, $entity_type) : $entity_ids; | |
unset($query); | |
print "Found " . count($filtered_entity_ids) . " entities.\n"; | |
if (!empty($filtered_entity_ids)) { | |
$operations[] = [ | |
'export_enqueue_entities', | |
[$entity_type, $bundle, $filtered_entity_ids] | |
]; | |
} | |
} | |
} | |
/** | |
* Provided a list of entities to export, it filters to not exported ones. | |
* | |
* @param array $candidate_entities | |
* An array of candidate entities to export | |
* @param string $entity_type | |
* The entity type. | |
* | |
* @return array | |
* An array of entities not previously exported. | |
*/ | |
function get_entities_not_exported($candidate_entities, $entity_type) { | |
/** @var \Drupal\Core\Database\Connection $database */ | |
$database = \Drupal::service('database'); | |
$query = $database->select(ContentHubEntitiesTracking::TABLE, 'acet'); | |
$query->addField('acet', 'entity_id'); | |
// Adding Conditions. | |
$query->condition('entity_id', $candidate_entities, 'IN') | |
->condition('entity_type', $entity_type, '=') | |
->condition('status_export', 'EXPORTED'); | |
$result = $query->execute()->fetchAllKeyed(); | |
$exported_entities = array_keys($result); | |
return array_diff($candidate_entities, $exported_entities); | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?php | |
/** | |
* @file | |
* [DEPRECATED: Only use it if the site has a few entities] | |
* Bulk Exports all entities to Content Hub. | |
* | |
* IMPORTANT: You should only use this script with the Export Queue Enabled. | |
* | |
* How to use: | |
* | |
* 1. Go to admin/config/services/acquia-contenthub/configuration and select | |
* all the entity types and bundles that you want to export to Content Hub. | |
* | |
* 2. Be gentle to the exporting site by increasing the export queue waiting | |
* time to 5 seconds and a queue batch size of 5 entities if your site is not | |
* packed with too many dependencies. You should decide according to your | |
* content structure how many entities per item and batch size would fit | |
* your situation better. | |
* | |
* 3. Execute this script in the following way: | |
* $drush scr <PATH TO FILE>/bulkexport.php | |
* | |
* This will queue all selected entity types and bundles in the export queue. | |
* | |
* 4. Go to admin/config/services/acquia-contenthub/export-queue and run the | |
* export queue. | |
*/ | |
use Drupal\acquia_contenthub\ContentHubEntityDependency; | |
/** | |
* Number of entities per iteration. Decrease this number if your site has | |
* too many dependencies per node. | |
* | |
* @var int $entities_per_iteration | |
*/ | |
$entities_per_iteration = 5; | |
$entity_manager = \Drupal::getContainer()->get('acquia_contenthub.entity_manager'); | |
// Read all configured entity types and bundles from Content Hub. | |
/** | |
* @var Drupal\acquia_contenthub\ContentHubEntityTypeConfigInterface[] $contenthub_types */ | |
$contenthub_types = $entity_manager->getContentHubEntityTypeConfigurationEntities(); | |
// Move nodes to the end of the array. | |
// The reason is because we want first to export all entity types that a node | |
// might have as dependencies. Then when we export nodes, those entities that | |
// could potentially be dependencies of those nodes won't be exported anymore, | |
// thus decreasing the export load. | |
if (isset($contenthub_types['node'])) { | |
$node_types_config = $contenthub_types['node']; | |
unset($contenthub_types['node']); | |
} | |
// Obtain list of dependent entity types. | |
$dependent_entity_type_ids = ContentHubEntityDependency::getPostDependencyEntityTypes(); | |
foreach ($contenthub_types as $entity_type => $entity_type_config) { | |
// If it is a dependent entity, do not process because they will be | |
// imported as dependencies. | |
if (in_array($entity_type, $dependent_entity_type_ids)) { | |
continue; | |
} | |
process_entity_type($entity_type, $entity_type_config); | |
} | |
// Finally process the node. | |
process_entity_type('node', $node_types_config); | |
/** | |
* Exports all entities for a specific entity type. | |
* | |
* @param string $entity_type | |
* The entity type. | |
* @param \Drupal\acquia_contenthub\ContentHubEntityTypeConfigInterface $entity_type_config | |
* The Content Hub Entity Type Configuration Entity. | |
*/ | |
function process_entity_type($entity_type, $entity_type_config) { | |
/** @var \Drupal\acquia_contenthub\Controller\ContentHubEntityExportController $export_controller */ | |
$export_controller = \Drupal::service('acquia_contenthub.acquia_contenthub_export_entities'); | |
$bundles = array_keys($entity_type_config->getBundles()); | |
// $drupal_bundles = \Drupal::entityManager()->getBundleInfo($entity_type); | |
/** @var \Drupal\Core\Entity\EntityTypeManagerInterface $entity_type_manager */ | |
$entity_type_manager = \Drupal::entityTypeManager(); | |
$bundle_key = $entity_type_manager->getDefinition($entity_type)->getKey('bundle'); | |
foreach ($bundles as $bundle) { | |
if ($entity_type_config->isEnableIndex($bundle) === FALSE) { | |
// Do not process bundle if it is not enabled. | |
continue; | |
} | |
print("Checking type = $entity_type, bundle = $bundle..."); | |
// For all enabled bundles. | |
/** @var \Drupal\Core\Entity\Query\QueryInterface $query */ | |
$query = \Drupal::entityQuery($entity_type); | |
if ($entity_type === 'node') { | |
$query->condition('status', 1); | |
} | |
// If we do not have file bundles but files have been marked as "exportable" | |
// then do not include a condition for bundle. | |
if (!empty($bundle_key)) { | |
$query->condition($bundle_key, $bundle); | |
} | |
$entity_ids = array_values($query->execute()); | |
print "Found " . count($entity_ids) . " entities.\n"; | |
$entities_ids = array_chunk($entity_ids, $entities_per_iteration); | |
foreach ($entities_ids as $ids) { | |
$entities = \Drupal::entityTypeManager()->getStorage($entity_type)->loadMultiple($ids); | |
$export_controller->exportEntities($entities); | |
print "-> Exporting '$entity_type' ($bundle) entities with IDs: " . implode(',', $ids) . "\n"; | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment