Skip to content

Instantly share code, notes, and snippets.

@abarriosr
Last active November 9, 2018 22:31
Show Gist options
  • Save abarriosr/7ba5b476e4c7f08c61337b94eac9753f to your computer and use it in GitHub Desktop.
Save abarriosr/7ba5b476e4c7f08c61337b94eac9753f to your computer and use it in GitHub Desktop.
Bulk exports all entities according to the selection on the Entity Configuration page. IMPORTANT: Use only with Export Queue Enabled.
<?php
/**
* Process a subset of all the entities to be enqueued in a single request.
*
* @param $entity_type
* The entity type.
* @param $bundle
* The entity bundle.
* @param $bundle_key
* THe entity bundle key.
*/
function export_enqueue_entities($entity_type, $bundle, $entity_ids, &$context) {
/**
* Number of entities per iteration. Decrease this number if your site has
* too many dependencies per node.
*
* @var int $entities_per_iteration
*/
$entities_per_iteration = 5;
if (empty($context['sandbox'])) {
$context['sandbox']['progress'] = 0;
$context['sandbox']['max'] = count($entity_ids);
$context['results']['total'] = 0;
}
/** @var \Drupal\acquia_contenthub\EntityManager $entity_manager */
$entity_manager = \Drupal::service('acquia_contenthub.entity_manager');
/** @var \Drupal\acquia_contenthub\Controller\ContentHubEntityExportController $export_controller */
$export_controller = \Drupal::service('acquia_contenthub.acquia_contenthub_export_entities');
$slice_entity_ids = array_slice($entity_ids, $context['sandbox']['progress'], $entities_per_iteration);
$ids = array_values($slice_entity_ids);
if (!empty($ids)) {
$entities = \Drupal::entityTypeManager()
->getStorage($entity_type)
->loadMultiple($ids);
foreach ($entities as $entity) {
if ($entity_manager->isEligibleEntity($entity)) {
// Entity is eligible, then re-export.
$export_controller->exportEntities([$entity]);
}
}
}
$context['sandbox']['progress'] += count($ids);
$enqueued = implode(',', $ids);
$message = empty($enqueued) ? "Enqueuing '$entity_type' ($bundle) entities: No entities to queue." : "Enqueuing '$entity_type' ($bundle) entities with IDs: " . $enqueued . "\n";
$context['results']['total'] += count($ids);
$context['message'] = $message;
if ($context['sandbox']['progress'] != $context['sandbox']['max']) {
$context['finished'] = $context['sandbox']['progress'] / $context['sandbox']['max'];
}
}
function export_enqueue_finished($success, $results, $operations) {
// The 'success' parameter means no fatal PHP errors were detected. All
// other error management should be handled using 'results'.
if ($success) {
$message = 'Total number of enqueued entities: ' . $results['total'];
}
else {
$message = t('Finished with an error.');
}
drush_print($message);
}
<?php
/**
* @file
* Bulk Exports all entities to Content Hub.
*
* IMPORTANT: You should only use this script with the Export Queue Enabled.
*
* How to use:
*
* 1. Go to admin/config/services/acquia-contenthub/configuration and select
* all the entity types and bundles that you want to export to Content Hub.
*
* 2. Be gentle to the exporting site by increasing the export queue waiting
* time to 5 seconds and a queue batch size of 5 entities if your site is not
* packed with too many dependencies. You should decide according to your
* content structure how many entities per item and batch size would fit
* your situation better.
*
* 3. Execute this script in the following way:
* $drush scr ../scripts/ach-bulk-export.php
*
* (Assuming the script files were installed in the '/script' directory).
*
* This will queue all selected entity types and bundles in the export queue.
*
* 4. Go to admin/config/services/acquia-contenthub/export-queue and run the
* export queue.
*/
use Drupal\acquia_contenthub\ContentHubEntityDependency;
use Drupal\acquia_contenthub\ContentHubEntitiesTracking;
const IGNORE_EXPORTED_ENTITIES = TRUE;
$entity_manager = \Drupal::getContainer()->get('acquia_contenthub.entity_manager');
// Read all configured entity types and bundles from Content Hub.
/** @var Drupal\acquia_contenthub\ContentHubEntityTypeConfigInterface[] $contenthub_types */
$contenthub_types = $entity_manager->getContentHubEntityTypeConfigurationEntities();
// Move nodes to the end of the array.
// The reason is because we want first to export all entity types that a node
// might have as dependencies. Then when we export nodes, those entities that
// could potentially be dependencies of those nodes won't be exported anymore,
// thus decreasing the export load.
if (isset($contenthub_types['node'])) {
$node_types_config = $contenthub_types['node'];
unset($contenthub_types['node']);
}
$operations = [];
// Obtain list of dependent entity types.
$dependent_entity_type_ids = ContentHubEntityDependency::getPostDependencyEntityTypes();
foreach ($contenthub_types as $entity_type => $entity_type_config) {
// If it is a dependent entity, do not process because they will be
// imported as dependencies.
if (in_array($entity_type, $dependent_entity_type_ids)) {
continue;
}
process_entity_type($entity_type, $entity_type_config, $operations);
}
// Finally process the node.
process_entity_type('node', $node_types_config, $operations);
$batch = [
'title' => "Enqueue entities for export",
'file' => '../scripts/ach-bulk-export-batch-functions.php',
'operations' => $operations,
'finished' => 'export_enqueue_finished',
];
// Unsetting some variables not needed anymore.
unset($dependent_entity_type_ids);
unset($entity_manager);
unset($contenthub_types);
// Batch processing.
batch_set($batch);
// Start the batch process.
drush_backend_batch_process();
/**
* Exports all entities for a specific entity type.
*
* @param string $entity_type
* The entity type.
* @param \Drupal\acquia_contenthub\ContentHubEntityTypeConfigInterface $entity_type_config
* The Content Hub Entity Type Configuration Entity.
*/
function process_entity_type($entity_type, $entity_type_config, &$operations) {
$bundles = array_keys($entity_type_config->getBundles());
// $drupal_bundles = \Drupal::entityManager()->getBundleInfo($entity_type);
/** @var \Drupal\Core\Entity\EntityTypeManagerInterface $entity_type_manager */
$entity_type_manager = \Drupal::entityTypeManager();
$bundle_key = $entity_type_manager->getDefinition($entity_type)->getKey('bundle');
foreach ($bundles as $bundle) {
if ($entity_type_config->isEnableIndex($bundle) === FALSE) {
// Do not process bundle if it is not enabled.
continue;
}
print("Checking type = $entity_type, bundle = $bundle...");
// For all enabled bundles.
/** @var \Drupal\Core\Entity\Query\QueryInterface $query */
$query = \Drupal::entityQuery($entity_type);
if ($entity_type === 'node') {
$query->condition('status', 1);
}
// If we do not have file bundles but files have been marked as "exportable"
// then do not include a condition for bundle.
if (!empty($bundle_key)) {
$query->condition($bundle_key, $bundle);
}
$entity_ids = array_values($query->execute());
$filtered_entity_ids = (!empty($entity_ids) && IGNORE_EXPORTED_ENTITIES) ? get_entities_not_exported($entity_ids, $entity_type) : $entity_ids;
unset($query);
print "Found " . count($filtered_entity_ids) . " entities.\n";
if (!empty($filtered_entity_ids)) {
$operations[] = [
'export_enqueue_entities',
[$entity_type, $bundle, $filtered_entity_ids]
];
}
}
}
/**
* Provided a list of entities to export, it filters to not exported ones.
*
* @param array $candidate_entities
* An array of candidate entities to export
* @param string $entity_type
* The entity type.
*
* @return array
* An array of entities not previously exported.
*/
function get_entities_not_exported($candidate_entities, $entity_type) {
/** @var \Drupal\Core\Database\Connection $database */
$database = \Drupal::service('database');
$query = $database->select(ContentHubEntitiesTracking::TABLE, 'acet');
$query->addField('acet', 'entity_id');
// Adding Conditions.
$query->condition('entity_id', $candidate_entities, 'IN')
->condition('entity_type', $entity_type, '=')
->condition('status_export', 'EXPORTED');
$result = $query->execute()->fetchAllKeyed();
$exported_entities = array_keys($result);
return array_diff($candidate_entities, $exported_entities);
}
<?php
/**
* @file
* [DEPRECATED: Only use it if the site has a few entities]
* Bulk Exports all entities to Content Hub.
*
* IMPORTANT: You should only use this script with the Export Queue Enabled.
*
* How to use:
*
* 1. Go to admin/config/services/acquia-contenthub/configuration and select
* all the entity types and bundles that you want to export to Content Hub.
*
* 2. Be gentle to the exporting site by increasing the export queue waiting
* time to 5 seconds and a queue batch size of 5 entities if your site is not
* packed with too many dependencies. You should decide according to your
* content structure how many entities per item and batch size would fit
* your situation better.
*
* 3. Execute this script in the following way:
* $drush scr <PATH TO FILE>/bulkexport.php
*
* This will queue all selected entity types and bundles in the export queue.
*
* 4. Go to admin/config/services/acquia-contenthub/export-queue and run the
* export queue.
*/
use Drupal\acquia_contenthub\ContentHubEntityDependency;
/**
* Number of entities per iteration. Decrease this number if your site has
* too many dependencies per node.
*
* @var int $entities_per_iteration
*/
$entities_per_iteration = 5;
$entity_manager = \Drupal::getContainer()->get('acquia_contenthub.entity_manager');
// Read all configured entity types and bundles from Content Hub.
/**
* @var Drupal\acquia_contenthub\ContentHubEntityTypeConfigInterface[] $contenthub_types */
$contenthub_types = $entity_manager->getContentHubEntityTypeConfigurationEntities();
// Move nodes to the end of the array.
// The reason is because we want first to export all entity types that a node
// might have as dependencies. Then when we export nodes, those entities that
// could potentially be dependencies of those nodes won't be exported anymore,
// thus decreasing the export load.
if (isset($contenthub_types['node'])) {
$node_types_config = $contenthub_types['node'];
unset($contenthub_types['node']);
}
// Obtain list of dependent entity types.
$dependent_entity_type_ids = ContentHubEntityDependency::getPostDependencyEntityTypes();
foreach ($contenthub_types as $entity_type => $entity_type_config) {
// If it is a dependent entity, do not process because they will be
// imported as dependencies.
if (in_array($entity_type, $dependent_entity_type_ids)) {
continue;
}
process_entity_type($entity_type, $entity_type_config);
}
// Finally process the node.
process_entity_type('node', $node_types_config);
/**
* Exports all entities for a specific entity type.
*
* @param string $entity_type
* The entity type.
* @param \Drupal\acquia_contenthub\ContentHubEntityTypeConfigInterface $entity_type_config
* The Content Hub Entity Type Configuration Entity.
*/
function process_entity_type($entity_type, $entity_type_config) {
/** @var \Drupal\acquia_contenthub\Controller\ContentHubEntityExportController $export_controller */
$export_controller = \Drupal::service('acquia_contenthub.acquia_contenthub_export_entities');
$bundles = array_keys($entity_type_config->getBundles());
// $drupal_bundles = \Drupal::entityManager()->getBundleInfo($entity_type);
/** @var \Drupal\Core\Entity\EntityTypeManagerInterface $entity_type_manager */
$entity_type_manager = \Drupal::entityTypeManager();
$bundle_key = $entity_type_manager->getDefinition($entity_type)->getKey('bundle');
foreach ($bundles as $bundle) {
if ($entity_type_config->isEnableIndex($bundle) === FALSE) {
// Do not process bundle if it is not enabled.
continue;
}
print("Checking type = $entity_type, bundle = $bundle...");
// For all enabled bundles.
/** @var \Drupal\Core\Entity\Query\QueryInterface $query */
$query = \Drupal::entityQuery($entity_type);
if ($entity_type === 'node') {
$query->condition('status', 1);
}
// If we do not have file bundles but files have been marked as "exportable"
// then do not include a condition for bundle.
if (!empty($bundle_key)) {
$query->condition($bundle_key, $bundle);
}
$entity_ids = array_values($query->execute());
print "Found " . count($entity_ids) . " entities.\n";
$entities_ids = array_chunk($entity_ids, $entities_per_iteration);
foreach ($entities_ids as $ids) {
$entities = \Drupal::entityTypeManager()->getStorage($entity_type)->loadMultiple($ids);
$export_controller->exportEntities($entities);
print "-> Exporting '$entity_type' ($bundle) entities with IDs: " . implode(',', $ids) . "\n";
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment