dealing with Drupal 8 entity_load_multiple crashes on massive datasets via garbage collections.

gistfile1.txt

<?php

$nids = [];
// example nids has 10k+
// $nids = some_logic();
$node_records = []; $throughput = 50; $nids = array_chunk($nids, $throughput);
foreach ($nids as $subset_id => $subset_nids) {
  // var_dump([$subset_id,memory_get_peak_usage(),]);
  $nodes = [];
  $nodes = entity_load_multiple('node', $subset_nids, TRUE);
  // var_dump(['entity-load-multiple', memory_get_peak_usage(),]);
  foreach ($nodes as $nid => $node) {
  // some logic for processing
  // make sure to unset variables
  }
  drush_print_r("$subset_id of $total " . 'with memory usage of ' . cduxx_attempt_memory_reclaim());
}

function cduxx_attempt_memory_reclaim() {
  // inspired by https://www.drupal.org/project/drupal/issues/2701335
  // First, try resetting Drupal's static storage - this frequently releases
  // plenty of memory to continue.
  drupal_static_reset();
  // Entity storage can blow up with caches so clear them out.
  $manager = \Drupal::entityManager();
  foreach ($manager->getDefinitions() as $id => $definition) {
    $manager->getStorage($id)->resetCache();
  }
  // @TODO: explore resetting the container.
  // Run garbage collector to further reduce memory.
  gc_collect_cycles();
  return memory_get_usage();
}