Batch with Memory Efficiency

wp-scan-images-batch-v2.php

<?php
// Batch Process
function process_image_batch($batch_size = 100, $reset = false)
{

    $directory = wp_get_upload_dir()['basedir'];
    $json_file = trailingslashit(WP_CONTENT_DIR) . 'image.json';

    // First batch → reset file
    if ($reset || !file_exists($json_file)) {
        file_put_contents($json_file, json_encode([], JSON_PRETTY_PRINT));
    }

    // Get already stored images
    $stored = json_decode(file_get_contents($json_file), true);
    if (!is_array($stored)) {
        $stored = [];
    }

    $lookup = array_flip($stored); // Skip already stored images.

    $batch = [];
    $count = 0;

    $memory_limit_bytes = wp_convert_hr_to_bytes(ini_get('memory_limit'));
    if ($memory_limit_bytes <= 0) {
        $memory_limit_bytes = 256 * 1024 * 1024; // Default 256MB if unlimited
    }


    $safe_threshold = (int) ($memory_limit_bytes * 0.75); // 75% of memory limit
    // $safe_threshold = 40 * 1024 * 1024; // 40 MB For testing.

    // Generator to stream files
    foreach (get_images_generator($directory) as $image) {

        // Skip already stored images
        if (isset($lookup[$image])) continue; // O(1) lookup

        $batch[] = $image;
        $count++;
        
        // Check after every 10 items
        if (0 === $count % 10) {
            $current_memory = memory_get_usage(false);

            // Early return if memory usage goes ot threshold
            if ($current_memory >= $safe_threshold) {
                error_log("⚠️ Memory limit hit (" . round($current_memory / 1024 / 1024, 2) . " MB). Returning batch early.");
                gc_collect_cycles();
                usleep(500000); // pause 0.5 second for not hammering server.
                break;
            }
        }

        if ($count >= $batch_size) break; // stop after batch_size images
    }

    // Append new batch to JSON
    if (!empty($batch)) {
        $stored = array_merge($stored, $batch);
        file_put_contents($json_file, json_encode($stored, JSON_PRETTY_PRINT));
    }

    return $batch;
}

// Generator
function get_images_generator($directory)
{
    $iterator = new RecursiveIteratorIterator(
        new RecursiveDirectoryIterator($directory, FilesystemIterator::SKIP_DOTS)
    );

    foreach ($iterator as $file) {
        if ($file->isFile() && in_array(strtolower($file->getExtension()), ['jpg', 'jpeg', 'png'], true)) {
            yield $file->getPathname();
        }
    }
}