Example of appending requests on the fly using an ArrayIterator as generator and comparing that approach to $event->retry() and $client->send()

guzzle-append-requests-on-the-fly-generator-vs-retry.php

<?php
/** CAUTION!!!
 * You need to have a webserver running on localhost to make this work!
 */

use GuzzleHttp\Client;
use GuzzleHttp\Event\AbstractRetryableEvent;
use GuzzleHttp\Message\Request;
use GuzzleHttp\Pool;

include_once "vendor/autoload.php";

$client = new Client();

$num = 10; // number of total requests

$timeout = 1;

$url = "http://localhost/timeout.php?t={$timeout}"; // each response will take about 1 second to be returned
/** content of
// timeout.php
<?php
    if(isset($_GET["t"])){
        $t = (int)$_GET["t"];
        sleep($t);
        echo "Slept for $t seconds";
    }
    die();
?>
 */

// some helper functions
/**
 * @param int $id
 * @return Request|\GuzzleHttp\Message\Request
 */
$createRequestFn = function($id) use ($client, $url){
    $req = $client->createRequest("GET",$url);
    $req->getConfig()->set("id",$id);
    $requests[] = $req;
    return $req;
};
/**
 * @param int $num
 * @return \GuzzleHttp\Message\Request[] $createRequestsFn
 */
$createRequestsFn = function($num) use ($createRequestFn){
    $requests = [];
    for($i = 0; $i < $num; $i++){
        $requests [] = $createRequestFn($i);
    }
    return $requests;
};

/** @return callable */
$getGetTimeFn = function() {
    $start = microtime(true);
    $getTimeFn = function () use ($start) {
        $end = microtime(true);
        $time = round($end - $start, 2);
        return $time;
    };
    return $getTimeFn;
};

//generator
/** @var Request[] $requests */
$test = "parallel with generator";

$parallel = 2;
$requestNum = 5;
$maxTries = $num/$requestNum;
$total = 0;

echo "Testing '{$test}'\n";
$requests = $createRequestsFn(5);
$generator = new ArrayIterator($requests);

$getTimeFn = $getGetTimeFn();
$retryFn = function(AbstractRetryableEvent $ev) use (&$total, $maxTries, $getTimeFn, $generator, $createRequestFn){
    $request = $ev->getRequest();
    $total++;
    $retries = $request->getConfig()->get("retries");
    $response = $ev->getResponse();
    if($response === null){
        $status = "[ERROR]";
    }else{
        $status = $response->getStatusCode();
    }
    $time = $getTimeFn();
    $id = $request->getConfig()->get("id");
    if($retries === null){
        $retries = 0;
    }
    echo "[Request {$total} - {$time} s] Got request with id ".$id." and response status code ".$status."\n";
    $retries++;
    if($retries < $maxTries){
        /** @var Request $request */
        $request = $createRequestFn($id."_".$retries);
        $request->getConfig()->set("retries",$retries);
        $generator->append($request);
    }
};
$options = [
    "pool_size" => $parallel,
    "complete" => $retryFn,
    "error" => $retryFn,
];
$pool = new Pool($client,$generator,$options);
$pool->wait();
$time = $getTimeFn();
echo "Total time for '{$test}': ".$time."s\n\n";

//retry
/** @var Request[] $requests */
$test = "parallel with retry";

$parallel = 2;
$requestNum = 5;
$maxTries = $num/$requestNum;
$total = 0;

echo "Testing '{$test}'\n";
$requests = $createRequestsFn(5);
$getTimeFn = $getGetTimeFn();
$retryFn = function(AbstractRetryableEvent $ev) use (&$total, $maxTries, $getTimeFn){
    $request = $ev->getRequest();
    $total++;
    $retries = $request->getConfig()->get("retries");
    $response = $ev->getResponse();
    if($response === null){
        $status = "[ERROR]";
    }else{
        $status = $response->getStatusCode();
    }
    $time = $getTimeFn();
    $id = $request->getConfig()->get("id");
    if($retries === null){
        $retries = 0;
    }
    echo "[Request {$total} - {$time} s] Got request with id ".$id." (try $retries) and response status code ".$status."\n";
    $retries++;
    if($retries < $maxTries){
        $request->getConfig()->set("retries",$retries);
        $ev->retry();
    }
};
$options = [
    "pool_size" => $parallel,
    "complete" => $retryFn,
    "error" => $retryFn,
];
$pool = new Pool($client,$requests,$options);
$pool->wait();
$time = $getTimeFn();
echo "Total time for '{$test}': ".$time."s\n\n";


//sequential
/** @var Request[] $requests */
$test = "Sequential";
echo "Testing '{$test}'\n";
$total = 0;
$requests = $createRequestsFn($num);
$getTimeFn = $getGetTimeFn();
foreach($requests as $request){
    try {
        $response = $client->send($request);
        $status = $response->getStatusCode();
    }catch(Exception $e){
        $status = "[ERROR]";
    }
    $total++;
    $time = $getTimeFn();
    $id = $request->getConfig()->get("id");
    echo "[Request {$total} - {$time} s] Got request with id ".$id." and response status code ".$status."\n";
}
$time = $getTimeFn();
echo "Total time for '{$test}': ".$time."s\n";

/** Expected output:
Testing 'parallel with generator'
[Request 1 - 1.03 s] Got request with id 0 and response status code 200
[Request 2 - 1.05 s] Got request with id 1 and response status code 200
[Request 3 - 2.06 s] Got request with id 2 and response status code 200
[Request 4 - 2.07 s] Got request with id 3 and response status code 200
[Request 5 - 3.07 s] Got request with id 4 and response status code 200
[Request 6 - 3.08 s] Got request with id 0_1 and response status code 200
[Request 7 - 4.1 s] Got request with id 2_1 and response status code 200
[Request 8 - 4.1 s] Got request with id 1_1 and response status code 200
[Request 9 - 5.11 s] Got request with id 3_1 and response status code 200
[Request 10 - 6.11 s] Got request with id 4_1 and response status code 200
Total time for 'parallel with generator': 6.11s

Testing 'parallel with retry'
[Request 1 - 1.02 s] Got request with id 0 (try 0) and response status code 200
[Request 2 - 1.02 s] Got request with id 1 (try 0) and response status code 200
[Request 3 - 2.03 s] Got request with id 0 (try 1) and response status code 200
[Request 4 - 2.03 s] Got request with id 1 (try 1) and response status code 200
[Request 5 - 3.04 s] Got request with id 2 (try 0) and response status code 200
[Request 6 - 3.05 s] Got request with id 3 (try 0) and response status code 200
[Request 7 - 4.07 s] Got request with id 2 (try 1) and response status code 200
[Request 8 - 4.07 s] Got request with id 3 (try 1) and response status code 200
[Request 9 - 5.08 s] Got request with id 4 (try 0) and response status code 200
[Request 10 - 6.09 s] Got request with id 4 (try 1) and response status code 200
Total time for 'parallel with retry': 6.09s

Testing 'Sequential'
[Request 1 - 1.01 s] Got request with id 0 and response status code 200
[Request 2 - 2.02 s] Got request with id 1 and response status code 200
[Request 3 - 3.03 s] Got request with id 2 and response status code 200
[Request 4 - 4.03 s] Got request with id 3 and response status code 200
[Request 5 - 5.04 s] Got request with id 4 and response status code 200
[Request 6 - 6.05 s] Got request with id 5 and response status code 200
[Request 7 - 7.07 s] Got request with id 6 and response status code 200
[Request 8 - 8.08 s] Got request with id 7 and response status code 200
[Request 9 - 9.09 s] Got request with id 8 and response status code 200
[Request 10 - 10.1 s] Got request with id 9 and response status code 200
Total time for 'Sequential': 10.1s
*/