cURL asynchronous requests using curl_multi and callbacks

example.php

<?php

require('Requests.php');

// We want the requests to be mixed fast/slow so we can see if it's working

$slow = 'http://localhost/curl_multi/slow.php';
$fast = 'http://localhost/curl_multi/fast.php';

$urls = array();
for ($i = 0; $i < 10; $i++)
{
	if($i % 2)
	{
		print "Slow\n";
		$urls[] = $slow;
	}
	else
	{
		print "Fast\n";
		$urls[] = $fast;
	}
}


$callback = function($data, $info)
{
	print "Callback: ";
	print_r($data);
	print_r($info);
};


$requests = new Requests();
$requests->process($urls, $callback);

fast.php

<?php

usleep(mt_rand(10000, 200000));
print basename(__FILE__) . "\n";

Requests.php

<?php
/**
 * Make asynchronous requests to different resources as fast as possible and process the results as they are ready.
 */
class Requests
{
	public $handle;

	public function __construct()
	{
		$this->handle = curl_multi_init();
	}

	public function process($urls, $callback)
	{
		foreach($urls as $url)
		{
			$ch = curl_init($url);
			curl_setopt_array($ch, array(CURLOPT_RETURNTRANSFER => TRUE));
			curl_multi_add_handle($this->handle, $ch);
		}

		do {
			$mrc = curl_multi_exec($this->handle, $active);

			if ($state = curl_multi_info_read($this->handle))
			{
				//print_r($state);
				$info = curl_getinfo($state['handle']);
				//print_r($info);
				$callback(curl_multi_getcontent($state['handle']), $info);
				curl_multi_remove_handle($this->handle, $state['handle']);
			}

			usleep(10000); // stop wasting CPU cycles and rest for a couple ms

		} while ($mrc == CURLM_CALL_MULTI_PERFORM || $active);

	}

	public function __destruct()
	{
		curl_multi_close($this->handle);
	}
}

rolling_curl.php

<?php
// This limits the request rate to the size of the $rolling_window
// http://www.onlineaspect.com/2009/01/26/how-to-use-curl_multi-without-blocking/
function rolling_curl($urls, $callback, $custom_options = null)
{

	// make sure the rolling window isn't greater than the # of urls
	$rolling_window = 5;
	$rolling_window = (sizeof($urls) < $rolling_window) ? sizeof($urls) : $rolling_window;

	$master = curl_multi_init();
	$curl_arr = array();

	// add additional curl options here
	$std_options = array(
		CURLOPT_RETURNTRANSFER => true,
		CURLOPT_FOLLOWLOCATION => true,
		CURLOPT_MAXREDIRS => 5
	);

	$options = ($custom_options) ? ($std_options + $custom_options) : $std_options;

	// start the first batch of requests
	for ($i = 0; $i < $rolling_window; $i++)
	{
		$ch = curl_init();
		$options[CURLOPT_URL] = $urls[$i];
		curl_setopt_array($ch,$options);
		curl_multi_add_handle($master, $ch);
	}

	do {
		while(($execrun = curl_multi_exec($master, $running)) == CURLM_CALL_MULTI_PERFORM);
			if($execrun != CURLM_OK)
				break;

		// a request was just completed -- find out which one
		while($done = curl_multi_info_read($master))
		{
			$info = curl_getinfo($done['handle']);

			$output = curl_multi_getcontent($done['handle']);

			// request successful.  process output using the callback function.
			$callback($output, $info);

			if(isset($urls[$i + 1]))
			{
				// start a new request (it's important to do this before removing the old one)
				$ch = curl_init();
				$options[CURLOPT_URL] = $urls[$i++];  // increment i
				curl_setopt_array($ch,$options);
				curl_multi_add_handle($master, $ch);
			}

			// remove the curl handle that just completed
			curl_multi_remove_handle($master, $done['handle']);
		}

		usleep(10000); // stop wasting CPU cycles and rest for a couple ms

	} while ($running);

	curl_multi_close($master);
}

slow.php

<?php

usleep(mt_rand(100000, 600000));
print basename(__FILE__) . "\n";