Skip to content

Instantly share code, notes, and snippets.

@sasezaki
Created May 12, 2012 00:19
Show Gist options
  • Select an option

  • Save sasezaki/2663276 to your computer and use it in GitHub Desktop.

Select an option

Save sasezaki/2663276 to your computer and use it in GitHub Desktop.
just sandbox prototype for async spider
<?php
namespace Miffie;
use Zend\Stdlib\Parameters as StdlibParameters;
class Parameters extends StdlibParameters
{
public function __invoke($response)
{
}
}
class Runner
{
public function run()
{
$options = // 設定ファイルから
$parameterSet// = ;
$this->handleParameterSet();
}
protected function handleParameterSet()
{
//$spider = new Spider();
foreach ($parameterSet as $parameter) {
//$request = $parameter->getRequest();
//$response = $client->run($request);
//$parameter->handle($response);
$request = $this->getSpiderFirstRequest($parameter);
$response = $this->client->send($request);
$spider->invoke($response);
}
do {
$nextprocess();
}
// or
do {
$this->getLocator()->getQueue();
}
}
}
class MultiRunner
{
protected function handleParameterSet()
{
$spider = new Spider;
foreach ($parameterSet as $parameter) {
//$spider = new Spider($parameter);
$spider->setParameter($parameter);
//$request = $spider->tofirstRequest($parameter);
$request = $this->getSpiderFirstRequest($parameter);
$request->on('comlete', function ($response, $pool) use ($spider) {
$spider->invoke($response);
});
//$request->getDispatch()->addListener('success', function($event)
$this->pool->attach($request);
// $requests[] = $request;
}
}
}
class Spider
{
public function invoke($response)
{
$result = $this->scrape($response);
if ($request = $this->getNextRequest($result)) {
$spider = clone $this;
//$request->on
$this->nextprocess = function () use ($spider, $request) {
//$this->queue = function ()
$response = $spider->getLocator()->get('client')->send($send);
$spider->invoke($response);
};
}
//$this->getLocator()->get('pool');
}
}
class Spider0
{
public function run($parameter)
{
$page = 1;
$request = $parameters->getRequest();
do {
$page++;
retry:
try {
$response = $this->client->send($request);
$result = $parameters($response);
} catch (\Exception $e) {
goto retry;
}
if ($page == $parameter->get('depth')) {
goto loop_exit;
}
if ($result['nextLink']) {
//$request->setUrl($result['nextLink']);
}
} while (true);
loop_exit:
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment