Sequential dynamic requests

callback.dynamic.sequential.pl

#!/usr/bin/perl
use Mojo::Base -strict;

use Mojo::IOLoop;
use Mojo::UserAgent;
use Mojo::URL;

my $ua = Mojo::UserAgent->new(max_redirects => 3);

sub find_new_url {
  my ($url, $dom) = @_;
  return undef unless my $new_url = $dom->find('a[href]')->shuffle->first->attr('href');
  $new_url = Mojo::URL->new($new_url)->to_abs($url);
  say $new_url->to_string;
  return $new_url;
}

my $callback;
$callback = sub {
  shift; # skip $ua
  my $tx = shift;

  my $url = find_new_url($tx->req->url, $tx->res->dom);
  return Mojo::IOLoop->stop unless $url;

  $ua->get($url => $callback);
};

$ua->get('https://ru.wikipedia.org/' => $callback);

Mojo::IOLoop->start;

delay.dynamic.sequential.pl

#!/usr/bin/perl
use Mojo::Base -strict;

use Mojo::IOLoop;
use Mojo::UserAgent;
use Mojo::URL;

my $ua = Mojo::UserAgent->new(max_redirects => 3);

sub find_new_url {
  my ($url, $dom) = @_;
  # find url from response and return;
  return undef unless my $new_url = $dom->find('a[href]')->shuffle->first->attr('href');
  $new_url = Mojo::URL->new($new_url)->to_abs($url);
  say $new_url->to_string;
  return $new_url;
}

my $delay_step;
$delay_step = sub {
  shift; # skip delay
  my $tx = shift;

  my $url = find_new_url($tx->req->url, $tx->res->dom);
  return Mojo::IOLoop->stop unless $url;

  my $delay = Mojo::IOLoop->delay($delay_step);
  $ua->get($url => $delay->begin);
};

my $delay = Mojo::IOLoop->delay($delay_step);
$ua->get('https://ru.wikipedia.org/' => $delay->begin);

Mojo::IOLoop->start;

delay.dynamic.sequential2.pl

#!/usr/bin/perl
use Mojo::Base -strict;

use Mojo::IOLoop;
use Mojo::UserAgent;

my $ua = Mojo::UserAgent->new(max_redirects => 3);

my $urls = ['https://www.google.ru', 'https://twitter.com', 'http://algolist.ru'];

my $delay = Mojo::IOLoop->delay;

for my $u (@$urls) {
  push @{$delay->remaining}, sub {
    my ($delay, $tx) = @_;
    if ($tx) {
      # do what you want
    }
    say $u;
    $ua->get($u => $delay->begin);
  };
}

# last callback
push @{$delay->remaining}, sub {
  my ($delay, $tx) = @_;
  if ($tx) {
    # do what you want
  }
  say 'Done';
};


Mojo::IOLoop->next_tick($delay->begin());
$delay->wait;