Skip to content

Instantly share code, notes, and snippets.

@wchristian
Created November 3, 2010 14:57
Show Gist options
  • Save wchristian/661175 to your computer and use it in GitHub Desktop.
Save wchristian/661175 to your computer and use it in GitHub Desktop.
#!/usr/bin/perl
use 5.10.1;
use strict;
use warnings;
use HTTP::Request::Common 'POST';
use AnyEvent::HTTP;
use Benchmark;
my $t0 = Benchmark->new;
my @ids = ( 34..100 );
my @reqs = map POST( "http://eve-metrics.com/api/item.json", [ type_ids => $_ ] ), @ids;
my @responses = parallel_download( @reqs );
# wait till all finished
print "Finished\n";
for ( @responses ) {
print $_->[1]{Status};
}
my $t1 = Benchmark->new;
my $td = timediff($t1, $t0);
print "the code took:",timestr($td),"\n";
sub parallel_download {
my ( @requests ) = @_;
my $workers = 10;
my $real_worker_max = sanitize_worker_max( \@requests, $workers );
my @responses;
my $cv = AnyEvent->condvar;
for ( 1 .. 10 ) {
$cv->begin;
my $req = pop @requests;
do_request( \@requests, $req, $cv, \@responses );
}
$cv->recv;
return @responses;
}
sub do_request {
my ( $queue, $req, $cv, $responses ) = @_;
http_request(
$req->method => $req->uri->as_string,
body => $req->content,
sub {
push @{$responses}, [ @_ ];
return $cv->end if !@{$queue};
my $next_req = pop @{$queue};
do_request( $queue, $next_req, $cv, $responses );
}
);
return;
}
sub sanitize_worker_max {
my ( $queue, $asked_max ) = @_;
die "max should be 0 or more" if $asked_max < 0;
my $queue_size = @{ $queue };
return $queue_size if !$asked_max; # 0 = as many parallel as possible
return $queue_size if $asked_max > $queue_size; # not more than the request count
return $asked_max;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment