Skip to content

Instantly share code, notes, and snippets.

@silvioq
Created December 14, 2010 21:49
Show Gist options
  • Save silvioq/741169 to your computer and use it in GitHub Desktop.
Save silvioq/741169 to your computer and use it in GitHub Desktop.
#!/usr/bin/env perl
use strict;
use LWP;
use Mojo::DOM;
my $start = @ARGV[0];
my $end = @ARGV[1] || 100;
while( $start <= $end ){
my $req = HTTP::Request->new(GET => 'http://www.google.com/search?q="Web Hosting by Yahoo!"&start=' . $start);
my $ua = LWP::UserAgent->new( agent => "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.11) Gecko/2009060215 Firefox/3.0.11 (.NET CLR 3.5.30729)" );
my $res = $ua->request( $req );
my $dom = Mojo::DOM->new;
$dom->parse( $res->decoded_content );
$dom->find( "h3 a" )->each( sub(){
print "\"" . $_->text . "\" => \"" . $_->attrs->{href} . "\"\n" if $_->attrs->{class} ;
} );
sleep_ms( 400 + ( ( rand() - 0.5 ) * 100.0 ) );
$start += 10;
}
print "\n";
sub sleep_ms($){
my( $sleep ) = @_;
# print "Waiting $sleep\n";
select(undef, undef, undef, $sleep / 1000.0 );
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment