robhammond

package WebCrawl::Crawl;
use Mojo::Base 'Mojolicious::Controller';

sub crawl {
  my $self = shift;

	# Increase inactivity timeout for connection a bit
	Mojo::IOLoop->stream($self->tx->connection)->timeout(15);

	# Change content type

robhammond

#!/usr/bin/env perl
use strict;
use Modern::Perl;
use Domain::PublicSuffix;
use URI;

my $keyword = 'broadband';

my @urls = qw(http://www.moneysupermarket.com/broadband/
http://www.o2.co.uk/broadband

robhammond

#!/usr/bin/env perl
use Modern::Perl;
use Mojo::UserAgent;
use Mojo::IOLoop;
use Mojo::URL;

# FIFO queue
my $seed_url = 'http://www.google.co.uk/';
my @urls = ($seed_url);

robhammond

#!/usr/bin/env perl
###
# This snippet will recursively scan directories for use/require statements
# in perl scripts or modules, making it easy to build a list of dependencies
# from code you've inherited or neglected to document.
###
use Perl::PrereqScanner;
use File::Spec::Functions qw( catfile );
use File::Find qw(finddepth);
use Data::Dumper;