gistfile1.pl

#!/usr/bin/perl

use strict;
use warnings;
use Web::Scraper;
use URI;
use Data::Dumper;
use File::Spec;
use utf8;
use open IO  => ":utf8";
use Encode;

my $input_dir = $ARGV[0] or die "usage: perl scrapeRelatedQueries.pl input_dir";
opendir my $dir, $input_dir or die "Cannot open directory: $!";
my @files = readdir $dir;

@files = grep { $_ =~ /\.html$/ } @files;
@files = sort @files;

my $scraper = scraper {
  process 'div.brs_col p a', 'related_queries[]' => 'TEXT';
  process 'a.l', 'titles[]' => 'TEXT';
  process 'div.s span.st', 'snippets[]' => 'TEXT';
  process 'div.f cite', 'displayurls[]' => 'TEXT';
  process 'li.g a.l', 'urls[]' => '@href';
};

for my $file (@files) {
  my $path = File::Spec->catfile($input_dir, $file);
  my $html = \do { open my $fh, $path or die "$file: $!"; join '', <$fh> };
  my $result = $scraper->scrape($html);

  my $related_queries_ref = $result->{related_queries};
  my $titles_ref          = $result->{titles};
  my $snippets_ref        = $result->{snippets};
  my $displayurls_ref     = $result->{displayurls};
  my $urls_ref            = $result->{urls};

  my $i = 1;
  my $query_id = substr($file, 0, index($file, '.'));
  for my $related_query (@$related_queries_ref) {
    print 'SubQQgr-' . $query_id . '-' . $i . "\t" . $related_query . "\n";
    $i++;
  }

  my $results_size = scalar @$titles_ref;
  for my $i (0..$results_size - 1) {
    my $title      = $titles_ref->[$i] || '';
    my $snippet    = $snippets_ref->[$i] || '';
    my $displayurl = $displayurls_ref->[$i] || '';
    my $url        = $urls_ref->[$i] || '';

    print '-' x 30, "\n";
    print "Title: $title\n";
    print "Snippet: $snippet\n";
    print "Display URL: $displayurl\n";
    print "URL: $url\n";
  }
}