keitaiall.pl

#!perl
use strict;
use warnings;
use utf8;
use URI;
use Web::Scraper;
use Text::CSV_XS;

my $detail = scraper {
    process '//dl', 'data[]', sub {
        my $elem = shift;
        my $key = $elem->find('dd')->as_trimmed_text;
        (my $value = $elem->find('dt')->as_trimmed_text) =~ s/^\s|\s$//g;
        ( $key => $value );
    };
    process '//p[text()="USER AGENT"]/following-sibling::p[1]', 'agent', sub {
        my $elem = shift;
        return [ grep {! ref $_} @{$elem->content_array_ref} ];
    };
    process 'id("dataGroup1")//img', 'image', '@src';
    process '//b[@class="modelNm1"]', 'model', 'TEXT';
};

my $urls = scraper {
    process '//table[@class="listTable"]//a' => 'urls[]' => '@href';
    result 'urls';
}->scrape(URI->new('http://keitaiall.jp/table.html'));

my (@headers, @lines);

open my $fh, '>:encoding(UTF-8)', 'attributes.csv';
open my $fh_agent, '>:encoding(UTF-8)', 'agent.csv';

my $csv = Text::CSV_XS->new({binary => 1});

for my $url (@$urls) {
    my $result = $detail->scrape($url);
    my $data = { @{$result->{data}}, 'image' => "$result->{image}" };
    my @line = ($result->{model});
    for my $key (@headers) {
        push @line, delete $data->{$key};
    }
    for my $key (sort keys %$data) {
        push @headers, $key;
        push @line, delete $data->{$key};
    }
    push @lines, \@line;
    for (@{$result->{agent}}) {
        $csv->print($fh_agent, [$_, $result->{model}]);
        print $fh_agent "\n";
    }
}
$csv->print($fh, ['model', @headers]);
print $fh "\n";
for my $line (@lines) {
    $csv->print($fh, $line);
    print $fh "\n";
}