DeafNewsJp by Perl

README

# DeafNews_bot
# 
# NAME: DeafNews Twitter Bot
# 
# Directory:
# 
# Root
#  `-DeafNews
#     |-deaf_news_db.pl
#     |-deaf_news_bot.pl
#     `-DB.pm

DB.pm

#!/usr/bin/perl;
#DB.pm
package DB;
use DBIx::Skinny connect_info => +{
    dsn      => 'dbi:mysql:deafnews:localhost',
    username => 'root',
    password => 'password',
    connect_options => +{
        RaiseError => 1,
        PrintError => 0,
        AutoCommit => 1,
    },
};

1;

package DB::Schema;
use DBIx::Skinny::Schema;

install_table deaf_news => schema {
    pk 'id';
    columns qw/id title keyword url bitly_url publisher publisheddate flag created_at updated_at/;

};

1;

deaf_news_bot.pl

#!/usr/bin/perl;
# deaf_news_bot.pl

use strict;
use warnings;

use DateTime::Format::MySQL;
use Perl6::Say;

use lib '.';#cronで走らせる場合は、フルパスで書くこと。
use DB;
use Net::Twitter;

my $username = "username";
my $password = "password";

my $twt_clientname  = 'DeafNewsJp';
my $twt_clientver   = '0.1';
my $twt_clienturl   = 'http://www.makotoworld.com/';
my $table = "deaf_news";

my @id;

my $db = DB->new;
my $nt = Net::Twitter->new(
    traits      => ['API::REST'],
    username    => $username,
    password    => $password,
    clientname  => $twt_clientname,
    clienturl   => $twt_clienturl,
    useragent   => $twt_clientname,
    source      => $twt_clientname,
    clientver   => $twt_clientver,
);

my @rows = $db->search($table, {flag => '0'}, {order_by => [{publisheddate => 'desc'}], limit => 10});
foreach(@rows){
    push(@id, $_->id);
    my $dt = DateTime::Format::MySQL->parse_datetime($_->publisheddate);
    my $tweet = $_->title." k:".$_->keyword." ".$_->bitly_url." "."[".$_->publisher."]"." #DeafNewsJp";
    say $tweet;
    $nt->update($tweet);
}
$db->update($table,{flag => 1}, {id => \@id});

__END__

deaf_news_db.pl

#!/usr/bin/perl
# deaf_news_db.pl

use strict;
use warnings;

use WebService::Simple;
use WebService::Simple::Parser::JSON;
use Perl6::Say;
use Data::Dumper;
use Encode;
use DateTime;
use DateTime::Format::Mail;
use HTML::ExtractContent;
use LWP::UserAgent;

use lib '.';#cronで走らせる場合は、フルパスで書くこと。
use DB;

my @query = qw/聴覚障害 手話 聴覚障がい 聴力障害 ろうあ者/;
my $start = 0;
my $db = DB->new;

sub bit_ly{
    my ($URL) = @_;
    my $parser = WebService::Simple::Parser::JSON->new;
    $parser->{json}->utf8(1);
    my $bit_ly = WebService::Simple->new(
        base_url        => "http://api.bit.ly/shorten",
        response_parser => $parser,
        params          => { version => "2.0.1", login => "bitly_username", apiKey => "bitly_APIKEY", longUrl => $URL }
    );
    my $response = $bit_ly->get();
    return "http://j.mp/".$response->parse_response->{results}->{$URL}->{userHash};# userHash or shortUrl
}

sub translate{
    my ($query) = @_;
    my $parser = WebService::Simple::Parser::JSON->new;
    $parser->{json}->utf8(1);
    my $translate = WebService::Simple->new(
        base_url        => "http://ajax.googleapis.com/ajax/services/language/translate?v=1.0&langpair=|ja&q=".$query,
        response_parser => $parser,
    );
    my $response = $translate->get();
    return $response->parse_response->{responseData}->{detectedSourceLanguage};
}

sub google_news{
    my ($query) = @_;
    my $url;
    my $parser = WebService::Simple::Parser::JSON->new;
    $parser->{json}->utf8(1);
    my $google = WebService::Simple->new(
        base_url        => "http://ajax.googleapis.com/ajax/services/search/news",
        response_parser => $parser,
        params          => { v => "1.0", rsz => "large", hl => "ja", scoring => "d", ned => "jp" }
    );
    my $response = $google->get( { q => $query, start => $start } );
    my $count = $response->parse_response->{responseData}->{cursor}->{estimatedResultCount} % 8;
    for(my $i = 0; $i <= 8; $i++){
        $response = $google->get( { q => $query, start => $i * 8 } );
        foreach my $item (@{$response->parse_response->{responseData}->{results}}){
            my $dt = DateTime::Format::Mail->parse_datetime($item->{publishedDate});
            $dt = $dt->strftime('%Y-%m-%d %H:%M:%S');
            my $agent = LWP::UserAgent->new;
            my $res = $agent->get($url);
            my $extractor = HTML::ExtractContent->new;
            $extractor->extract($res->decoded_content);
            my $content = $extractor->as_text;
            if(translate($item->{titleNoFormatting}) eq "ja"){
                if($content !~ m/難聴取/ ){ # 複数の場合はm/難聴取|視覚障害/
                    if($db->search('deaf_news',{url => $item->{unescapedUrl})->count == 0){
                        my $row = $db->find_or_create('deaf_news',
                            {
                                title         => $item->{titleNoFormatting},
                                url           => $item->{unescapedUrl},
                                publisher     => $item->{publisher},
                                bitly_url     => bit_ly($item->{unescapedUrl}),
                                publisheddate => $dt
                            }
                        );
                    }
                }
            }
        }
    }
}

foreach(@query){
    utf8::decode($_);
    google_news($_);
}

__END__

deafnews_bot.sql

# DB MySQL
CREATE TABLE `deaf_news` (
  `id` int(10) NOT NULL AUTO_INCREMENT,
  `keyword` varchar(20) NOT NULL,
  `title` varchar(255) NOT NULL,
  `url` varchar(255) NOT NULL,
  `bitly_url` varchar(20) NOT NULL,
  `publisher` varchar(100) NOT NULL,
  `publisheddate` varchar(50) NOT NULL,
  `flag` int(1) NOT NULL DEFAULT '0',
  `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
  `updated_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
  PRIMARY KEY (`id`),
  UNIQUE KEY `url` (`url`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;