0

update_files.pl

#!/usr/bin/perl
use strict;
use warnings FATAL => qw( all );

use Net::Twitter::Lite::WithAPIv1_1;
use Lingua::EN::Inflect qw(NO NUMWORDS);
use HTML::Entities qw(encode_entities_numeric);
use Try::Tiny;

use lib '../files/lib';
use Base::Data qw(data_file data_directory get_hash);
use Base::Convert qw(textify hashtagify);
use Base::Nifty qw(line);
use Twitter qw(twitter_accounts nt_with_retry);
use Twitter::Data;

sub transform_user_data {
  my $string = shift;

  if ($string) {
    $string = encode_entities_numeric($string,'^\n\x20-\x25\x27-\x7e');
    $string =~ s/[\r\n]+//g;
    $string =~ s/\|/|/g;
  }
  else {
    $string = '';
  }

  return $string;
}

my @accounts = twitter_accounts();
die "First account is not Lady Aleena" if $accounts[0] ne 'Lady_Aleena';

my @accounts_totals;

my %is_account;
$is_account{$_}++ for @accounts;

my %statuses = (
  get_lists => 'owner',
  list_memberships => 'member',
  list_subscriptions => 'subscriber'
);

my $nt = Net::Twitter::Lite::WithAPIv1_1->new(
  consumer_key    => Twitter::consumer_key,
  consumer_secret => Twitter::consumer_key_secret,
  legacy_lists_api => 1,
);

my %LA_people = get_hash(
  file => data_file("Twitter/users/Lady_Aleena","people.txt"),
  headings => [qw(id screen_name name time_zone location last_status friends followers greet)],
);

for my $account (@accounts) {
  line(0,$account);

  $nt->{access_token}        = Twitter::access_key($account);
  $nt->{access_token_secret} = Twitter::access_key_secret($account);

  my $short_account = $account;
     $short_account =~ s/LadyAleena_//g;

  my $directory = "Twitter/users/$account";
  my $full_path = data_directory($directory);
  mkdir($full_path) or die unless (-d $full_path);

  my %files;
  for (qw(people lists mentions retweets)) {
    $files{$_} = data_file($directory,"$_.txt");
  }

  # Start account totals

  my $account_totals = nt_with_retry($nt,'account_totals');
  push @accounts_totals, "$account|".join("|",map($account_totals->{$_},qw(followers friends updates)));
  for my $heading (qw(updates friends followers)) {
    my $value = $account_totals->{$heading};
    line(1,"$account has $value $heading");
  }

  # End account totals
  # Start followers and friends

  my %people;
  if (-s $files{people}) {
    %people = get_hash(
      file => $files{people},
      headings => [qw(id screen_name name time_zone location last_status friends followers greet)],
    );
  }

  for my $people_type (qw(friends followers)) {
    my $method_ids = "$people_type\_ids";
    my %ids;
    for ( my $cursor = -1; $cursor; ) {
      my $people_list = nt_with_retry($nt,$method_ids => $account);
      for (@{$people_list->{ids}}) {
        $people{$_}{greet} = 0 if !exists($people{$_});
        $people{$_}{id} = $_;
        $people{$_}{$people_type} = 1;
      }
      for (@{$people_list->{ids}}) {
        $ids{$_}++;
      }
      $cursor = $people_list->{next_cursor};
    }

    for my $person (keys %people) {
      if (!exists($ids{$person}) && $people{$person}{$people_type} && $people{$person}{$people_type} == 1) {
        $people{$person}{$people_type} = -1;
        line(2,"$people{$person}{screen_name} is now part of de$people_type.");
      }
    }
  }

  my @people_ids = keys %people;
  my @lookups;
  while (@people_ids) {
    push @lookups, [splice(@people_ids,0,100)];
  }

  for (@lookups) {
    my $user_list = nt_with_retry($nt,lookup_users => { user_id => [@{$_}], include_entities => 'true' });
    for my $user (@{$user_list}) {
      my $id = $user->{id};
      
      $people{$id}{id} = $id;
      $people{$id}{screen_name} = $user->{screen_name};
      $people{$id}{name} = transform_user_data($user->{name});
      $people{$id}{time_zone} = $user->{time_zone} ? $user->{time_zone} : '';
      $people{$id}{location} = transform_user_data($user->{location});
      $people{$id}{last_status} = $user->{status}{created_at};
    }
  }

  my %groups;
  my @greetings;
  for my $person (keys %people) {
    for (qw(friends followers)) {
      $people{$person}{$_} = 0 if !defined($people{$person}{$_});
    }

    if ($people{$person}{followers} == 1 && $people{$person}{friends} != 1 && $people{$person}{greet} == 0) {
      push @greetings, $people{$person}{screen_name};
    }

    $groups{companions}++  if $people{$person}{friends} == 1   && $people{$person}{followers} == 1;
    $groups{unfriends}++   if $people{$person}{friends} == 0   && $people{$person}{followers} != -1;
    $groups{unfollowers}++ if $people{$person}{followers} == 0 && $people{$person}{friends} != -1;
    $groups{defriends}++   if $people{$person}{friends} == -1;
    $groups{defollowers}++ if $people{$person}{followers} == -1;
  }

  for my $group (qw(companions unfriends unfollowers defriends defollowers)) {
    if ($groups{$group}) {
      my $singular = $group;
         $singular =~ s/s$//;
      my $count = NO($singular,$groups{$group});
      line(1,"$account has $count.");
    }
  }

  if ($account ne 'Lady_Aleena' && @greetings > 0) {
    line(1,"$account\'s follower greetings");
    for my $greetee (@greetings) {
      if ($greetee) {
        my $greeting = "\@$greetee Thank you for following \@Lady_Aleena\'s mostly automated $short_account account. I hope you don\'t get too bored here.";
        line(2,$greeting);
        nt_with_retry($nt,update => { status => $greeting });

        for (keys %people) {
          if ($people{$_}{screen_name} eq $greetee) {
            $people{$_}{greet} = 1;
          }
        }
      }
    }
  }

  my @lines;
  for my $person (sort {$a <=> $b} keys %people) {
    my $line = join '|', map $people{$person}{$_} // 0, qw(id screen_name name time_zone location last_status friends followers greet);
    push @lines, $line;
  }
  my $info = join("\n",@lines);

  open(my $to_file,'>',data_file($directory,'people.txt')) or die "Can't open file $!";
  print $to_file $info;

  # End followers and friends
  # Start list updates

  # link to lists on tumblr http://tmblr.co/ZZVoswE7_RE1

  line(1,"$account lists");
  my %lists = Twitter::Data::lists_data($account);

  my $lists_directory = "Twitter/users/$account/lists";
  my $lists_full_path = data_directory($lists_directory);
  mkdir($lists_full_path) or die "Can't make $lists_full_path $!" unless (-d $lists_full_path);

  my @new_people;
  for my $list_type (qw(get_lists list_memberships list_subscriptions)) {
    my $list_heading = ucfirst textify($statuses{$list_type});
    my $status = $statuses{$list_type};
    line(2,$list_heading);
    for ( my $cursor = -1; $cursor; ) {
      my $lists_data = nt_with_retry($nt,$list_type => {
        user => $account,
        cursor => $cursor,
      });
      
      for my $list (sort {$a->{name} cmp $b->{name}} @{$lists_data->{lists}}) {
        my $id = $list->{id};
        $lists{$id}{id} = $id;
        my $list_name = $list->{name};
        push @{$lists{$id}{status}}, $status if !grep /$status/, @{$lists{$id}{status}};
        
        line(3,$list_name);

        next if $lists{$id}{done};
        $lists{$id}{done} = 1;
        
        my $list_slug = $lists{$id}{slug} = $list->{slug};
        $lists{$id}{name} = $list->{name};
        my $user_screen_name = $lists{$id}{user} = $list->{user}{screen_name};
        
        my %changes = ();

        my $member_count = $list->{member_count};
        $changes{list_members} = $lists{$id}{members_change} = $member_count - ( $lists{$id}{members_count} // 0 );
        my $subscriber_count = $list->{subscriber_count};
        $changes{list_subscribers} = $lists{$id}{subscribers_change} = $subscriber_count - ( $lists{$id}{subscribers_count} // 0 );
        
        $lists{$id}{members_count} = $member_count;
        $lists{$id}{subscribers_count} = $subscriber_count;
        
        next if $list_type ne "get_lists";

        my $list_full_path = data_directory("Twitter/users/$account/lists/$list_slug");
        mkdir($list_full_path) or die "Can't make $list_full_path $!" unless (-d $list_full_path);

        for my $list_group (qw(list_members list_subscribers)) {
          next if $changes{$list_group} == 0;

          my @list_group_people;
          for ( my $cursor = -1; $cursor; ) {
            my $list_people = nt_with_retry($nt,$list_group => {
              owner_screen_name => $user_screen_name,
              slug              => $list_slug,
              cursor            => $cursor,
            });
            push @list_group_people, map $_->{screen_name}, @{$list_people->{users}};
            $cursor = $list_people->{next_cursor};
          }
          
          my $list_group_file_name = $list_group;
             $list_group_file_name =~ s/^list_//;

          open(my $list_group_fh, '>', "$list_full_path/$list_group_file_name.txt") or die "Can't open $list_full_path/$list_group_file_name.txt $!";
          print $list_group_fh join("\n",@list_group_people);
          close($list_group_fh);

          next if ($account eq "Lady_Aleena" || $list_group ne "list_subscribers");

          for my $person (@list_group_people) {
            my $friends = 0;
            for (values %people) {
              $friends = $_->{friends} if $_->{screen_name} eq $person;
            }

            next if $friends != 0;
            next if grep { $_->{screen_name} eq $person } values %people;
            
            try {
              my $new_user = nt_with_retry($nt,create_friend => {
                screen_name => $person,
                follow => 'true',
              });

              nt_with_retry($nt,update_friendship => {
                screen_name => $person,
                retweets => 'false',
              });
              
              my $id = $new_user->{id};

              $people{$id}{id} = $id;
              $people{$id}{screen_name} = $new_user->{screen_name};
              $people{$id}{name} = transform_user_data($new_user->{name});
              $people{$id}{time_zone} = $new_user->{time_zone};
              $people{$id}{location} = transform_user_data($new_user->{location});
              $people{$id}{last_status} = $new_user->{status}{created_at};
              $people{$id}{friends} = 1;

              line(4,"$person subscribed to $list_name so $short_account followed.");

              my $line = join '|', map $people{$id}{$_} // 0, qw(id screen_name name time_zone location last_status friends followers greet);
              push @new_people, $line;
            }
            catch {
              line(4,"Could not follow $person.");
            }
          }
        }
      }
      $cursor = $lists_data->{next_cursor};
    }
  }

  if (scalar(@new_people) > 0) {
    open(my $np_fh, '>>', $files{people}) or die "Can't open $files{people} $!";
    print $np_fh join("\n",@new_people);
    close($np_fh);
  }

  my @list_lines;
  for my $list (values %lists) {
    my $line = '';
    for my $value (qw(id slug name user members_count members_change subscribers_count subscribers_change)) {
      $line .= length $line ? "|".($list->{$value} // 0) : $list->{$value} // 0;
    }
    my $status = join(',',@{$list->{status}});
    $line .= "|$status";
    push @list_lines, $line;
  }

  if (scalar(@list_lines) > 0) {
    open(my $al_fh, '>', $files{lists}) or die "Can't open $files{lists} $!";
    print $al_fh join("\n",@list_lines);
    close($al_fh);
  }

  # End list updates
  # Start messages

  my %last_messages;
  for my $message (qw(mentions retweets)) {
    my ($last_id,$last_data);
    if (-s $files{$message}) {
      open (my $message_fh, '<', $files{$message}) or die "Can't open $files{$message}. $!";
      my @message_array = <$message_fh>;
      chomp(@message_array);
      my $last_line = pop @message_array;
      ($last_id,$last_data) = split(/\|/,$last_line,2);
    }
    $last_messages{$message} = $last_id ? $last_id : 1;
  }

  # End lists
  # Start mentions

  line(1,"$account mentions & replies");

  my $mentions = nt_with_retry($nt,mentions => {
    count => 200,
    include_entities => 1,
    since_id => $last_messages{mentions} || 1,
  });

  my @new_mentions;
  for my $mention (@{$mentions}) {
    my $tweet_id = $mention->{id};
    my $tweet_dt = $mention->{created_at};
    my $tweet_text = encode_entities_numeric($mention->{text},'^\n\x20-\x25\x27-\x7e');
       $tweet_text =~ s/[\r\n]+//g;
    my $tweet_reply_to_screen_name = $mention->{in_reply_to_screen_name} || '';
    my $tweet_reply_to_id = $mention->{in_reply_to_status_id} || '';
    my $user_id = $mention->{user}{id};
    my $user_name = $mention->{user}{screen_name};

    for my $url (@{$mention->{entities}{urls}}) {
      my $short_url = quotemeta($url->{url});
      my $long_url = $url->{expanded_url} ? $url->{expanded_url} : "http://$short_url";
      $tweet_text =~ s/$short_url/$long_url/;
    }

    if (!exists($is_account{$user_name})) {
      push @new_mentions, "$tweet_id|$user_name|$tweet_dt|$tweet_text|$tweet_reply_to_screen_name|$tweet_reply_to_id";
    }

    try {
      if ($tweet_text =~ /^#FF/i || $tweet_text =~ /^#FollowFriday/i) {
        $nt->update({ 
          status => "\@$user_name Thank you for the \#FF.",
          in_reply_to_status_id => $tweet_id,
        });
      }
    }
    catch {
      print STDERR $_;
    }
  }

  if (scalar(@new_mentions) > 0) {
    local $\ = undef;
    open (my $message_fh, '>>', $files{mentions}) or die "Can't open $files{mentions}. $!";
    print join("\n",reverse @new_mentions);
    print $message_fh "\n".join("\n",reverse @new_mentions);
    close($message_fh);
    print "\n";
  }

  # End mentions
  # Start retweets

  line(1,"$account reweets");

  my %stored_retweets;
  if (-f $files{retweets}) {
    %stored_retweets = get_hash(
      file => $files{retweets}, 
      headings => [qw(id tweet retweeters)],
    );

    my @retweets_ids = (sort {$a <=> $b} keys %stored_retweets);
    my @last_five_retweets = scalar(@retweets_ids) > 5 ? @retweets_ids[-5..-1] : @retweets_ids;
    
    for my $retweet_id (@last_five_retweets) {
      my $retweet_by = nt_with_retry($nt, retweets => $retweet_id);
      my @retweeters_list = split(/,/,$stored_retweets{$retweet_id}{retweeters});
      for my $retweeter (@{$retweet_by}) {
        my $screen_name = $retweeter->{user}{screen_name};
        push @retweeters_list, $screen_name if !grep($screen_name,@retweeters_list);
      }
      $stored_retweets{$retweet_id}{retweeters} = scalar(@retweeters_list) > 0 ? join(',',@retweeters_list) : '';
    }
  }

  my $retweets = nt_with_retry($nt,retweets_of_me => {
    count => 100,
    include_entities => 1,
    since_id => $last_messages{retweets} || 1,
  });

  for my $retweet (@{$retweets}) {
    my $id = $retweet->{id};
    my $retweet_by = nt_with_retry($nt, retweets => $id);

    if (!exists($stored_retweets{$id})) {
      my $retweet_text = encode_entities_numeric($retweet->{text},'^\n\x20-\x25\x27-\x7e');
      for my $url (@{$retweet->{entities}{urls}}) {
        my $short_url = quotemeta($url->{url});
        my $long_url = $url->{expanded_url} ? $url->{expanded_url} : "http://$short_url";
        $retweet_text =~ s/$short_url/$long_url/;
      }
      $stored_retweets{$id}{tweet} = $retweet_text;

      my @retweeters_list;
      for my $retweeter (@{$retweet_by}) {
        my $screen_name = $retweeter->{user}{screen_name};
        push @retweeters_list, $screen_name;
      }
      $stored_retweets{$id}{retweeters} = scalar(@retweeters_list) > 0 ? join(',',@retweeters_list) : '';
    }
    
    $stored_retweets{$id}{id} = $id;
  }

  my @updated_retweets;
  for my $tweet (sort {$stored_retweets{$a}{id} <=> $stored_retweets{$b}{id}} keys %stored_retweets) {
    my $id = $stored_retweets{$tweet}{id};
    my $retweet_text = $stored_retweets{$tweet}{tweet};
    my $retweeters = $stored_retweets{$tweet}{retweeters};

    push @updated_retweets, "$id|$retweet_text|$retweeters";
  }

  if (scalar(@updated_retweets) > 0) {
    local $\ = undef;
    open (my $message_fh, '>', $files{retweets}) or die "Can't open $files{retweets}. $!";
    print join("\n",@updated_retweets);
    print $message_fh join("\n",@updated_retweets);
    close($message_fh);
    print "\n";
  }

  # End retweets
  # End messages
}

open(my $gen_acct_file,'>',data_file('Twitter','account_totals.txt')) or die "Can't open account_totals.txt. $!";
print $gen_acct_file join("\n",@accounts_totals);
close($gen_acct_file);