Last active
July 22, 2017 22:29
-
-
Save dlangille/6a7c81ff17cf1f446581660d96a33a14 to your computer and use it in GitHub Desktop.
Would someone convert this to use File::Find for me please? I wrote this a loooong time ago. Thank you.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/perl -w | |
# | |
# $Id: queue-status.pl,v 1.3 2012/10/17 18:10:22 dan Exp $ | |
# | |
# Copyright (c) 2001-2006 DVL Software | |
##!/usr/bin/perl -w | |
# | |
# $Id: queue-status.pl,v 1.3 2012/10/17 18:10:22 dan Exp $ | |
# | |
# Copyright (c) 2001-2006 DVL Software | |
# | |
use strict; | |
use config; | |
use database; | |
use utilities; | |
use status; | |
use POSIX qw( strftime ); | |
use File::Find; | |
sub SendNotice($) { | |
my $Msg = shift; | |
my $hostname = `hostname`; | |
chomp $hostname; | |
my $Body = 'At ' . $hostname . ' ' . $Msg . "\n"; | |
print $Body; | |
} | |
print strftime("%Y-%m-%d %H:%M:%S\n", localtime); | |
my $base=$FreshPorts::Config::QueueBaseDir; | |
my %queues = ('incoming' => '*.txt', 'retry' => '*.txt', 'recent' => '*.xml'); | |
my %queue_names = ('incoming' => 'incoming', 'retry' => 'retry', 'recent' => 'processed'); | |
my %report_non_zero = ('retry' => 1, 'incoming' => 1); | |
my $Interval = '10 minutes'; | |
my $send_report = 0; | |
my $msg = ''; | |
my $CountRecent; | |
undef($CountRecent); | |
print strftime("%Y-%m-%d %H:%M:%S\n", localtime); | |
my $dbh = FreshPorts::Database::GetDBHandle(); | |
print strftime("%Y-%m-%d %H:%M:%S\n", localtime); | |
foreach my $site (@FreshPorts::Status::sites) { | |
print strftime("%Y-%m-%d %H:%M:%S\n", localtime); | |
$msg .= "SITE: $site "; | |
foreach my $queue (keys %queues) { | |
my $pattern = $queues{$queue} || '*'; | |
my $glob = "$base/$site/msgs/FreeBSD/$queue/$pattern"; | |
my @files = glob( $glob ); | |
print strftime("%Y-%m-%d %H:%M:%S\n", localtime); | |
print "working on '$glob'\n"; | |
my $Count = scalar @files; | |
$msg .= " $queue: $Count "; | |
if ($Count && defined($report_non_zero{$queue})) { | |
if (!defined($CountRecent)) { | |
$CountRecent = FreshPorts::Utilities::CommitCountPeriod($dbh, $Interval); | |
} | |
if ($Count > $CountRecent) { | |
$send_report = 1; | |
} | |
} | |
print "done foreach my queue\n"; | |
} | |
$msg .= " "; | |
print "done foreach my site\n"; | |
} | |
print strftime("%Y-%m-%d %H:%M:%S\n", localtime); | |
print "done\n"; | |
if ($send_report) { | |
# Sys::Syslog::syslog('notice', 'There is a problem with the FreshPorts queues'); | |
SendNotice($msg); | |
$dbh->disconnect(); | |
exit(1) | |
} else { | |
print 'Queues are OK'; | |
$dbh->disconnect(); | |
} | |
print strftime("%Y-%m-%d %H:%M:%S\n", localtime); | |
use strict; | |
use config; | |
use database; | |
use utilities; | |
use status; | |
sub SendNotice($) { | |
my $Msg = shift; | |
my $hostname = `hostname`; | |
chomp $hostname; | |
my $Body = 'At ' . $hostname . ' ' . $Msg . "\n"; | |
print $Body; | |
} | |
my $base=$FreshPorts::Config::QueueBaseDir; | |
my %queues = ('incoming' => '*.txt', 'retry' => '*.txt', 'recent' => '*.xml'); | |
my %queue_names = ('incoming' => 'incoming', 'retry' => 'retry', 'recent' => 'processed'); | |
my %report_non_zero = ('retry' => 1, 'incoming' => 1); | |
my $Interval = '10 minutes'; | |
my $send_report = 0; | |
my $msg = ''; | |
my $CountRecent; | |
undef($CountRecent); | |
my $dbh = FreshPorts::Database::GetDBHandle(); | |
foreach my $site (@FreshPorts::Status::sites) { | |
$msg .= "SITE: $site "; | |
while (my ($queue, $pattern) = each %queues) { | |
my $Command = "find $base/$site/msgs/FreeBSD/$queue/"; | |
# print $Command . "\n"; | |
if ($pattern ne '') { | |
$Command .= " -name \"$pattern\""; | |
} | |
$Command .= ' -maxdepth 1 | wc -l'; | |
# print $Command . "\n"; | |
my $Count = `$Command`; | |
chomp $Count; | |
$Count = FreshPorts::Utilities::trim($Count); | |
$msg .= " $queue: $Count "; | |
if ($Count && defined($report_non_zero{$queue})) { | |
if (!defined($CountRecent)) { | |
$CountRecent = FreshPorts::Utilities::CommitCountPeriod($dbh, $Interval); | |
} | |
if ($Count > $CountRecent) { | |
$send_report = 1; | |
} | |
} | |
} | |
$msg .= " "; | |
} | |
if ($send_report) { | |
# Sys::Syslog::syslog('notice', 'There is a problem with the FreshPorts queues'); | |
SendNotice($msg); | |
$dbh->disconnect(); | |
exit(1) | |
} else { | |
print 'Queues are OK'; | |
$dbh->disconnect(); | |
} |
Another run, with $msg output:
[dan@jester:~/scripts] $ perl new-queue-status-details.pl
2017-07-22 22:07:41
2017-07-22 22:07:41
2017-07-22 22:07:41
2017-07-22 22:07:41
2017-07-22 22:07:41
working on '/usr/local/FreshPorts/freshports.org/msgs/FreeBSD/incoming/*.txt'
msg is 'SITE: freshports.org incoming: 63 '
done foreach my queue
2017-07-22 22:09:32
working on '/usr/local/FreshPorts/freshports.org/msgs/FreeBSD/recent/*.xml'
msg is 'SITE: freshports.org incoming: 63 recent: 190 '
done foreach my queue
2017-07-22 22:09:32
working on '/usr/local/FreshPorts/freshports.org/msgs/FreeBSD/retry/*.txt'
msg is 'SITE: freshports.org incoming: 63 recent: 190 retry: 0 '
done foreach my queue
done foreach my site
2017-07-22 22:09:32
done
At jester.int.unixathome.org SITE: freshports.org incoming: 63 recent: 190 retry: 0
Commenting out line 153:
$ perl new-queue-status-details.pl
2017-07-22 22:14:18
2017-07-22 22:14:18
2017-07-22 22:14:19
2017-07-22 22:14:19
2017-07-22 22:14:19
working on '/usr/local/FreshPorts/freshports.org/msgs/FreeBSD/incoming/*.txt'
msg is 'SITE: freshports.org incoming: 63 '
done foreach my queue
2017-07-22 22:14:19
working on '/usr/local/FreshPorts/freshports.org/msgs/FreeBSD/retry/*.txt'
msg is 'SITE: freshports.org incoming: 63 retry: 0 '
done foreach my queue
2017-07-22 22:14:19
working on '/usr/local/FreshPorts/freshports.org/msgs/FreeBSD/recent/*.xml'
msg is 'SITE: freshports.org incoming: 63 retry: 0 recent: 190 '
done foreach my queue
done foreach my site
2017-07-22 22:14:19
done
At jester.int.unixathome.org SITE: freshports.org incoming: 63 retry: 0 recent: 190
Here's the problem. It's reading the entire table.
$ perl new-queue-status-details.pl
2017-07-22 22:18:03
2017-07-22 22:18:03
2017-07-22 22:18:04
2017-07-22 22:18:04
2017-07-22 22:18:04
working on '/usr/local/FreshPorts/freshports.org/msgs/FreeBSD/recent/*.xml'
msg is 'SITE: freshports.org recent: 190 '
done foreach my queue
2017-07-22 22:18:04
working on '/usr/local/FreshPorts/freshports.org/msgs/FreeBSD/incoming/*.txt'
SELECT count(*) AS count FROM commit_log WHERE date_added > NOW() - INTERVAL '10 minutes'
msg is 'SITE: freshports.org recent: 190 incoming: 63 '
done foreach my queue
2017-07-22 22:19:24
working on '/usr/local/FreshPorts/freshports.org/msgs/FreeBSD/retry/*.txt'
msg is 'SITE: freshports.org recent: 190 incoming: 63 retry: 0 '
done foreach my queue
done foreach my site
2017-07-22 22:19:24
done
At jester.int.unixathome.org SITE: freshports.org recent: 190 incoming: 63 retry: 0
freshports.org=# explain analyse SELECT count(*) AS count FROM commit_log WHERE date_added > NOW() - INTERVAL '10 minutes';
QUERY PLAN
-----------------------------------------------------------------------------------------------------------------------
Aggregate (cost=39792.56..39792.57 rows=1 width=8) (actual time=59024.984..59024.984 rows=1 loops=1)
-> Seq Scan on commit_log (cost=0.00..39792.39 rows=67 width=0) (actual time=59024.978..59024.978 rows=0 loops=1)
Filter: (date_added > (now() - '00:10:00'::interval))
Rows Removed by Filter: 670002
Planning time: 0.806 ms
Execution time: 59025.098 ms
(6 rows)
freshports.org=#
There is no index on date_added.
Fixed.
freshports.org=# explain analyse SELECT count(*) AS count FROM commit_log WHERE date_added > '2017-07-22 22:14:13.385744+00';
^CCancel request sent
ERROR: canceling statement due to user request
freshports.org=# create index commit_log_date_added on commit_log(date_added);
CREATE INDEX
freshports.org=# explain analyse SELECT count(*) AS count FROM commit_log WHERE date_added > '2017-07-22 22:14:13.385744+00';
QUERY PLAN
---------------------------------------------------------------------------------------------------------------------------------------------
Aggregate (cost=6.71..6.72 rows=1 width=8) (actual time=0.011..0.012 rows=1 loops=1)
-> Index Only Scan using commit_log_date_added on commit_log (cost=0.42..6.70 rows=1 width=0) (actual time=0.006..0.006 rows=0 loops=1)
Index Cond: (date_added > '2017-07-22 22:14:13.385744+00'::timestamp with time zone)
Heap Fetches: 0
Planning time: 0.291 ms
Execution time: 0.048 ms
(6 rows)
freshports.org=#
Now we get:
[dan@jester:~/scripts] $ perl new-queue-status-details.pl
2017-07-22 22:29:23
2017-07-22 22:29:23
2017-07-22 22:29:24
2017-07-22 22:29:24
2017-07-22 22:29:24
working on '/usr/local/FreshPorts/freshports.org/msgs/FreeBSD/retry/*.txt'
msg is 'SITE: freshports.org retry: 0 '
done foreach my queue
2017-07-22 22:29:24
working on '/usr/local/FreshPorts/freshports.org/msgs/FreeBSD/incoming/*.txt'
SELECT count(*) AS count FROM commit_log WHERE date_added > NOW() - INTERVAL '10 minutes'
msg is 'SITE: freshports.org retry: 0 incoming: 65 '
done foreach my queue
2017-07-22 22:29:24
working on '/usr/local/FreshPorts/freshports.org/msgs/FreeBSD/recent/*.xml'
msg is 'SITE: freshports.org retry: 0 incoming: 65 recent: 190 '
done foreach my queue
done foreach my site
2017-07-22 22:29:24
done
At jester.int.unixathome.org SITE: freshports.org retry: 0 incoming: 65 recent: 190
[dan@jester:~/scripts] $
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
A full run, with Fresh's changes: