fix DXCron, lockout and close_gracefully

[spider.git] / perl / DXCron.pm

diff --git a/perl/DXCron.pm b/perl/DXCron.pm
index 0cc6ce88d85ec36e7fa8617627fb724f41d9bb68..2a02bfd36f75bb340896dbf103637ca27a6c10d1 100644 (file)
--- a/perl/DXCron.pm
+++ b/perl/DXCron.pm
@@ -14,8 +14,8 @@ use DXM;
 use DXDebug;
 use IO::File;
 use DXLog;
-
-use Mojo::IOLoop::Subprocess;
+use Time::HiRes qw(gettimeofday tv_interval);
+use DXSubprocess;
 
 use strict;
 
@@ -62,28 +62,36 @@ sub cread
        my $line = 0;
        my @out;
 
-       dbg("cron: reading $fn\n") if isdbg('cron');
+       dbg("DXCron::cread reading $fn\n") if isdbg('cron');
        open($fh, $fn) or confess("cron: can't open $fn $!");
-       while (<$fh>) {
+       while (my $l = <$fh>) {
                $line++;
-               chomp;
-               next if /^\s*#/o or /^\s*$/o;
-               my ($min, $hour, $mday, $month, $wday, $cmd) = /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(.+)$/o;
+               chomp $l;
+               next if $l =~ /^\s*#/o or $l =~ /^\s*$/o;
+               my ($min, $hour, $mday, $month, $wday, $cmd) = $l =~ /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(.+)$/o;
                next unless defined $min;
                my $ref = bless {};
-               my $err;
-               
-               $err |= parse($ref, 'min', $min, 0, 60);
-               $err |= parse($ref, 'hour', $hour, 0, 23);
-               $err |= parse($ref, 'mday', $mday, 1, 31);
-               $err |= parse($ref, 'month', $month, 1, 12, "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec");
-               $err |= parse($ref, 'wday', $wday, 0, 6, "sun", "mon", "tue", "wed", "thu", "fri", "sat");
-               if (!$err) {
-                       $ref->{cmd} = $cmd;
-                       push @out, $ref;
-                       dbg("cron: adding $_\n") if isdbg('cron');
+               my $err = '';
+
+               if (defined $min && defined $hour && defined $cmd) { # it isn't all of them, but should be enough to tell if this is a real line
+                       $err .= parse($ref, 'min', $min, 0, 60);
+                       $err .= parse($ref, 'hour', $hour, 0, 23);
+                       $err .= parse($ref, 'mday', $mday, 1, 31);
+                       $err .= parse($ref, 'month', $month, 1, 12, "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec");
+                       $err .= parse($ref, 'wday', $wday, 0, 6, "sun", "mon", "tue", "wed", "thu", "fri", "sat");
+                       if (!$err) {
+                               $ref->{cmd} = $cmd;
+                               push @out, $ref;
+                               dbg("DXCron::cread: adding $l\n") if isdbg('cron');
+                       } else {
+                               $err =~ s/^, //;
+                               LogDbg('cron', "DXCron::cread: error $err on line $line '$l'");
+                       }
                } else {
-                       dbg("cron: error on line $line '$_'\n") if isdbg('cron');
+                       LogDbg('cron', "DXCron::cread error on line $line '$l'");
+                       my @s = ($min, $hour, $mday, $month, $wday, $cmd);
+                       my $s = "line $line splits as " . join(', ', (map {defined $_ ? qq{$_} : q{'undef'}} @s));
+                       LogDbg('cron', $s);
                }
        }
        close($fh);
@@ -102,7 +110,7 @@ sub parse
        # handle '*' values
        if ($val eq '*') {
                $ref->{$sort} = 0;
-               return 0;
+               return;
        }
 
        # handle comma delimited values
@@ -110,20 +118,20 @@ sub parse
        for (@comma) {
                my @minus = split /-/o;
                if (@minus == 2) {
-                       return 1 if $minus[0] < $low || $minus[0] > $high;
-                       return 1 if $minus[1] < $low || $minus[1] > $high;
+                       return  ", $sort should be $low >= $minus[0] <= $high" if $minus[0] < $low || $minus[0] > $high;
+                       return  ", $sort should be $low >= $minus[1] <= $high" if $minus[1] < $low || $minus[1] > $high;
                        my $i;
                        for ($i = $minus[0]; $i <= $minus[1]; ++$i) {
                                push @req, 0 + $i; 
                        }
                } else {
-                       return 1 if $_ < $low || $_ > $high;
+                       return ", $sort should be $low >= $val <= $high" if $_ < $low || $_ > $high;
                        push @req, 0 + $_;
                }
        }
        $ref->{$sort} = \@req;
        
-       return 0;
+       return;
 }
 
 # process the cronjobs
@@ -245,10 +253,16 @@ sub start_connect
 sub spawn
 {
        my $line = shift;
+       my $t0 = [gettimeofday];
 
-       my $fc = Mojo::IOLoop::Subprocess->new();
+       dbg("DXCron::spawn: $line") if isdbg("cron");
+       my $fc = DXSubprocess->new();
        $fc->run(
-                        sub {my @res = `$line`; return @res},
+                        sub {
+                                my @res = `$line`;
+#                               diffms("DXCron spawn 1", $line, $t0, scalar @res) if isdbg('chan');
+                                return @res
+                        },
                         sub {
                                 my ($fc, $err, @res) = @_; 
                                 if ($err) {
@@ -260,6 +274,7 @@ sub spawn
                                         chomp;
                                         dbg("DXCron::spawn: $_") if isdbg("cron");
                                 }
+                                diffms("by DXCron::spawn", $line, $t0, scalar @res) if isdbg('progress');
                         }
                        );
 }
@@ -267,26 +282,30 @@ sub spawn
 sub spawn_cmd
 {
        my $line = shift;
+       my $t0 = [gettimeofday];
 
-       dbg("spawn_cmd run: $line") if isdbg('cron');
-       my $fc = Mojo::IOLoop::Subprocess->new();
+       dbg("DXCron::spawn_cmd run: $line") if isdbg('cron');
+       my $fc = DXSubprocess->new();
        $fc->run(
                         sub {
-                                $main::me->{_nospawn} = 1;
+                                ++$main::me->{_nospawn};
                                 my @res = $main::me->run_cmd($line);
-                                delete $main::me->{_nospawn};
+#                               diffms("DXCron spawn_cmd 1", $line, $t0, scalar @res) if isdbg('chan');
                                 return @res;
                         },
                         sub {
                                 my ($fc, $err, @res) = @_; 
+                                --$main::me->{_nospawn};
+                                delete $main::me->{_nospawn} if exists $main::me->{_nospawn} && $main::me->{_nospawn} <= 0;
                                 if ($err) {
-                                        my $s = "spawn_cmd: error $err";
+                                        my $s = "DXCron::spawn_cmd: error $err";
                                         dbg($s);
                                 }
                                 for (@res) {
                                         chomp;
-                                        dbg("spawn_cmd: $_") if isdbg("cron");
+                                        dbg("DXCron::spawn_cmd: $_") if isdbg("cron");
                                 }
+                                diffms("by DXCron::spawn_cmd", $line, $t0, scalar @res) if isdbg('progress');
                         }
                        );
 }
@@ -308,11 +327,11 @@ sub rcmd
 sub run_cmd
 {
        my $line = shift;
+       dbg("DXCron::run_cmd: $line") if isdbg('cron');
        my @in = $main::me->run_cmd($line);
-       dbg("cmd run: $line") if isdbg('cron');
        for (@in) {
-               s/\s*$//og;
-               dbg("cmd out: $_") if isdbg('cron');
+               s/\s*$//;
+               dbg("DXCron::cmd out: $_") if isdbg('cron');
        }
 }