remove duplicate fields in DXUser to match DXuser
[spider.git] / perl / DXUser.pm
index 1af675ab52913e94a12bbc8e24bfe0b70dc95e02..b5f56c2a442f1ab52de8af97b1fa855a6ae8d8e9 100644 (file)
@@ -24,7 +24,7 @@ use DXJSON;
 
 use strict;
 
-use vars qw(%u $dbm $filename %valid $lastoperinterval $lasttime $lru $lrusize $tooold $v3);
+use vars qw(%u $dbm $filename %valid $lastoperinterval $lasttime $lru $lrusize $tooold $veryold $v3);
 
 %u = ();
 $dbm = undef;
@@ -32,7 +32,8 @@ $filename = undef;
 $lastoperinterval = 60*24*60*60;
 $lasttime = 0;
 $lrusize = 5000;
-$tooold = 86400 * 365;         # this marks an old user who hasn't given enough info to be useful
+$tooold = 86400 * 365 * 2;             # this marks an old user who hasn't given enough info to be useful
+$veryold = $tooold * 6;                # Ancient default 12 years
 $v3 = 0;
 our $maxconnlist = 3;                  # remember this many connection time (duration) [start, end] pairs
 
@@ -40,75 +41,76 @@ my $json;
 
 # hash of valid elements and a simple prompt
 %valid = (
-                 call => '0,Callsign',
-                 alias => '0,Real Callsign',
-                 name => '0,Name',
-                 qth => '0,Home QTH',
-                 lat => '0,Latitude,slat',
-                 long => '0,Longitude,slong',
-                 qra => '0,Locator',
-                 email => '0,E-mail Address,parray',
-                 priv => '9,Privilege Level',
-                 lastin => '0,Last Time in,cldatetime',
-                 lastseen => '0,Last Seen,cldatetime',
-                 passwd => '9,Password,yesno',
-                 passphrase => '9,Pass Phrase,yesno',
-                 addr => '0,Full Address',
                  'sort' => '0,Type of User', # A - ak1a, U - User, S - spider cluster, B - BBS
-                 xpert => '0,Expert Status,yesno',
+                 addr => '0,Full Address',
+                 alias => '0,Real Callsign',
+                 annok => '9,Accept Announces?,yesno', # accept his announces?
                  bbs => '0,Home BBS',
-                 node => '0,Last Node',
-                 homenode => '0,Home Node',
-                 lockout => '9,Locked out?,yesno',     # won't let them in at all
+                 believe => '1,Believable nodes,parray',
+                 buddies => '0,Buddies,parray',
+                 build => '1,Build',
+                 call => '0,Callsign',
+                 connlist => '1,Connections,parraydifft',
                  dxok => '9,Accept DX Spots?,yesno', # accept his dx spots?
-                 annok => '9,Accept Announces?,yesno', # accept his announces?
-                 lang => '0,Language',
-                 hmsgno => '0,Highest Msgno',
+                 email => '0,E-mail Address,parray',
                  group => '0,Group,parray',    # used to create a group of users/nodes for some purpose or other
-                 buddies => '0,Buddies,parray',
+                 hmsgno => '0,Highest Msgno',
+                 homenode => '0,Home Node',
                  isolate => '9,Isolate network,yesno',
-                 wantbeep => '0,Req Beep,yesno',
-                 wantann => '0,Req Announce,yesno',
-                 wantwwv => '0,Req WWV,yesno',
-                 wantwcy => '0,Req WCY,yesno',
-                 wantecho => '0,Req Echo,yesno',
-                 wanttalk => '0,Req Talk,yesno',
-                 wantwx => '0,Req WX,yesno',
-                 wantdx => '0,Req DX Spots,yesno',
-                 wantemail => '0,Req Msgs as Email,yesno',
+                 K => '9,Seen on PC92 K,yesno',
+                 lang => '0,Language',
+                 lastin => '0,Last Time in,cldatetime',
+                 lastoper => '9,Last for/oper,cldatetime',
+                 lastping => '1,Last Ping at,ptimelist',
+                 lastseen => '0,Last Seen,cldatetime',
+                 lat => '0,Latitude,slat',
+                 lockout => '9,Locked out?,yesno',     # won't let them in at all
+                 long => '0,Longitude,slong',
+                 maxconnect => '1,Max Connections',
+                 name => '0,Name',
+                 node => '0,Last Node',
+                 nopings => '9,Ping Obs Count',
+                 nothere => '0,Not Here Text',
                  pagelth => '0,Current Pagelth',
+                 passphrase => '9,Pass Phrase,yesno',
+                 passwd => '9,Password,yesno',
                  pingint => '9,Node Ping interval',
-                 nopings => '9,Ping Obs Count',
-                 wantlogininfo => '0,Login Info Req,yesno',
-          wantgrid => '0,Show DX Grid,yesno',
+                 priv => '9,Privilege Level',
+                 prompt => '0,Required Prompt',
+                 qra => '0,Locator',
+                 qth => '0,Home QTH',
+                 rbnseeme => '0,RBN See Me',
+                 registered => '9,Registered?,yesno',
+                 startt => '0,Start Time,cldatetime',
+                 version => '1,Version',
+                 wantann => '0,Req Announce,yesno',
                  wantann_talk => '0,Talklike Anns,yesno',
-                 wantpc16 => '9,Want Users from node,yesno',
-                 wantsendpc16 => '9,Send PC16,yesno',
-                 wantroutepc19 => '9,Route PC19,yesno',
-                 wantusstate => '0,Show US State,yesno',
+                 wantbeacon => '0,Want RBN Beacon,yesno',
+                 wantbeep => '0,Req Beep,yesno',
+                 wantcw => '0,Want RBN CW,yesno',
+                 wantdx => '0,Req DX Spots,yesno',
                  wantdxcq => '0,Show CQ Zone,yesno',
                  wantdxitu => '0,Show ITU Zone,yesno',
+                 wantecho => '0,Req Echo,yesno',
+                 wantemail => '0,Req Msgs as Email,yesno',
+                 wantft => '0,Want RBN FT4/8,yesno',
                  wantgtk => '0,Want GTK interface,yesno',
+                 wantlogininfo => '0,Login Info Req,yesno',
+                 wantpc16 => '9,Want Users from node,yesno',
                  wantpc9x => '0,Want PC9X interface,yesno',
+                 wantpsk => '0,Want RBN PSK,yesno',
                  wantrbn => '0,Want RBN spots,yesno',
-                 wantft => '0,Want RBN FT4/8,yesno',
-                 wantcw => '0,Want RBN CW,yesno',
+                 wantroutepc19 => '9,Route PC19,yesno',
                  wantrtty => '0,Want RBN RTTY,yesno',
-                 wantpsk => '0,Want RBN PSK,yesno',
-                 wantbeacon => '0,Want RBN Beacon,yesno',
-                 lastoper => '9,Last for/oper,cldatetime',
-                 nothere => '0,Not Here Text',
-                 registered => '9,Registered?,yesno',
-                 prompt => '0,Required Prompt',
-                 version => '1,Version',
-                 build => '1,Build',
-                 believe => '1,Believable nodes,parray',
-                 lastping => '1,Last Ping at,ptimelist',
-                 maxconnect => '1,Max Connections',
-                 startt => '0,Start Time,cldatetime',
-                 connlist => '1,Connections,parraydifft',
+                 wantsendpc16 => '9,Send PC16,yesno',
+                 wanttalk => '0,Req Talk,yesno',
+                 wantusstate => '0,Show US State,yesno',
+                 wantwcy => '0,Req WCY,yesno',
+                 wantwwv => '0,Req WWV,yesno',
+                 wantwx => '0,Req WX,yesno',
                  width => '0,Preferred Width',
-                 rbnseeme => '0,RBN See Me',
+                 xpert => '0,Expert Status,yesno',
+          wantgrid => '0,Show DX Grid,yesno',
                 );
 
 #no strict;
@@ -211,7 +213,6 @@ sub new
 #      confess "can't create existing call $call in User\n!" if $u{$call};
 
        my $self = $pkg->alloc($call);
-       $self->{lastseen} = $main::systime;
        $self->put;
        return $self;
 }
@@ -229,14 +230,12 @@ sub get
        # is it in the LRU cache?
        my $ref = $lru->get($call);
        if ($ref && ref $ref eq 'DXUser') {
-               $ref->{lastseen} = $main::systime;
                return $ref;
        }
        
        # search for it
        unless ($dbm->get($call, $data)) {
                eval { $ref = decode($data); };
-               
                if ($ref) {
                        if (!UNIVERSAL::isa($ref, 'DXUser')) {
                                dbg("DXUser::get: got strange answer from decode of $call". ref $ref. " ignoring");
@@ -251,7 +250,6 @@ sub get
                        }
                        return undef;
                }
-               $ref->{lastseen} = $main::systime;
                $lru->put($call, $ref);
                return $ref;
        }
@@ -300,9 +298,9 @@ sub put
        my $call = $self->{call};
 
        $dbm->del($call);
-       delete $self->{annok} if $self->{annok};
-       delete $self->{dxok} if $self->{dxok};
-
+       delete $self->{annok};
+       delete $self->{dxok};
+       $self->{lastseen} = $main::systime;
        $lru->put($call, $self);
        my $ref = $self->encode;
        $dbm->put($call, $ref);
@@ -343,8 +341,8 @@ sub close
        my $self = shift;
        my $startt = shift;
        my $ip = shift;
-       $self->{lastseen} = $self->{lastin} = $main::systime;
        # add a record to the connect list
+       $self->{lastin} = $main::systime;
        my $ref = [$startt || $self->{startt}, $main::systime];
        push @$ref, $ip if $ip;
        push @{$self->{connlist}}, $ref;
@@ -383,16 +381,27 @@ sub export
        my $fn = $name ne 'user_json' ? $name : "$main::local_data/$name";                       # force use of local
        
        # save old ones
+       copy $fn, "$fn.keep" unless -e "$fn.keep";
+       copy "$fn.ooooo", "$fn.backstop" unless -e "$fn,backstop";
+
        move "$fn.oooo", "$fn.ooooo" if -e "$fn.oooo";
        move "$fn.ooo", "$fn.oooo" if -e "$fn.ooo";
        move "$fn.oo", "$fn.ooo" if -e "$fn.oo";
        move "$fn.o", "$fn.oo" if -e "$fn.o";
        move "$fn", "$fn.o" if -e "$fn";
 
+       
        my $ta = [gettimeofday];
        my $count = 0;
        my $err = 0;
        my $del = 0;
+       my $spurious = 0;
+       my $unlocked = 0;
+       my $old =  0;
+       my $ancient =  0;
+       my $nodes = 0;
+       my $renamed = 0;
+               
        my $fh = new IO::File ">$fn" or return "cannot open $fn ($!)";
        if ($fh) {
                my $key = 0;
@@ -484,30 +493,80 @@ print "There are $count user records and $err errors in $diff mS\n";
                        my $ref;
                        eval {$ref = decode($val); };
                        if ($ref) {
-                               my $t = $ref->{lastin} || 0;
-                               if ($ref->is_user && !$ref->{priv} && $main::systime > $t + $tooold) {
-                                       unless ($ref->{lat} && $ref->{long} || $ref->{qth} || $ref->{qra}) {
+                               my $t = $ref->{lastseen} if exists $ref->{lastseen};
+                               $t ||= $ref->{lastin} if exists $ref->{lastin};
+                               $t ||= $ref->{lastoper} if exists $ref->{lastoper};
+                               $t //= 0;
+                               
+                               if ($ref->is_user) {
+                                       if (!$ref->{priv} && $main::systime > $t + $tooold) {
+                                               unless (($ref->{lat} && $ref->{long}) || $ref->{qth} || $ref->{name} || $ref->{qra}) {
+                                                       LogDbg('DXCommand', sprintf("$ref->{call} deleted, empty and too Old at %s", difft($t, ' ')));
+                                                       ++$del;
+                                                       ++$old;
+                                                       eval {$dbm->del($key)};
+                                                       dbg(carp("Export Error2: delete '$key' => '$val' $@")) if $@;
+                                                       next;
+                                               }
+                                       }
+                                       if ($main::systime > $t + $veryold) {
+                                               LogDbg('DXCommand', sprintf("$ref->{call} deleted, POSITIVELY ANCIENT at %s", difft($t, ' ')));
+                                               ++$del;
+                                               ++$ancient;
+                                               eval {$dbm->del($key)};
+                                               dbg(carp("Export Error2: delete '$key' => '$val' $@")) if $@;
+                                               next;
+                                       }
+                                       if (exists $ref->{lockout} && $ref->{lockout} == 1 && exists $ref->{priv} && $ref->{priv} == 1) {
+                                               LogDbg('DXCommand', "$ref->{call} depriv'd and unlocked");
+                                               $ref->{lockout} = $ref->{priv} = 0;
+                                               $ref->put;
+                                               ++$unlocked;
+                                       }
+                                       if ($ref->is_node && $main::systime > $t + $veryold) {
+                                               LogDbg('DXCommand', sprintf("NODE $ref->{call} deleted (%s) old", difft($t, ' ')));
+                                               ++$del;
+                                               ++$nodes;
                                                eval {$dbm->del($key)};
                                                dbg(carp("Export Error2: delete '$key' => '$val' $@")) if $@;
-                                               LogDbg('DXCommand', "$ref->{call} deleted, too old");
-                                               $del++;
+                                               next;
+                                       }
+                                       
+                                       my $normcall = normalise_call($key);
+                                       if ($normcall ne $key) {
+                                               # if the normalised call does not exist, create it from the duff call.
+                                               my $nref = DXUser::get_current($normcall);
+                                               unless ($nref) {
+                                                       $ref->{call} = $normcall;
+                                                       $ref->put;
+                                                       LogDbg('DXCommand', "DXProt: spurious call $key normalises to $normcall renaming $key -> $normcall");
+                                                       ++$renamed;
+                                               } 
+                                               LogDbg('DXCommand', "DXProt: spurious call $key (should be $normcall), removing");
+                                               eval {$dbm->del($key)};
+                                               dbg(carp("Export Error1: delete $key => '$val' $@")) if $@;
+                                               ++$spurious;
+                                               ++$del;
                                                next;
                                        }
                                }
-                               # only store users that are reasonably active or have useful information
-                               print $fh "$key\t" . encode($ref) . "\n";
-                               ++$count;
                        } else {
                                LogDbg('DXCommand', "Export Error3: '$key'\t" . carp($val) ."\n$@");
                                eval {$dbm->del($key)};
                                dbg(carp("Export Error3: delete '$key' => '$val' $@")) if $@;
                                ++$err;
+                               next;
                        }
-               } 
-        $fh->close;
-    }
+                       
+                       # only store users that are reasonably active or have useful information
+                       print $fh "$key\t" . encode($ref) . "\n";
+                       ++$count;
+               }
+       } 
+       $fh->close;
+
        my $diff = _diffms($ta);
-       my $s = qq{Exported users to $fn - $count Users $del Deleted $err Errors in $diff mS ('sh/log Export' for details)};
+       my $s = qq{Exported users to $fn - $count Users,  $del Deleted ($old empty \& too old, $ancient ancient, $nodes nodes, $spurious spurious), $renamed renamed, $unlocked Unlocked, $err Errors in $diff mS ('sh/log Export' for details)};
        LogDbg('command', $s);
        return $s;
 }
@@ -836,6 +895,21 @@ sub lastping
        $b->{$call} = shift if @_;
        return $b->{$call};     
 }
+
+#sub registered
+#{
+#      my $self = shift;
+#      my $val;
+#      if (defined $_[0]) {
+#              $val = unpad($_[0]);
+#              $self->{registered} = $val;
+#      }
+#      if (exists $self->{registered}) {
+#              $val = $self->{registered} // 0;
+#      }
+#      return $val // 0 ;                                      # to stop undef warnings
+#}
+
 1;
 __END__