X-Git-Url: http://dxcluster.net/gitweb/gitweb.cgi?a=blobdiff_plain;f=perl%2FDXUser.pm;h=91f3a3f02caab2900ac3deb4f21d2ea81ffa088e;hb=refs%2Fheads%2Fnewusers;hp=c1ab95aeb52b46314a5ca8b274ea15441c65255d;hpb=83e360f5c54444dc63a14de5124ad0a78624bf5a;p=spider.git diff --git a/perl/DXUser.pm b/perl/DXUser.pm index c1ab95ae..91f3a3f0 100644 --- a/perl/DXUser.pm +++ b/perl/DXUser.pm @@ -1,9 +1,60 @@ # # DX cluster user routines # -# Copyright (c) 1998 - Dirk Koopman G1TLH +# Copyright (c) 1998-2020 - Dirk Koopman G1TLH # -# $Id$ +# The new internal structure of the users system looks like this: +# +# The users.v4 file formatted as a file of lines containing: \t{json serialised version of user record}\n +# +# You can look at it with any text tools or your favourite editor :-) +# +# In terms of internal structure, the main user hash remains as %u, keyed on callsign as before. +# +# The value is a one or two element array [position] or [position, ref], depending on whether the record has been "get()ed" +# [i.e. got from disk] or not. The 'position' is simply the start of each line in the file. The function "get()" simply returns +# the stored reference in array[1], if present, or seeks to the position from array[0], reads a line, json_decodes it, +# stores that reference into array[1] and returns that. That reference will be used from that time onwards. +# +# The routine writeoutjson() will (very) lazily write out a copy of %u WITHOUT STORING ANY EXTRA CURRENTLY UNREFERENCED CALLSIGN +# records to users.v4.n. It, in effect, does a sort of random accessed merge of the current user file and any "in memory" +# versions of any user record. This can be done with a spawned command because it will just be reading %u and merging +# loaded records, not altering the current users.v4 file in any way. +# +# %u -> $u{call} -> [position of json line in users.v4 (, reference -> {call=>'G1TLH', ...} if this record is in use)]. +# +# On my machine, it takes about 250mS to read the entire users.v4 file of 190,000 records and to create a +# $u{callsign}->[record position in users.v4] for every callsign in the users.v4 file. Loading ~19,000 records +# (read from disk, decode json, store reference) takes about 110mS (or 580nS/record). +# +# A periodic dump of users.v4.n, with said ~19,000 records in memory takes about 750mS to write (this can be speeded up, +# by at least a half, if it becomes a problem!). As this periodic dump will be spawned off, it will not interrupt the data +# stream. +# +# This is the first rewrite of DXUsers since inception. In the mojo branch we will no longer use Storable but use JSON instead. +# We will now be storing all the keys in memory and will use opportunistic loading of actual records in "get()". So out of +# say 200,000 known users it is unlikely that we will have more than 10% (more likely less) of the user records in memory. +# This will mean that there will be a increase in memory requirement, but it is modest. I estimate it's unlikely be more +# than 30 or so MB. +# +# At the moment that means that the working users.v4 is "immutable". +# +# In normal operation, when first calling 'init()', the keys and positions will be read from the newer of users.v4.n and +# users.v4. If there is no users.v4.n, then users.v4 will be used. As time wears on, %u will then accrete active user records. +# Once an hour the current %u will be saved to users.v4.n. +# +# If it becomes too much of a problem then we are likely to chuck off "close()d" users onto the end of the current users.v4 +# leaving existing users intact, but updating the pointer to the (now cleared out) user ref to the new location. This will +# be a sort of write behind log file. The users.v4 file is still immutable for the starting positions, but any chucked off +# records (or even "updates") will be written to the end of that file. If this has to be reread at any time, then the last +# entry for any callsign "wins". But this will only happen if I think the memory requirements over time become too much. +# +# As there is no functional difference between the users.v4 and export_user generated "user_json" file(s), other than the latter +# will be in sorted order with the record elements in "canonical" order. There will now longer be any code to execute to +# "restore the users file". Simply copy one of the "user_json" files to users.v4, remove users.v4.n and restart. +# +# Hopefully though, this will put to rest the need to do all that messing about ever again... Pigs may well be seen flying over +# your node as well :-) # package DXUser; @@ -13,28 +64,39 @@ use DB_File; use Data::Dumper; use Fcntl; use IO::File; -use DXDebug; use DXUtil; use LRU; +use File::Copy; +use JSON; +use DXDebug; +use Data::Structure::Util qw(unbless); +use Time::HiRes qw(gettimeofday tv_interval); +use IO::File; use strict; -use vars qw($VERSION $BRANCH); -$VERSION = sprintf( "%d.%03d", q$Revision$ =~ /(\d+)\.(\d+)/ ); -$BRANCH = sprintf( "%d.%03d", q$Revision$ =~ /\d+\.\d+\.(\d+)\.(\d+)/ || (0,0)); -$main::build += $VERSION; -$main::branch += $BRANCH; - -use vars qw(%u $dbm $filename %valid $lastoperinterval $lasttime $lru $lrusize $tooold $v3); +use vars qw(%u $filename %valid $lastoperinterval $lasttime $lru $lrusize $tooold $v3 $v4); %u = (); -$dbm = undef; $filename = undef; $lastoperinterval = 60*24*60*60; $lasttime = 0; $lrusize = 2000; -$tooold = 86400 * 365; # this marks an old user who hasn't given enough info to be useful +$tooold = 86400 * 365 + 31; # this marks an old user who hasn't given enough info to be useful $v3 = 0; +$v4 = 0; +my $json; + +our $maxconnlist = 3; # remember this many connection time (duration) [start, end] pairs + +our $newusers = 0; # per execution stats +our $modusers = 0; +our $totusers = 0; +our $delusers = 0; +our $cachedusers = 0; + +my $ifh; # the input file, initialised by readinjson() + # hash of valid elements and a simple prompt %valid = ( @@ -61,7 +123,8 @@ $v3 = 0; annok => '9,Accept Announces?,yesno', # accept his announces? lang => '0,Language', hmsgno => '0,Highest Msgno', - group => '0,Access Group,parray', # used to create a group of users/nodes for some purpose or other + group => '0,Group,parray', # used to create a group of users/nodes for some purpose or other + buddies => '0,Buddies,parray', isolate => '9,Isolate network,yesno', wantbeep => '0,Req Beep,yesno', wantann => '0,Req Announce,yesno', @@ -75,20 +138,34 @@ $v3 = 0; pagelth => '0,Current Pagelth', pingint => '9,Node Ping interval', nopings => '9,Ping Obs Count', - wantlogininfo => '9,Login info req,yesno', - wantgrid => '0,DX Grid Info,yesno', + wantlogininfo => '0,Login Info Req,yesno', + wantgrid => '0,Show DX Grid,yesno', wantann_talk => '0,Talklike Anns,yesno', - wantpc90 => '1,Req PC90,yesno', - wantnp => '1,Req New Protocol,yesno', wantpc16 => '9,Want Users from node,yesno', wantsendpc16 => '9,Send PC16,yesno', wantroutepc19 => '9,Route PC19,yesno', + wantusstate => '0,Show US State,yesno', + wantdxcq => '0,Show CQ Zone,yesno', + wantdxitu => '0,Show ITU Zone,yesno', + wantgtk => '0,Want GTK interface,yesno', + wantpc9x => '0,Want PC9X interface,yesno', + wantrbn => '0,Want RBN spots,yesno', + wantft => '0,Want RBN FT4/8,yesno', + wantcw => '0,Want RBN CW,yesno', + wantrtty => '0,Want RBN RTTY,yesno', + wantpsk => '0,Want RBN PSK,yesno', + wantbeacon => '0,Want (RBN) Beacon,yesno', lastoper => '9,Last for/oper,cldatetime', nothere => '0,Not Here Text', registered => '9,Registered?,yesno', prompt => '0,Required Prompt', version => '1,Version', build => '1,Build', + believe => '1,Believable nodes,parray', + lastping => '1,Last Ping at,ptimelist', + maxconnect => '1,Max Connections', + startt => '0,Start Time,cldatetime', + connlist => '1,Connections,parraydifft', ); #no strict; @@ -114,66 +191,37 @@ sub AUTOLOAD # sub init { - my ($pkg, $fn, $mode) = @_; + my $mode = shift; - confess "need a filename in User" if !$fn; - - my $ufn; - my $convert; - - eval { - require Storable; - }; - -# eval "use Storable qw(nfreeze thaw)"; - - if ($@) { - $ufn = "$fn.v2"; - } else { - import Storable qw(nfreeze thaw); - - $ufn = "$fn.v3"; - $v3 = 1; - $convert = ! -e $ufn; - } + my $convert = "$main::root/perl/convert-users-v3-to-v4.pl"; + my $export; + + $json = JSON->new()->canonical(1); + $filename = localdata("users.v4"); - if ($mode) { - $dbm = tie (%u, 'DB_File', $ufn, O_CREAT|O_RDWR, 0666, $DB_BTREE) or confess "can't open user file: $fn ($!) [rebuild it from user_asc?]"; + if (-e $filename || -e "$filename.n" || -e "$filename.o") { + $v4 = 1; } else { - $dbm = tie (%u, 'DB_File', $ufn, O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn ($!) [rebuild it from user_asc?]"; - } - - # do a conversion if required - if ($convert) { - my ($key, $val, $action, $count, $err) = ('','',0,0,0); +# if (-e localdata('users.v3')) { +# LogDbg('DXUser', "Converting " . localdata('users.v3') . " to new json version of users file, please wait"); +# if (-x $convert) { +# system($convert); +# ++$export; +# } +# } - my %oldu; - dbg("Converting the User File to V3 (I suggest you go and have cup of strong tea)"); - my $odbm = tie (%oldu, 'DB_File', "${fn}.v2", O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn ($!) [rebuild it from user_asc?]"; - for ($action = R_FIRST; !$odbm->seq($key, $val, $action); $action = R_NEXT) { - my $ref = asc_decode($val); - if ($ref) { - $ref->put; - $count++; - } else { - $err++ - } - } - undef $odbm; - untie %oldu; - dbg("Conversion completed $count records $err errors"); + die "User file $filename missing, please run $convert or copy a user_json backup from somewhere\n" unless -e "$filename.n" || -s $filename; } - $filename = $ufn; - $lru = LRU->newbase("DXUser", $lrusize); + readinjson(); + copy $filename, "$filename.n" unless -e "$filename.n"; + export() if $export; } sub del_file { - my ($pkg, $fn) = @_; - - confess "need a filename in User" if !$fn; - $fn .= $v3 ? ".v3" : ".v2"; - unlink $fn; + # with extreme prejudice + unlink "$main::data/users.v4"; + unlink "$main::local_data/users.v4"; } # @@ -181,10 +229,10 @@ sub del_file # sub process { - if ($main::systime > $lasttime + 15) { - $dbm->sync; - $lasttime = $main::systime; - } +# if ($main::systime > $lasttime + 15) { +# #$dbm->sync if $dbm; +# $lasttime = $main::systime; +# } } # @@ -193,26 +241,35 @@ sub process sub finish { - undef $dbm; - untie %u; + + writeoutjson(); } # # new - create a new user # -sub new +sub alloc { my $pkg = shift; my $call = uc shift; + my $self = bless {call => $call, 'sort'=>'U'}, $pkg; + return $self; +} + +sub new +{ + my $pkg = shift; + my $call = shift; # $call =~ s/-\d+$//o; -# confess "can't create existing call $call in User\n!" if $u{$call}; + confess "can't create existing call $call in User\n!" if $u{$call}; - my $self = bless {}, $pkg; - $self->{call} = $call; - $self->{'sort'} = 'U'; + my $self = $pkg->alloc($call); + $u{$call} = [0, $self]; $self->put; + ++$newusers; + ++$totusers; return $self; } @@ -223,41 +280,91 @@ sub new sub get { - my $pkg = shift; my $call = uc shift; - my $data; + my $nodecode = shift; + my $ref = $u{$call}; + return undef unless $ref; - # is it in the LRU cache? - my $ref = $lru->get($call); - return $ref if $ref; - - # search for it - unless ($dbm->get($call, $data)) { - $ref = decode($data); - $lru->put($call, $ref); - return $ref; + unless ($ref->[1]) { + $ifh->seek($ref->[0], 0); + my $l = $ifh->getline; + if ($l) { + my ($k,$s) = split /\t/, $l; + return $s if $nodecode; + my $j = json_decode($s); + if ($j) { + $ref->[1] = $j; + ++$cachedusers; + } + } + } elsif ($nodecode) { + return json_encode($ref->[1]); + } + return $ref->[1]; +} + +# +# get an "ephemeral" reference - i.e. this will give you new temporary copy of +# the call's user record, but without storing it (if it isn't already there) +# +# This is not as quick as get()! But it will allow safe querying of the +# user file. Probably in conjunction with get_some_calls feeding it. +# +# NOTE: for cached records this, in effect, is a faster version of Storable's +# dclone - only about 3.5 times as fast! +# + +sub get_tmp +{ + my $call = uc shift; + my $ref = $u{$call}; + if ($ref) { + if ($ref->[1]) { + return json_decode(json_encode($ref->[1])); + } + $ifh->seek($ref->[0], 0); + my $l = $ifh->getline; + if ($l) { + my ($k,$s) = split /\t/, $l; + my $j = json_decode($s); + return $j; + } } return undef; } # -# get an existing either from the channel (if there is one) or from the database +# Master branch: +# get an existing record either from the channel (if there is one) or from the database # # It is important to note that if you have done a get (for the channel say) and you # want access or modify that you must use this call (and you must NOT use get's all # over the place willy nilly!) # +# NOTE: mojo branch with newusers system: +# There is no longer any function difference between get_current() +# and get() as they will always reference the same record as held in %u. This is because +# there is no more (repeated) thawing of stored records from the underlying "database". +# +# BUT: notice the difference between this and the get_tmp() function. A get() will online an +# othewise unused record, so for poking around looking for that locked out user: +# MAKE SURE you use get_tmp(). It will likely still be quicker than DB_File and Storable! +# sub get_current { - my $pkg = shift; - my $call = uc shift; - - my $dxchan = DXChannel->get($call); - return $dxchan->user if $dxchan; - my $rref = Route::get($call); - return $rref->user if $rref && exists $rref->{user}; - return $pkg->get($call); + goto &get; + +# my $call = uc shift; +# +# my $dxchan = DXChannel::get($call); +# if ($dxchan) { +# my $ref = $dxchan->user; +# return $ref if $ref && UNIVERSAL::isa($ref, 'DXUser'); +# +# dbg("DXUser::get_current: got invalid user ref for $call from dxchan $dxchan->{call} ". ref $ref. " ignoring"); +# } +# return get($call); } # @@ -269,6 +376,49 @@ sub get_all_calls return (sort keys %u); } +# +# get some calls - provide a qr// style selector string as a partial key +# + +sub get_some_calls +{ + my $pattern = shift || qr/.*/; + return sort grep {$pattern} keys %u; +} + +# +# if I understand the term correctly, this is a sort of monad. +# +# Scan through the whole user file and select records that you want +# to process further. This routine returns lines of json, yu +# +# the CODE ref should look like: +# sub { +# my $key = shift; +# my $line = shift; +# # maybe do a rough check to see if this is a likely candidate +# return unless $line =~ /something/; +# my $r = json_decode($l); +# return (condition ? wanted thing : ()); +# } +# + +sub scan +{ + my $c = shift; + my @out; + + if (ref($c) eq 'CODE') { + foreach my $k (get_all_calls()) { + my $l = get($k, 1); # get the offline json line or the jsoned online version + push @out, $c->($k, $l) if $l; + } + } else { + dbg("DXUser::scan supplied argument is not a code ref"); + } + return @out; +} + # # put - put a user # @@ -277,63 +427,44 @@ sub put { my $self = shift; confess "Trying to put nothing!" unless $self && ref $self; - my $call = $self->{call}; - # delete all instances of this -# for ($dbm->get_dup($call)) { -# $dbm->del_dup($call, $_); -# } - $dbm->del($call); - delete $self->{annok} if $self->{annok}; - delete $self->{dxok} if $self->{dxok}; - $lru->put($call, $self); - my $ref = $self->encode; - $dbm->put($call, $ref); + $self->{lastin} = $main::systime; + ++$modusers; # new or existing, it's still been modified } # freeze the user sub encode { - goto &asc_encode unless $v3; - my $self = shift; - return nfreeze($self); + goto &json_encode; } # thaw the user sub decode { - goto &asc_decode unless $v3; - return thaw(shift); -} - -# -# create a string from a user reference (in_ascii) -# -sub asc_encode -{ - my $self = shift; - my $dd = new Data::Dumper([$self]); - $dd->Indent(0); - $dd->Terse(1); - $dd->Quotekeys($] < 5.005 ? 1 : 0); - return $dd->Dumpxs; + goto &json_decode; } -# -# create a hash from a string (in ascii) -# -sub asc_decode +sub json_decode { my $s = shift; - my $ref; - eval '$ref = ' . $s; - if ($@) { - dbg($@); - Log('err', $@); - $ref = undef; + my $ref; + eval { $ref = $json->decode($s) }; + if ($ref && !$@) { + return bless $ref, 'DXUser'; + } else { + LogDbg('DXUser', "DXUser::json_decode: on '$s' $@"); } - return $ref; + return undef; } +sub json_encode +{ + my $ref = shift; + unbless($ref); + my $s = $json->encode($ref); + bless $ref, 'DXUser'; + return $s; +} + # # del - delete a user # @@ -342,12 +473,10 @@ sub del { my $self = shift; my $call = $self->{call}; - # delete all instances of this -# for ($dbm->get_dup($call)) { -# $dbm->del_dup($call, $_); -# } - $lru->remove($call); - $dbm->del($call); + ++$delusers; + --$totusers; + --$cachedusers if $u{$call}->[1]; + delete $u{$call}; } # @@ -357,8 +486,14 @@ sub del sub close { my $self = shift; - $self->{lastin} = time; - $self->put(); + my $startt = shift; + my $ip = shift; + $self->{lastin} = $main::systime; + # add a record to the connect list + my $ref = [$startt || $self->{startt}, $main::systime]; + push @$ref, $ip if $ip; + push @{$self->{connlist}}, $ref; + shift @{$self->{connlist}} if @{$self->{connlist}} > $maxconnlist; } # @@ -367,7 +502,7 @@ sub close sub sync { - $dbm->sync; +# $dbm->sync; } # @@ -386,15 +521,21 @@ sub fields sub export { - my $fn = shift; + my $name = shift; + + my $fn = $name || localdata("user_json"); # force use of local_data + my $ta = [gettimeofday]; # save old ones - rename "$fn.oooo", "$fn.ooooo" if -e "$fn.oooo"; - rename "$fn.ooo", "$fn.oooo" if -e "$fn.ooo"; - rename "$fn.oo", "$fn.ooo" if -e "$fn.oo"; - rename "$fn.o", "$fn.oo" if -e "$fn.o"; - rename "$fn", "$fn.o" if -e "$fn"; - + move "$fn.oooo", "$fn.ooooo" if -e "$fn.oooo"; + move "$fn.ooo", "$fn.oooo" if -e "$fn.ooo"; + move "$fn.oo", "$fn.ooo" if -e "$fn.oo"; + move "$fn.o", "$fn.oo" if -e "$fn.o"; + move "$fn", "$fn.o" if -e "$fn"; + + my $json = JSON->new; + $json->canonical(1);; + my $count = 0; my $err = 0; my $del = 0; @@ -402,106 +543,31 @@ sub export if ($fh) { my $key = 0; my $val = undef; - my $action; - my $t = scalar localtime; - print $fh q{#!/usr/bin/perl -# -# The exported userfile for a DXSpider System -# -# Input file: $filename -# Time: $t -# - -package main; - -# search local then perl directories -BEGIN { - umask 002; - - # root of directory tree for this system - $root = "/spider"; - $root = $ENV{'DXSPIDER_ROOT'} if $ENV{'DXSPIDER_ROOT'}; - - unshift @INC, "$root/perl"; # this IS the right way round! - unshift @INC, "$root/local"; - - # try to detect a lockfile (this isn't atomic but - # should do for now - $lockfn = "$root/local/cluster.lck"; # lock file name - if (-e $lockfn) { - open(CLLOCK, "$lockfn") or die "Can't open Lockfile ($lockfn) $!"; - my $pid = ; - chomp $pid; - die "Lockfile ($lockfn) and process $pid exists - cluster must be stopped first\n" if kill 0, $pid; - close CLLOCK; - } -} - -package DXUser; - -use DXVars; -use DXUser; - -if (@ARGV) { - $main::userfn = shift @ARGV; - print "user filename now $userfn\n"; -} - -DXUser->del_file($main::userfn); -DXUser->init($main::userfn, 1); -%u = (); -my $count = 0; -my $err = 0; -while () { - chomp; - my @f = split /\t/; - my $ref = asc_decode($f[1]); - if ($ref) { - $ref->put(); - $count++; - } else { - print "# Error: $f[0]\t$f[1]\n"; - $err++ - } -} -DXUser->sync; DXUser->finish; -print "There are $count user records and $err errors\n"; -}; - print $fh "__DATA__\n"; - - for ($action = R_FIRST; !$dbm->seq($key, $val, $action); $action = R_NEXT) { - if (!is_callsign($key) || $key =~ /^0/) { - Log('DXCommand', "Export Error1: $key\t$val"); - eval {$dbm->del($key)}; - dbg(carp("Export Error1: $key\t$val\n$@")) if $@; - ++$err; - next; - } - my $ref = decode($val); - if ($ref) { - my $t = $ref->{lastin} || 0; - if ($main::systime > $t + $tooold) { - unless ($ref->{lat} && $ref->{long} || $ref->{qth} || $ref->{qra}) { - eval {$dbm->del($key)}; - dbg(carp("Export Error2: $key\t$val\n$@")) if $@; - Log('DXCommand', "$ref->{call} deleted, too old"); - $del++; - next; - } + foreach my $k (sort keys %u) { + my $r = get($k); + if ($r->{sort} eq 'U' && !$r->{priv} && $main::systime > $r->{lastin}+$tooold ) { + unless ($r->{lat} || $r->{long} || $r->{qra} || $r->{qth} || $r->{name}) { + LogDbg('export', "DXUser::export deleting $k - too old, last in " . cldatetime($r->lastin) . " " . difft([$r->lastin, $main::systime])); + delete $u{$k}; + ++$del; + next; } - # only store users that are reasonably active or have useful information - print $fh "$key\t" . $ref->asc_encode . "\n"; - ++$count; - } else { - Log('DXCommand', "Export Error3: $key\t$val"); - eval {$dbm->del($key)}; - dbg(carp("Export Error3: $key\t$val\n$@")) if $@; - ++$err; } - } + eval {$val = json_encode($r);}; + if ($@) { + LogDbg('export', "DXUser::export error encoding call: $k $@"); + ++$err; + next; + } + $fh->print("$k\t$val\n"); + ++$count; + } $fh->close; - } - return "$count Users $del Deleted $err Errors ('sh/log Export' for details)"; + } + my $t = _diffms($ta); + my $s = qq{Exported users to $fn - $count Users $del Deleted $err Errors in $t mS ('sh/log Export' for details)}; + LogDbg('DXUser', $s); + return $s; } # @@ -565,6 +631,13 @@ sub new_group $self->{group} = [ 'local' ]; } +# set up empty buddies (only happens for them's that connect direct) +sub new_buddies +{ + my $self = shift; + $self->{buddies} = [ ]; +} + # # return a prompt for a field # @@ -583,6 +656,8 @@ sub sort } # some accessors + +# want is default = 1 sub _want { my $n = shift; @@ -593,6 +668,17 @@ sub _want return exists $self->{$s} ? $self->{$s} : 1; } +# wantnot is default = 0 +sub _wantnot +{ + my $n = shift; + my $self = shift; + my $val = shift; + my $s = "want$n"; + $self->{$s} = $val if defined $val; + return exists $self->{$s} ? $self->{$s} : 0; +} + sub wantbeep { return _want('beep', @_); @@ -663,6 +749,31 @@ sub wantroutepc16 return _want('routepc16', @_); } +sub wantusstate +{ + return _want('usstate', @_); +} + +sub wantdxcq +{ + return _want('dxcq', @_); +} + +sub wantdxitu +{ + return _want('dxitu', @_); +} + +sub wantgtk +{ + return _want('gtk', @_); +} + +sub wantpc9x +{ + return _want('pc9x', @_); +} + sub wantlogininfo { my $self = shift; @@ -674,13 +785,25 @@ sub wantlogininfo sub is_node { my $self = shift; - return $self->{sort} =~ /[ACRSX]/; + return $self->{sort} =~ /^[ACRSX]$/; +} + +sub is_local_node +{ + my $self = shift; + return grep $_ eq 'local_node', @{$self->{group}}; } sub is_user { my $self = shift; - return $self->{sort} eq 'U'; + return $self->{sort} =~ /^[UW]$/; +} + +sub is_web +{ + my $self = shift; + return $self->{sort} eq 'W'; } sub is_bbs @@ -719,16 +842,144 @@ sub is_ak1a return $self->{sort} eq 'A'; } +sub is_rbn +{ + my $self = shift; + return $self->{sort} eq 'N' +} + sub unset_passwd { my $self = shift; delete $self->{passwd}; + $self->put; } sub unset_passphrase { my $self = shift; delete $self->{passphrase}; + $self->put; +} + +sub set_believe +{ + my $self = shift; + my $call = uc shift; + $self->{believe} ||= []; + unless (grep $_ eq $call, @{$self->{believe}}) { + push @{$self->{believe}}, $call; + $self->put; + }; +} + +sub unset_believe +{ + my $self = shift; + my $call = uc shift; + if (exists $self->{believe}) { + $self->{believe} = [grep {$_ ne $call} @{$self->{believe}}]; + delete $self->{believe} unless @{$self->{believe}}; + $self->put; + } +} + +sub believe +{ + my $self = shift; + return exists $self->{believe} ? @{$self->{believe}} : (); +} + +sub lastping +{ + my $self = shift; + my $call = shift; + $self->{lastping} ||= {}; + $self->{lastping} = {} unless ref $self->{lastping}; + my $b = $self->{lastping}; + $b->{$call} = shift if @_; + return $b->{$call}; +} + +# +# read in the latest version of the user file. As this file is immutable, the file one really wants is +# a later (generated) copy. But, if the plain users.v4 file is all we have, we'll use that. +# + +sub readinjson +{ + my $fn = $filename; + my $nfn = "$fn.n"; + my $ofn = "$fn.o"; + + my $ta = [gettimeofday]; + my $count = 0; + my $s; + my $err = 0; + + if (-e $nfn && -e $fn && (stat($nfn))[9] > (stat($fn))[9]) { + # move the old file to .o + unlink $ofn; + move($fn, $ofn); + move($nfn, $fn); + }; + + # if we don't have a users.v4 at this point, look for a backup users.v4.json, users.v4.n then users.v4.o + unless (-e $fn) { + move($nfn, $fn) unless -e $fn; # the users.v4 isn't there (maybe convert-users-v3-to-v4.pl + move("$fn.json", $fn); # from a run of convert-users-v3-to-v4.pl + move($ofn, $fn) unless -e $fn; # desperate now... + } + + if ($ifh) { + $ifh->seek(0, 0); + } else { + LogDbg("DXUser","DXUser::readinjson: opening $fn as users file"); + $ifh = IO::File->new("+<$fn") or die "Cannot open $fn ($!)"; + } + my $pos = $ifh->tell; + while (<$ifh>) { + chomp; + my @f = split /\t/; + $u{$f[0]} = [$pos]; + $count++; + $pos = $ifh->tell; + } + $ifh->seek(0, 0); + + # $ifh is "global" and should not be closed + + LogDbg('DXUser',"DXUser::readinjson $count record headers read from $fn in ". _diffms($ta) . " mS"); + return $totusers = $count; +} + +# +# Write a newer copy of the users.v4 file to users.v4.n, which is what will be read in. +# This means that the existing users.v4 is not touched during a run of dxspider, or at least +# not yet. + +sub writeoutjson +{ + my $ofn = shift || "$filename.n"; + my $ta = [gettimeofday]; + + my $ofh = IO::File->new(">$ofn") or die "$ofn write error $!"; + my $count = 0; + $ifh->seek(0, 0); + for my $k (sort keys %u) { + my $l = get($k, 1); + if ($l) { + chomp $l; + print $ofh "$k\t$l\n"; + ++$count; + } else { + LogDbg('DXUser', "DXUser::writeoutjson callsign $k not found") + } + } + + $ofh->close; + LogDbg('DXUser',"DXUser::writeoutjson $count records written to $ofn in ". _diffms($ta) . " mS"); + return $count; } 1; __END__