2 # DX cluster user routines
4 # Copyright (c) 1998-2020 - Dirk Koopman G1TLH
6 # The new internal structure of the users system looks like this:
8 # The users.v4 file formatted as a file of lines containing: <callsign>\t{json serialised version of user record}\n
10 # You can look at it with any text tools or your favourite editor :-)
12 # In terms of internal structure, the main user hash remains as %u, keyed on callsign as before.
14 # The value is a one or two element array [position] or [position, ref], depending on whether the record has been "get()ed"
15 # [i.e. got from disk] or not. The 'position' is simply the start of each line in the file. The function "get()" simply returns
16 # the stored reference in array[1], if present, or seeks to the position from array[0], reads a line, json_decodes it,
17 # stores that reference into array[1] and returns that. That reference will be used from that time onwards.
19 # The routine writeoutjson() will (very) lazily write out a copy of %u WITHOUT STORING ANY EXTRA CURRENTLY UNREFERENCED CALLSIGN
20 # records to users.v4.n. It, in effect, does a sort of random accessed merge of the current user file and any "in memory"
21 # versions of any user record. This can be done with a spawned command because it will just be reading %u and merging
22 # loaded records, not altering the current users.v4 file in any way.
24 # %u -> $u{call} -> [position of json line in users.v4 (, reference -> {call=>'G1TLH', ...} if this record is in use)].
26 # On my machine, it takes about 250mS to read the entire users.v4 file of 190,000 records and to create a
27 # $u{callsign}->[record position in users.v4] for every callsign in the users.v4 file. Loading ~19,000 records
28 # (read from disk, decode json, store reference) takes about 110mS (or 580nS/record).
30 # A periodic dump of users.v4.n, with said ~19,000 records in memory takes about 750mS to write (this can be speeded up,
31 # by at least a half, if it becomes a problem!). As this periodic dump will be spawned off, it will not interrupt the data
34 # This is the first rewrite of DXUsers since inception. In the mojo branch we will no longer use Storable but use JSON instead.
35 # We will now be storing all the keys in memory and will use opportunistic loading of actual records in "get()". So out of
36 # say 200,000 known users it is unlikely that we will have more than 10% (more likely less) of the user records in memory.
37 # This will mean that there will be a increase in memory requirement, but it is modest. I estimate it's unlikely be more
40 # At the moment that means that the working users.v4 is "immutable".
42 # In normal operation, when first calling 'init()', the keys and positions will be read from the newer of users.v4.n and
43 # users.v4. If there is no users.v4.n, then users.v4 will be used. As time wears on, %u will then accrete active user records.
44 # Once an hour the current %u will be saved to users.v4.n.
46 # If it becomes too much of a problem then we are likely to chuck off "close()d" users onto the end of the current users.v4
47 # leaving existing users intact, but updating the pointer to the (now cleared out) user ref to the new location. This will
48 # be a sort of write behind log file. The users.v4 file is still immutable for the starting positions, but any chucked off
49 # records (or even "updates") will be written to the end of that file. If this has to be reread at any time, then the last
50 # entry for any callsign "wins". But this will only happen if I think the memory requirements over time become too much.
52 # As there is no functional difference between the users.v4 and export_user generated "user_json" file(s), other than the latter
53 # will be in sorted order with the record elements in "canonical" order. There will now longer be any code to execute to
54 # "restore the users file". Simply copy one of the "user_json" files to users.v4, remove users.v4.n and restart.
56 # Hopefully though, this will put to rest the need to do all that messing about ever again... Pigs may well be seen flying over
57 # your node as well :-)
72 use Data::Structure::Util qw(unbless);
73 use Time::HiRes qw(gettimeofday tv_interval);
78 use vars qw(%u $filename %valid $lastoperinterval $lasttime $lru $lrusize $tooold $v3 $v4);
82 $lastoperinterval = 60*24*60*60;
85 $tooold = 86400 * 365 + 31; # this marks an old user who hasn't given enough info to be useful
90 our $maxconnlist = 3; # remember this many connection time (duration) [start, end] pairs
92 our $newusers = 0; # per execution stats
98 my $ifh; # the input file, initialised by readinjson()
101 # hash of valid elements and a simple prompt
103 call => '0,Callsign',
104 alias => '0,Real Callsign',
107 lat => '0,Latitude,slat',
108 long => '0,Longitude,slong',
110 email => '0,E-mail Address,parray',
111 priv => '9,Privilege Level',
112 lastin => '0,Last Time in,cldatetime',
113 passwd => '9,Password,yesno',
114 passphrase => '9,Pass Phrase,yesno',
115 addr => '0,Full Address',
116 'sort' => '0,Type of User', # A - ak1a, U - User, S - spider cluster, B - BBS
117 xpert => '0,Expert Status,yesno',
119 node => '0,Last Node',
120 homenode => '0,Home Node',
121 lockout => '9,Locked out?,yesno', # won't let them in at all
122 dxok => '9,Accept DX Spots?,yesno', # accept his dx spots?
123 annok => '9,Accept Announces?,yesno', # accept his announces?
124 lang => '0,Language',
125 hmsgno => '0,Highest Msgno',
126 group => '0,Group,parray', # used to create a group of users/nodes for some purpose or other
127 buddies => '0,Buddies,parray',
128 isolate => '9,Isolate network,yesno',
129 wantbeep => '0,Req Beep,yesno',
130 wantann => '0,Req Announce,yesno',
131 wantwwv => '0,Req WWV,yesno',
132 wantwcy => '0,Req WCY,yesno',
133 wantecho => '0,Req Echo,yesno',
134 wanttalk => '0,Req Talk,yesno',
135 wantwx => '0,Req WX,yesno',
136 wantdx => '0,Req DX Spots,yesno',
137 wantemail => '0,Req Msgs as Email,yesno',
138 pagelth => '0,Current Pagelth',
139 pingint => '9,Node Ping interval',
140 nopings => '9,Ping Obs Count',
141 wantlogininfo => '0,Login Info Req,yesno',
142 wantgrid => '0,Show DX Grid,yesno',
143 wantann_talk => '0,Talklike Anns,yesno',
144 wantpc16 => '9,Want Users from node,yesno',
145 wantsendpc16 => '9,Send PC16,yesno',
146 wantroutepc19 => '9,Route PC19,yesno',
147 wantusstate => '0,Show US State,yesno',
148 wantdxcq => '0,Show CQ Zone,yesno',
149 wantdxitu => '0,Show ITU Zone,yesno',
150 wantgtk => '0,Want GTK interface,yesno',
151 wantpc9x => '0,Want PC9X interface,yesno',
152 lastoper => '9,Last for/oper,cldatetime',
153 nothere => '0,Not Here Text',
154 registered => '9,Registered?,yesno',
155 prompt => '0,Required Prompt',
156 version => '1,Version',
158 believe => '1,Believable nodes,parray',
159 lastping => '1,Last Ping at,ptimelist',
160 maxconnect => '1,Max Connections',
161 startt => '0,Start Time,cldatetime',
162 connlist => '1,Connections,parraydifft',
169 my $name = $AUTOLOAD;
171 return if $name =~ /::DESTROY$/;
174 confess "Non-existant field '$AUTOLOAD'" if !$valid{$name};
175 # this clever line of code creates a subroutine which takes over from autoload
176 # from OO Perl - Conway
177 *$AUTOLOAD = sub {@_ > 1 ? $_[0]->{$name} = $_[1] : $_[0]->{$name}};
184 # initialise the system
195 $json = JSON->new()->canonical(1);
196 $filename = $ufn = localdata("$fn.v4");
198 if (-e localdata("$fn.v4")) {
206 if ( ! -e localdata("users.v3") && -e localdata("users.v2") ) {
209 LogDbg('',"the module Storable appears to be missing!!");
210 LogDbg('',"trying to continue in compatibility mode (this may fail)");
211 LogDbg('',"please install Storable from CPAN as soon as possible");
213 import Storable qw(nfreeze thaw);
214 $convert = 3 if -e localdata("users.v3") && !-e $ufn;
218 # do a conversion if required
220 my ($key, $val, $action, $count, $err) = ('','',0,0,0);
221 my $ta = [gettimeofday];
224 LogDbg('',"Converting the User File from V$convert to $fn.v4 ");
225 LogDbg('',"This will take a while, I suggest you go and have cup of strong tea");
226 my $odbm = tie (%oldu, 'DB_File', localdata("users.v$convert"), O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn.v$convert ($!) [rebuild it from user_asc?]";
227 for ($action = R_FIRST; !$odbm->seq($key, $val, $action); $action = R_NEXT) {
230 eval { $ref = storable_decode($val) };
232 eval { $ref = asc_decode($val) };
242 Log('err', "DXUser: error decoding $@");
247 my $t = _diffms($ta);
248 LogDbg('',"Conversion from users.v$convert to users.v4 completed $count records $err errors $t mS");
250 # now write it away for future use
251 $ta = [gettimeofday];
253 $count = writeoutjson();
255 LogDbg('',"New Userfile users.v4 write completed $count records $err errors $t mS");
256 LogDbg('',"Now restarting..");
259 # otherwise (i.e normally) slurp it in
267 # with extreme prejudice
269 unlink "$main::data/users.v3";
270 unlink "$main::local_data/users.v3";
273 unlink "$main::data/users.v4";
274 unlink "$main::local_data/users.v4";
279 # periodic processing
283 # if ($main::systime > $lasttime + 15) {
284 # #$dbm->sync if $dbm;
285 # $lasttime = $main::systime;
300 # new - create a new user
307 my $self = bless {call => $call, 'sort'=>'U'}, $pkg;
315 # $call =~ s/-\d+$//o;
317 confess "can't create existing call $call in User\n!" if $u{$call};
319 my $self = $pkg->alloc($call);
320 $u{$call} = [0, $self];
328 # get - get an existing user - this seems to return a different reference everytime it is
335 my $nodecode = shift;
337 return undef unless $ref;
340 $ifh->seek($ref->[0], 0);
341 my $l = $ifh->getline;
343 my ($k,$s) = split /\t/, $l;
344 return $s if $nodecode;
345 my $j = json_decode($s);
351 } elsif ($nodecode) {
352 return json_encode($ref->[1]);
358 # get an "ephemeral" reference - i.e. this will give you new temporary copy of
359 # the call's user record, but without storing it (if it isn't already there)
361 # This is not as quick as get()! But it will allow safe querying of the
362 # user file. Probably in conjunction with get_some_calls feeding it.
364 # NOTE: for cached records this, in effect, is a faster version of Storable's
365 # dclone - only about 3.5 times as fast!
374 return json_decode(json_encode($ref->[1]));
376 $ifh->seek($ref->[0], 0);
377 my $l = $ifh->getline;
379 my ($k,$s) = split /\t/, $l;
380 my $j = json_decode($s);
389 # get an existing record either from the channel (if there is one) or from the database
391 # It is important to note that if you have done a get (for the channel say) and you
392 # want access or modify that you must use this call (and you must NOT use get's all
393 # over the place willy nilly!)
395 # NOTE: mojo branch with newusers system:
396 # There is no longer any function difference between get_current()
397 # and get() as they will always reference the same record as held in %u. This is because
398 # there is no more (repeated) thawing of stored records from the underlying "database".
400 # BUT: notice the difference between this and the get_tmp() function. A get() will online an
401 # othewise unused record, so for poking around looking for that locked out user:
402 # MAKE SURE you use get_tmp(). It will likely still be quicker than DB_File and Storable!
409 # my $call = uc shift;
411 # my $dxchan = DXChannel::get($call);
413 # my $ref = $dxchan->user;
414 # return $ref if $ref && UNIVERSAL::isa($ref, 'DXUser');
416 # dbg("DXUser::get_current: got invalid user ref for $call from dxchan $dxchan->{call} ". ref $ref. " ignoring");
422 # get all callsigns in the database
427 return (sort keys %u);
431 # get some calls - provide a qr// style selector string as a partial key
436 my $pattern = shift || qr/.*/;
437 return sort grep {$pattern} keys %u;
441 # if I understand the term correctly, this is a sort of monad.
443 # Scan through the whole user file and select records that you want
444 # to process further. This routine returns lines of json, yu
446 # the CODE ref should look like:
450 # # maybe do a rough check to see if this is a likely candidate
451 # return unless $line =~ /something/;
452 # my $r = json_decode($l);
453 # return (condition ? wanted thing : ());
462 if (ref($c) eq 'CODE') {
463 foreach my $k (get_all_calls()) {
464 my $l = get($k, 1); # get the offline json line or the jsoned online version
465 push @out, $c->($k, $l) if $l;
468 dbg("DXUser::scan supplied argument is not a code ref");
480 confess "Trying to put nothing!" unless $self && ref $self;
481 $self->{lastin} = $main::systime;
482 ++$modusers; # new or existing, it's still been modified
488 goto &json_encode if $v4;
489 goto &asc_encode unless $v3;
491 return nfreeze($self);
497 goto &json_decode if $v4;
498 goto &storable_decode if $v3;
502 # should now be obsolete for mojo branch build 238 and above
512 # create a hash from a string (in ascii)
518 $s =~ s/\%([0-9A-F][0-9A-F])/chr(hex($1))/eg;
521 LogDbg('err', "DXUser::asc_decode: on '$s' $@");
531 eval { $ref = $json->decode($s) };
533 return bless $ref, 'DXUser';
535 LogDbg('err', "DXUser::json_decode: on '$s' $@");
544 my $s = $json->encode($ref);
545 bless $ref, 'DXUser';
550 # del - delete a user
556 my $call = $self->{call};
559 --$cachedusers if $u{$call}->[1];
564 # close - close down a user
572 $self->{lastin} = $main::systime;
573 # add a record to the connect list
587 # return a list of valid elements
597 # export the database to an ascii file
604 my $fn = $name || localdata("user_json"); # force use of local_data
607 move "$fn.oooo", "$fn.ooooo" if -e "$fn.oooo";
608 move "$fn.ooo", "$fn.oooo" if -e "$fn.ooo";
609 move "$fn.oo", "$fn.ooo" if -e "$fn.oo";
610 move "$fn.o", "$fn.oo" if -e "$fn.o";
611 move "$fn", "$fn.o" if -e "$fn";
613 my $json = JSON->new;
619 my $fh = new IO::File ">$fn" or return "cannot open $fn ($!)";
623 foreach my $k (sort keys %u) {
625 if ($r->{sort} eq 'U' && !$r->{priv} && $main::systime > $r->{lastin}+$tooold ) {
626 unless ($r->{lat} || $r->{long} || $r->{qra} || $r->{qth} || $r->{name}) {
627 LogDbg('err', "DXUser::export deleting $k - too old, last in " . cldatetime($r->lastin) . " " . difft([$r->lastin, $main::systime]));
633 eval {$val = json_encode($r);};
635 LogDbg('err', "DXUser::export error encoding call: $k $@");
639 $fh->print("$k\t$val\n");
644 my $s = qq{Exported users to $fn - $count Users $del Deleted $err Errors ('sh/log Export' for details)};
645 LogDbg('command', $s);
653 # add one or more groups
657 my $ref = $self->{group} || [ 'local' ];
658 $self->{group} = $ref if !$self->{group};
659 push @$ref, @_ if @_;
662 # remove one or more groups
666 my $ref = $self->{group} || [ 'local' ];
669 $self->{group} = $ref if !$self->{group};
671 @$ref = map { my $a = $_; return (grep { $_ eq $a } @in) ? () : $a } @$ref;
674 # does this thing contain all the groups listed?
678 my $ref = $self->{group};
681 return 0 if !$ref || @_ == 0;
682 return 1 if @$ref == 0 && @_ == 0;
683 for ($n = 0; $n < @_; ) {
686 $n++ if grep $_ eq $a, @_;
692 # simplified group test just for one group
697 my $ref = $self->{group};
700 return grep $_ eq $s, $ref;
703 # set up a default group (only happens for them's that connect direct)
707 $self->{group} = [ 'local' ];
710 # set up empty buddies (only happens for them's that connect direct)
714 $self->{buddies} = [ ];
718 # return a prompt for a field
723 my ($self, $ele) = @_;
727 # some variable accessors
731 @_ ? $self->{'sort'} = shift : $self->{'sort'} ;
736 # want is default = 1
743 $self->{$s} = $val if defined $val;
744 return exists $self->{$s} ? $self->{$s} : 1;
747 # wantnot is default = 0
754 $self->{$s} = $val if defined $val;
755 return exists $self->{$s} ? $self->{$s} : 0;
760 return _want('beep', @_);
765 return _want('ann', @_);
770 return _want('wwv', @_);
775 return _want('wcy', @_);
780 return _want('echo', @_);
785 return _want('wx', @_);
790 return _want('dx', @_);
795 return _want('talk', @_);
800 return _want('grid', @_);
805 return _want('email', @_);
810 return _want('ann_talk', @_);
815 return _want('pc16', @_);
820 return _want('sendpc16', @_);
825 return _want('routepc16', @_);
830 return _want('usstate', @_);
835 return _want('dxcq', @_);
840 return _want('dxitu', @_);
845 return _want('gtk', @_);
850 return _want('pc9x', @_);
857 $self->{wantlogininfo} = $val if defined $val;
858 return $self->{wantlogininfo};
864 return $self->{sort} =~ /^[ACRSX]$/;
870 return grep $_ eq 'local_node', @{$self->{group}};
876 return $self->{sort} =~ /^[UW]$/;
882 return $self->{sort} eq 'W';
888 return $self->{sort} eq 'B';
894 return $self->{sort} eq 'S';
900 return $self->{sort} eq 'C';
906 return $self->{sort} eq 'X';
912 return $self->{sort} eq 'R';
918 return $self->{sort} eq 'A';
924 delete $self->{passwd};
931 delete $self->{passphrase};
939 $self->{believe} ||= [];
940 unless (grep $_ eq $call, @{$self->{believe}}) {
941 push @{$self->{believe}}, $call;
950 if (exists $self->{believe}) {
951 $self->{believe} = [grep {$_ ne $call} @{$self->{believe}}];
952 delete $self->{believe} unless @{$self->{believe}};
960 return exists $self->{believe} ? @{$self->{believe}} : ();
967 $self->{lastping} ||= {};
968 $self->{lastping} = {} unless ref $self->{lastping};
969 my $b = $self->{lastping};
970 $b->{$call} = shift if @_;
975 # read in the latest version of the user file. As this file is immutable, the file one really wants is
976 # a later (generated) copy. But, if the plain users.v4 file is all we have, we'll use that.
985 my $ta = [gettimeofday];
991 dbg("DXUser $fn not found - probably about to convert");
995 if (-e $nfn && -e $fn && (stat($nfn))[9] > (stat($fn))[9]) {
996 # move the old file to .o
1005 $ifh = IO::File->new("+<$fn") or die "$fn read error $!";
1007 my $pos = $ifh->tell;
1017 # $ifh is "global" and should not be closed
1019 dbg("DXUser::readinjson $count record headers read from $fn in ". _diffms($ta) . " mS");
1020 return $totusers = $count;
1024 # Write a newer copy of the users.v4 file to users.v4.n, which is what will be read in.
1025 # This means that the existing users.v4 is not touched during a run of dxspider, or at least
1030 my $ofn = shift || "$filename.n";
1031 my $ta = [gettimeofday];
1033 my $ofh = IO::File->new(">$ofn") or die "$ofn write error $!";
1036 for my $k (sort keys %u) {
1040 print $ofh "$k\t$l\n";
1043 LogDbg('DXCommand', "DXUser::writeoutjson callsign $k not found")
1048 dbg("DXUser::writeoutjson $count records written to $ofn in ". _diffms($ta) . " mS");