X-Git-Url: http://dxcluster.net/gitweb/gitweb.cgi?a=blobdiff_plain;f=perl%2FDXUser.pm;h=91f3a3f02caab2900ac3deb4f21d2ea81ffa088e;hb=refs%2Fheads%2Fnewusers;hp=b996e5c536bc6d377265cadbde0d8375a8f0bbea;hpb=f0323dc530e2aa3fe5b8cd50b1c8c35aac67c52e;p=spider.git

diff --git a/perl/DXUser.pm b/perl/DXUser.pm
index b996e5c5..91f3a3f0 100644
--- a/perl/DXUser.pm
+++ b/perl/DXUser.pm
@@ -1,9 +1,60 @@
 #
 # DX cluster user routines
 #
-# Copyright (c) 1998 - Dirk Koopman G1TLH
+# Copyright (c) 1998-2020 - Dirk Koopman G1TLH
 #
-# $Id$
+# The new internal structure of the users system looks like this:
+#
+# The users.v4 file formatted as a file of lines containing: <callsign>\t{json serialised version of user record}\n
+#
+# You can look at it with any text tools or your favourite editor :-)
+#
+# In terms of internal structure, the main user hash remains as %u, keyed on callsign as before.
+#
+# The value is a one or two element array [position] or [position, ref], depending on whether the record has been "get()ed"
+# [i.e. got from disk] or not. The 'position' is simply the start of each line in the file. The function "get()" simply returns
+# the stored reference in array[1], if present, or seeks to the  position from array[0], reads a line, json_decodes it,
+# stores that reference into array[1] and returns that. That reference will be used from that time onwards.
+#
+# The routine writeoutjson() will (very) lazily write out a copy of %u WITHOUT STORING ANY EXTRA CURRENTLY UNREFERENCED CALLSIGN
+# records to users.v4.n. It, in effect, does a sort of random accessed merge of the current user file and any "in memory"
+# versions of any user record. This can be done with a spawned command because it will just be reading %u and merging
+# loaded records, not altering the current users.v4 file in any way. 
+#
+# %u -> $u{call} -> [position of json line in users.v4 (, reference -> {call=>'G1TLH', ...} if this record is in use)].
+#
+# On my machine, it takes about 250mS to read the entire users.v4 file of 190,000 records and to create a
+# $u{callsign}->[record position in users.v4] for every callsign in the users.v4 file. Loading ~19,000 records
+# (read from disk, decode json, store reference) takes about 110mS (or 580nS/record).
+#
+# A periodic dump of users.v4.n, with said ~19,000 records in memory takes about 750mS to write (this can be speeded up,
+# by at least a half, if it becomes a problem!). As this periodic dump will be spawned off, it will not interrupt the data
+# stream.
+#
+# This is the first rewrite of DXUsers since inception. In the mojo branch we will no longer use Storable but use JSON instead.
+# We will now be storing all the keys in memory and will use opportunistic loading of actual records in "get()". So out of
+# say 200,000 known users it is unlikely that we will have more than 10% (more likely less) of the user records in memory.
+# This will mean that there will be a increase in memory requirement, but it is modest. I estimate it's unlikely be more
+# than 30 or so MB.
+#
+# At the moment that means that the working users.v4 is "immutable". 
+#
+# In normal operation, when first calling 'init()', the keys and positions will be read from the newer of users.v4.n and
+# users.v4. If there is no users.v4.n, then users.v4 will be used. As time wears on, %u will then accrete active user records.
+# Once an hour the current %u will be saved to users.v4.n.
+#
+# If it becomes too much of a problem then we are likely to chuck off "close()d" users onto the end of the current users.v4
+# leaving existing users intact, but updating the pointer to the (now cleared out) user ref to the new location. This will
+# be a sort of write behind log file. The users.v4 file is still immutable for the starting positions, but any chucked off
+# records (or even "updates") will be written to the end of that file. If this has to be reread at any time, then the last
+# entry for any callsign "wins". But this will only happen if I think the memory requirements over time become too much. 
+#
+# As there is no functional difference between the users.v4 and export_user generated "user_json" file(s), other than the latter
+# will be in sorted order with the record elements in "canonical" order. There will now longer be any code to execute to
+# "restore the users file". Simply copy one of the "user_json" files to users.v4, remove users.v4.n and restart. 
+#
+# Hopefully though, this will put to rest the need to do all that messing about ever again... Pigs may well be seen flying over
+# your node as well :-)
 #
 
 package DXUser;
@@ -13,23 +64,39 @@ use DB_File;
 use Data::Dumper;
 use Fcntl;
 use IO::File;
+use DXUtil;
+use LRU;
+use File::Copy;
+use JSON;
 use DXDebug;
+use Data::Structure::Util qw(unbless);
+use Time::HiRes qw(gettimeofday tv_interval);
+use IO::File;
 
 use strict;
 
-use vars qw($VERSION $BRANCH);
-$VERSION = sprintf( "%d.%03d", q$Revision$ =~ /(\d+)\.(\d+)/ );
-$BRANCH = sprintf( "%d.%03d", q$Revision$ =~ /\d+\.\d+\.(\d+)\.(\d+)/ ) || 0;
-$main::build += $VERSION;
-$main::branch += $BRANCH;
-
-use vars qw(%u $dbm $filename %valid $lastoperinterval $lasttime);
+use vars qw(%u  $filename %valid $lastoperinterval $lasttime $lru $lrusize $tooold $v3 $v4);
 
 %u = ();
-$dbm = undef;
 $filename = undef;
-$lastoperinterval = 30*24*60*60;
+$lastoperinterval = 60*24*60*60;
 $lasttime = 0;
+$lrusize = 2000;
+$tooold = 86400 * 365 + 31;		# this marks an old user who hasn't given enough info to be useful
+$v3 = 0;
+$v4 = 0;
+my $json;
+
+our $maxconnlist = 3;			# remember this many connection time (duration) [start, end] pairs
+
+our $newusers = 0;					# per execution stats
+our $modusers = 0;
+our $totusers = 0;
+our $delusers = 0;
+our $cachedusers = 0;
+
+my $ifh;						# the input file, initialised by readinjson()
+
 
 # hash of valid elements and a simple prompt
 %valid = (
@@ -44,6 +111,7 @@ $lasttime = 0;
 		  priv => '9,Privilege Level',
 		  lastin => '0,Last Time in,cldatetime',
 		  passwd => '9,Password,yesno',
+		  passphrase => '9,Pass Phrase,yesno',
 		  addr => '0,Full Address',
 		  'sort' => '0,Type of User', # A - ak1a, U - User, S - spider cluster, B - BBS
 		  xpert => '0,Expert Status,yesno',
@@ -55,75 +123,105 @@ $lasttime = 0;
 		  annok => '9,Accept Announces?,yesno', # accept his announces?
 		  lang => '0,Language',
 		  hmsgno => '0,Highest Msgno',
-		  group => '0,Access Group,parray',	# used to create a group of users/nodes for some purpose or other
+		  group => '0,Group,parray',	# used to create a group of users/nodes for some purpose or other
+		  buddies => '0,Buddies,parray',
 		  isolate => '9,Isolate network,yesno',
-		  wantbeep => '0,Rec Beep,yesno',
-		  wantann => '0,Rec Announce,yesno',
-		  wantwwv => '0,Rec WWV,yesno',
-		  wantwcy => '0,Rec WCY,yesno',
-		  wantecho => '0,Rec Echo,yesno',
-		  wanttalk => '0,Rec Talk,yesno',
-		  wantwx => '0,Rec WX,yesno',
-		  wantdx => '0,Rec DX Spots,yesno',
-		  wantemail => '0,Rec Msgs as Email,yesno',
+		  wantbeep => '0,Req Beep,yesno',
+		  wantann => '0,Req Announce,yesno',
+		  wantwwv => '0,Req WWV,yesno',
+		  wantwcy => '0,Req WCY,yesno',
+		  wantecho => '0,Req Echo,yesno',
+		  wanttalk => '0,Req Talk,yesno',
+		  wantwx => '0,Req WX,yesno',
+		  wantdx => '0,Req DX Spots,yesno',
+		  wantemail => '0,Req Msgs as Email,yesno',
 		  pagelth => '0,Current Pagelth',
 		  pingint => '9,Node Ping interval',
 		  nopings => '9,Ping Obs Count',
-		  wantlogininfo => '9,Login info req,yesno',
-          wantgrid => '0,DX Grid Info,yesno',
+		  wantlogininfo => '0,Login Info Req,yesno',
+          wantgrid => '0,Show DX Grid,yesno',
 		  wantann_talk => '0,Talklike Anns,yesno',
+		  wantpc16 => '9,Want Users from node,yesno',
+		  wantsendpc16 => '9,Send PC16,yesno',
+		  wantroutepc19 => '9,Route PC19,yesno',
+		  wantusstate => '0,Show US State,yesno',
+		  wantdxcq => '0,Show CQ Zone,yesno',
+		  wantdxitu => '0,Show ITU Zone,yesno',
+		  wantgtk => '0,Want GTK interface,yesno',
+		  wantpc9x => '0,Want PC9X interface,yesno',
+		  wantrbn => '0,Want RBN spots,yesno',
+		  wantft => '0,Want RBN FT4/8,yesno',
+		  wantcw => '0,Want RBN CW,yesno',
+		  wantrtty => '0,Want RBN RTTY,yesno',
+		  wantpsk => '0,Want RBN PSK,yesno',
+		  wantbeacon => '0,Want (RBN) Beacon,yesno',
 		  lastoper => '9,Last for/oper,cldatetime',
 		  nothere => '0,Not Here Text',
 		  registered => '9,Registered?,yesno',
 		  prompt => '0,Required Prompt',
+		  version => '1,Version',
+		  build => '1,Build',
+		  believe => '1,Believable nodes,parray',
+		  lastping => '1,Last Ping at,ptimelist',
+		  maxconnect => '1,Max Connections',
+		  startt => '0,Start Time,cldatetime',
+		  connlist => '1,Connections,parraydifft',
 		 );
 
-no strict;
+#no strict;
 sub AUTOLOAD
 {
-	my $self = shift;
+	no strict;
 	my $name = $AUTOLOAD;
   
 	return if $name =~ /::DESTROY$/;
-	$name =~ s/.*:://o;
+	$name =~ s/^.*:://o;
   
 	confess "Non-existant field '$AUTOLOAD'" if !$valid{$name};
 	# this clever line of code creates a subroutine which takes over from autoload
 	# from OO Perl - Conway
-	*{$AUTOLOAD} = sub {@_ > 1 ? $_[0]->{$name} = $_[1] : $_[0]->{$name}} ;
-	if (@_) {
-		$self->{$name} = shift;
-	}
-	return $self->{$name};
+	*$AUTOLOAD = sub {@_ > 1 ? $_[0]->{$name} = $_[1] : $_[0]->{$name}};
+       goto &$AUTOLOAD;
 }
 
-use strict;
+#use strict;
 
 #
 # initialise the system
 #
 sub init
 {
-	my ($pkg, $fn, $mode) = @_;
+	my $mode = shift;
   
-	confess "need a filename in User" if !$fn;
-	$fn .= ".v2";
-	if ($mode) {
-		$dbm = tie (%u, 'DB_File', $fn, O_CREAT|O_RDWR, 0666, $DB_BTREE) or confess "can't open user file: $fn ($!)";
+	my $convert = "$main::root/perl/convert-users-v3-to-v4.pl";
+	my $export;
+		
+	$json = JSON->new()->canonical(1);
+	$filename = localdata("users.v4");
+	
+	if (-e $filename || -e "$filename.n" || -e "$filename.o") {
+		$v4 = 1;
 	} else {
-		$dbm = tie (%u, 'DB_File', $fn, O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn ($!)";
+#		if (-e localdata('users.v3')) {
+#			LogDbg('DXUser', "Converting " . localdata('users.v3') . " to new json version of users file, please wait");
+#			if (-x $convert) {
+#				system($convert);
+#				++$export;
+#			}
+#		}
+		
+		die "User file $filename missing, please run $convert or copy a user_json backup from somewhere\n" unless -e "$filename.n" || -s $filename;
 	}
-	
-	$filename = $fn;
+	readinjson();
+	copy $filename, "$filename.n" unless -e "$filename.n";
+	export() if $export;
 }
 
 sub del_file
 {
-	my ($pkg, $fn) = @_;
-  
-	confess "need a filename in User" if !$fn;
-	$fn .= ".v2";
-	unlink $fn;
+	# with extreme prejudice
+	unlink "$main::data/users.v4";
+	unlink "$main::local_data/users.v4";
 }
 
 #
@@ -131,10 +229,10 @@ sub del_file
 #
 sub process
 {
-	if ($main::systime > $lasttime + 15) {
-		$dbm->sync;
-		$lasttime = $main::systime;
-	}
+#	if ($main::systime > $lasttime + 15) {
+#		#$dbm->sync if $dbm;
+#		$lasttime = $main::systime;
+#	}
 }
 
 #
@@ -143,26 +241,35 @@ sub process
 
 sub finish
 {
-	undef $dbm;
-	untie %u;
+	
+	writeoutjson();
 }
 
 #
 # new - create a new user
 #
 
-sub new
+sub alloc
 {
 	my $pkg = shift;
 	my $call = uc shift;
+	my $self = bless {call => $call, 'sort'=>'U'}, $pkg;
+	return $self;
+}
+
+sub new
+{
+	my $pkg = shift;
+	my $call = shift;
 	#  $call =~ s/-\d+$//o;
   
-#	confess "can't create existing call $call in User\n!" if $u{$call};
+	confess "can't create existing call $call in User\n!" if $u{$call};
 
-	my $self = bless {}, $pkg;
-	$self->{call} = $call;
-	$self->{'sort'} = 'U';
+	my $self = $pkg->alloc($call);
+	$u{$call} = [0, $self];
 	$self->put;
+	++$newusers;
+	++$totusers;
 	return $self;
 }
 
@@ -173,35 +280,91 @@ sub new
 
 sub get
 {
-	my $pkg = shift;
 	my $call = uc shift;
-	my $data;
-	unless ($dbm->get($call, $data)) {
-		return decode($data);
+	my $nodecode = shift;
+	my $ref = $u{$call};
+	return undef unless $ref;
+	
+	unless ($ref->[1]) {
+		$ifh->seek($ref->[0], 0);
+		my $l = $ifh->getline;
+		if ($l) {
+			my ($k,$s) = split /\t/, $l;
+			return $s if $nodecode;
+			my $j = json_decode($s);
+			if ($j) {
+				$ref->[1] = $j;
+				++$cachedusers;
+			}
+		}
+	} elsif ($nodecode) {
+		return json_encode($ref->[1]);
+	}
+	return $ref->[1];
+}
+
+#
+# get an "ephemeral" reference - i.e. this will give you new temporary copy of
+# the call's user record, but without storing it (if it isn't already there)
+#
+# This is not as quick as get()! But it will allow safe querying of the
+# user file. Probably in conjunction with get_some_calls feeding it.
+#
+# NOTE: for cached records this, in effect, is a faster version of Storable's
+# dclone - only about 3.5 times as fast!
+#
+
+sub get_tmp
+{
+	my $call = uc shift;
+	my $ref = $u{$call};
+	if ($ref) {
+		if ($ref->[1]) {
+			return json_decode(json_encode($ref->[1]));
+		}
+		$ifh->seek($ref->[0], 0);
+		my $l = $ifh->getline;
+		if ($l) {
+			my ($k,$s) = split /\t/, $l;
+			my $j = json_decode($s);
+			return $j;
+		}
 	}
 	return undef;
 }
 
 #
-# get an existing either from the channel (if there is one) or from the database
+# Master branch:
+# get an existing record either from the channel (if there is one) or from the database
 #
 # It is important to note that if you have done a get (for the channel say) and you
 # want access or modify that you must use this call (and you must NOT use get's all
 # over the place willy nilly!)
 #
+# NOTE: mojo branch with newusers system:
+# There is no longer any function difference between get_current()
+# and get() as they will always reference the same record as held in %u. This is because
+# there is no more (repeated) thawing of stored records from the underlying "database".
+#
+# BUT: notice the difference between this and the get_tmp() function. A get() will online an
+# othewise unused record, so for poking around looking for that locked out user:
+# MAKE SURE you use get_tmp(). It will likely still be quicker than DB_File and Storable!
+#
 
 sub get_current
 {
-	my $pkg = shift;
-	my $call = uc shift;
-  
-	my $dxchan = DXChannel->get($call);
-	return $dxchan->user if $dxchan;
-	my $data;
-	unless ($dbm->get($call, $data)) {
-		return decode($data);
-	}
-	return undef;
+	goto &get;
+	
+#	my $call = uc shift;
+#  
+#	my $dxchan = DXChannel::get($call);
+#	if ($dxchan) {
+#		my $ref = $dxchan->user;
+#		return $ref if $ref && UNIVERSAL::isa($ref, 'DXUser');
+#
+#		dbg("DXUser::get_current: got invalid user ref for $call from dxchan $dxchan->{call} ". ref $ref. " ignoring");
+#	}
+#	return get($call);
 }
 
 #
@@ -213,6 +376,49 @@ sub get_all_calls
 	return (sort keys %u);
 }
 
+#
+# get some calls - provide a qr// style selector string as a partial key
+#
+
+sub get_some_calls
+{
+	my $pattern = shift || qr/.*/;
+	return sort grep {$pattern} keys %u;
+}
+
+#
+# if I understand the term correctly, this is a sort of monad.
+#
+# Scan through the whole user file and select records that you want
+# to process further. This routine returns lines of json, yu
+#
+# the CODE ref should look like:
+# sub {
+#   my $key = shift;
+# 	my $line = shift;
+#   # maybe do a rough check to see if this is a likely candidate
+#   return unless $line =~ /something/;
+#   my $r = json_decode($l);
+# 	return (condition ? wanted thing : ());
+# }
+#
+	
+sub scan
+{
+	my $c = shift;
+	my @out;
+	
+	if (ref($c) eq 'CODE') {
+		foreach my $k (get_all_calls()) {
+			my $l = get($k, 1);	# get the offline json line or the jsoned online version
+			push @out, $c->($k, $l) if $l;
+		}
+	} else {
+		dbg("DXUser::scan supplied argument is not a code ref");
+	}
+	return @out;
+}
+
 #
 # put - put a user
 #
@@ -221,46 +427,44 @@ sub put
 {
 	my $self = shift;
 	confess "Trying to put nothing!" unless $self && ref $self;
-	my $call = $self->{call};
-	# delete all instances of this 
-#	for ($dbm->get_dup($call)) {
-#		$dbm->del_dup($call, $_);
-#	}
-	$dbm->del($call);
-	delete $self->{annok} if $self->{annok};
-	delete $self->{dxok} if $self->{dxok};
-	$dbm->put($call, $self->encode);
+	$self->{lastin} = $main::systime;
+	++$modusers;				# new or existing, it's still been modified
 }
 
-# 
-# create a string from a user reference
-#
+# freeze the user
 sub encode
 {
-	my $self = shift;
-	my $dd = new Data::Dumper([$self]);
-	$dd->Indent(0);
-	$dd->Terse(1);
-    $dd->Quotekeys($] < 5.005 ? 1 : 0);
-	return $dd->Dumpxs;
+	goto &json_encode;
 }
 
-#
-# create a hash from a string
-#
+# thaw the user
 sub decode
+{
+	goto &json_decode;
+}
+
+sub json_decode
 {
 	my $s = shift;
-	my $ref;
-	eval '$ref = ' . $s;
-	if ($@) {
-		dbg($@);
-		Log('err', $@);
-		$ref = undef;
+    my $ref;
+	eval { $ref = $json->decode($s) };
+	if ($ref && !$@) {
+        return bless $ref, 'DXUser';
+	} else {
+		LogDbg('DXUser', "DXUser::json_decode: on '$s' $@");
 	}
-	return $ref;
+	return undef;
 }
 
+sub json_encode
+{
+	my $ref = shift;
+	unbless($ref);
+    my $s = $json->encode($ref);
+	bless $ref, 'DXUser';
+	return $s;
+}
+	
 #
 # del - delete a user
 #
@@ -269,11 +473,10 @@ sub del
 {
 	my $self = shift;
 	my $call = $self->{call};
-	# delete all instances of this 
-#	for ($dbm->get_dup($call)) {
-#		$dbm->del_dup($call, $_);
-#	}
-	$dbm->del($call);
+	++$delusers;
+	--$totusers;
+	--$cachedusers if $u{$call}->[1];
+	delete $u{$call};
 }
 
 #
@@ -283,8 +486,14 @@ sub del
 sub close
 {
 	my $self = shift;
-	$self->{lastin} = time;
-	$self->put();
+	my $startt = shift;
+	my $ip = shift;
+	$self->{lastin} = $main::systime;
+	# add a record to the connect list
+	my $ref = [$startt || $self->{startt}, $main::systime];
+	push @$ref, $ip if $ip;
+	push @{$self->{connlist}}, $ref;
+	shift @{$self->{connlist}} if @{$self->{connlist}} > $maxconnlist;
 }
 
 #
@@ -293,7 +502,7 @@ sub close
 
 sub sync
 {
-	$dbm->sync;
+#	$dbm->sync;
 }
 
 #
@@ -312,80 +521,53 @@ sub fields
 
 sub export
 {
-	my $fn = shift;
+	my $name = shift;
+
+	my $fn = $name || localdata("user_json"); # force use of local_data
+	my $ta = [gettimeofday];
 	
 	# save old ones
-        rename "$fn.oooo", "$fn.ooooo" if -e "$fn.oooo";
-        rename "$fn.ooo", "$fn.oooo" if -e "$fn.ooo";
-        rename "$fn.oo", "$fn.ooo" if -e "$fn.oo";
-        rename "$fn.o", "$fn.oo" if -e "$fn.o";
-        rename "$fn", "$fn.o" if -e "$fn";
-
+	move "$fn.oooo", "$fn.ooooo" if -e "$fn.oooo";
+	move "$fn.ooo", "$fn.oooo" if -e "$fn.ooo";
+	move "$fn.oo", "$fn.ooo" if -e "$fn.oo";
+	move "$fn.o", "$fn.oo" if -e "$fn.o";
+	move "$fn", "$fn.o" if -e "$fn";
+
+	my $json = JSON->new;
+	$json->canonical(1);;
+	
 	my $count = 0;
+	my $err = 0;
+	my $del = 0;
 	my $fh = new IO::File ">$fn" or return "cannot open $fn ($!)";
 	if ($fh) {
-		my $ref = 0;
 		my $key = 0;
-		my $action;
-		my $t = scalar localtime;
-		print $fh q{#!/usr/bin/perl
-#
-# The exported userfile for a DXSpider System
-#
-# Input file: $filename
-#       Time: $t
-#
-			
-package main;
-			
-# search local then perl directories
-BEGIN {
-	umask 002;
-				
-	# root of directory tree for this system
-	$root = "/spider"; 
-	$root = $ENV{'DXSPIDER_ROOT'} if $ENV{'DXSPIDER_ROOT'};
-	
-	unshift @INC, "$root/perl";	# this IS the right way round!
-	unshift @INC, "$root/local";
-	
-	# try to detect a lockfile (this isn't atomic but 
-	# should do for now
-	$lockfn = "$root/perl/cluster.lck";       # lock file name
-	if (-e $lockfn) {
-		open(CLLOCK, "$lockfn") or die "Can't open Lockfile ($lockfn) $!";
-		my $pid = <CLLOCK>;
-		chomp $pid;
-		die "Lockfile ($lockfn) and process $pid exists - cluster must be stopped first\n" if kill 0, $pid;
-		close CLLOCK;
-	}
-}
-
-package DXUser;
-
-use DXVars;
-use DXUser;
-
-if (@ARGV) {
-	$main::userfn = shift @ARGV;
-	print "user filename now $userfn\n";
-}
-
-DXUser->del_file($main::userfn);
-DXUser->init($main::userfn, 1);
-
-%u = (
-  };
-
-        for ($action = R_FIRST; !$dbm->seq($key, $ref, $action); $action = R_NEXT) {
-			print $fh "'$key' => q{$ref},\n";
+		my $val = undef;
+		foreach my $k (sort keys %u) {
+			my $r = get($k);
+			if ($r->{sort} eq 'U' && !$r->{priv} && $main::systime > $r->{lastin}+$tooold ) {
+				unless ($r->{lat} || $r->{long} || $r->{qra} || $r->{qth} || $r->{name}) {
+					LogDbg('export', "DXUser::export deleting $k - too old, last in " . cldatetime($r->lastin) . " " . difft([$r->lastin, $main::systime]));
+					delete $u{$k};
+					++$del;
+					next;
+				}
+			}
+			eval {$val = json_encode($r);};
+			if ($@) {
+				LogDbg('export', "DXUser::export error encoding call: $k $@");
+				++$err;
+				next;
+			} 
+			$fh->print("$k\t$val\n");
 			++$count;
-		} 
-        print $fh ");\n#\nprint \"there were $count records\\n\";\n#\n";
-        print $fh "DXUser->sync; DXUser->finish;\n#\n";
+		}
         $fh->close;
-    } 
-	return $count;
+    }
+	my $t = _diffms($ta);
+	my $s = qq{Exported users to $fn - $count Users $del Deleted $err Errors in $t mS ('sh/log Export' for details)};
+	LogDbg('DXUser', $s);
+	return $s;
 }
 
 #
@@ -449,6 +631,13 @@ sub new_group
 	$self->{group} = [ 'local' ];
 }
 
+# set up empty buddies (only happens for them's that connect direct)
+sub new_buddies
+{
+	my $self = shift;
+	$self->{buddies} = [  ];
+}
+
 #
 # return a prompt for a field
 #
@@ -467,6 +656,8 @@ sub sort
 }
 
 # some accessors
+
+# want is default = 1
 sub _want
 {
 	my $n = shift;
@@ -477,6 +668,17 @@ sub _want
 	return exists $self->{$s} ? $self->{$s} : 1;
 }
 
+# wantnot is default = 0
+sub _wantnot
+{
+	my $n = shift;
+	my $self = shift;
+	my $val = shift;
+	my $s = "want$n";
+	$self->{$s} = $val if defined $val;
+	return exists $self->{$s} ? $self->{$s} : 0;
+}
+
 sub wantbeep
 {
 	return _want('beep', @_);
@@ -532,6 +734,46 @@ sub wantann_talk
 	return _want('ann_talk', @_);
 }
 
+sub wantpc16
+{
+	return _want('pc16', @_);
+}
+
+sub wantsendpc16
+{
+	return _want('sendpc16', @_);
+}
+
+sub wantroutepc16
+{
+	return _want('routepc16', @_);
+}
+
+sub wantusstate
+{
+	return _want('usstate', @_);
+}
+
+sub wantdxcq
+{
+	return _want('dxcq', @_);
+}
+
+sub wantdxitu
+{
+	return _want('dxitu', @_);
+}
+
+sub wantgtk
+{
+	return _want('gtk', @_);
+}
+
+sub wantpc9x
+{
+	return _want('pc9x', @_);
+}
+
 sub wantlogininfo
 {
 	my $self = shift;
@@ -543,13 +785,25 @@ sub wantlogininfo
 sub is_node
 {
 	my $self = shift;
-	return $self->{sort} =~ /[ACRSX]/;
+	return $self->{sort} =~ /^[ACRSX]$/;
+}
+
+sub is_local_node
+{
+	my $self = shift;
+	return grep $_ eq 'local_node', @{$self->{group}};
 }
 
 sub is_user
 {
 	my $self = shift;
-	return $self->{sort} eq 'U';
+	return $self->{sort} =~ /^[UW]$/;
+}
+
+sub is_web
+{
+	my $self = shift;
+	return $self->{sort} eq 'W';
 }
 
 sub is_bbs
@@ -588,10 +842,144 @@ sub is_ak1a
 	return $self->{sort} eq 'A';
 }
 
+sub is_rbn
+{
+	my $self = shift;
+	return $self->{sort} eq 'N'
+}
+
 sub unset_passwd
 {
 	my $self = shift;
 	delete $self->{passwd};
+	$self->put;
+}
+
+sub unset_passphrase
+{
+	my $self = shift;
+	delete $self->{passphrase};
+	$self->put;
+}
+
+sub set_believe
+{
+	my $self = shift;
+	my $call = uc shift;
+	$self->{believe} ||= [];
+	unless (grep $_ eq $call, @{$self->{believe}}) {
+		push @{$self->{believe}}, $call;
+		$self->put;
+	};
+}
+
+sub unset_believe
+{
+	my $self = shift;
+	my $call = uc shift;
+	if (exists $self->{believe}) {
+		$self->{believe} = [grep {$_ ne $call} @{$self->{believe}}];
+		delete $self->{believe} unless @{$self->{believe}};
+		$self->put;
+	}
+}
+
+sub believe
+{
+	my $self = shift;
+	return exists $self->{believe} ? @{$self->{believe}} : ();
+}
+
+sub lastping
+{
+	my $self = shift;
+	my $call = shift;
+	$self->{lastping} ||= {};
+	$self->{lastping} = {} unless ref $self->{lastping};
+	my $b = $self->{lastping};
+	$b->{$call} = shift if @_;
+	return $b->{$call};	
+}
+
+#
+# read in the latest version of the user file. As this file is immutable, the file one really wants is
+# a later (generated) copy. But, if the plain users.v4 file is all we have, we'll use that.
+#
+
+sub readinjson
+{
+	my $fn = $filename;
+	my $nfn = "$fn.n";
+	my $ofn = "$fn.o";
+
+	my $ta = [gettimeofday];
+	my $count = 0;
+	my $s;
+	my $err = 0;
+
+	if (-e $nfn && -e $fn && (stat($nfn))[9] > (stat($fn))[9]) {
+		# move the old file to .o
+		unlink $ofn;
+		move($fn, $ofn);
+		move($nfn, $fn);
+	};
+
+	# if we don't have a users.v4 at this point, look for a backup users.v4.json, users.v4.n then users.v4.o
+	unless (-e $fn) {
+		move($nfn, $fn) unless -e $fn; 	# the users.v4 isn't there (maybe convert-users-v3-to-v4.pl
+		move("$fn.json", $fn);			# from a run of convert-users-v3-to-v4.pl
+		move($ofn, $fn) unless -e $fn;	# desperate now...
+	}
+	
+	if ($ifh) {
+		$ifh->seek(0, 0);
+	} else {
+		LogDbg("DXUser","DXUser::readinjson: opening $fn as users file");
+		$ifh = IO::File->new("+<$fn") or die "Cannot open $fn ($!)";
+	}
+	my $pos = $ifh->tell;
+	while (<$ifh>) {
+		chomp;
+		my @f = split /\t/;
+		$u{$f[0]} = [$pos];
+		$count++;
+		$pos = $ifh->tell;
+	}
+	$ifh->seek(0, 0);
+
+	# $ifh is "global" and should not be closed
+	
+	LogDbg('DXUser',"DXUser::readinjson $count record headers read from $fn in ". _diffms($ta) . " mS");
+	return $totusers = $count;
+}
+
+#
+# Write a newer copy of the users.v4 file to users.v4.n, which is what will be read in.
+# This means that the existing users.v4 is not touched during a run of dxspider, or at least
+# not yet.
+
+sub writeoutjson
+{
+	my $ofn = shift || "$filename.n";
+	my $ta = [gettimeofday];
+	
+	my $ofh = IO::File->new(">$ofn") or die "$ofn write error $!";
+	my $count = 0;
+	$ifh->seek(0, 0);
+	for my $k (sort keys %u) {
+		my $l = get($k, 1);
+		if ($l) {
+			chomp $l;
+			print $ofh "$k\t$l\n";
+			++$count;
+		} else {
+			LogDbg('DXUser', "DXUser::writeoutjson callsign $k not found")
+		}
+	}
+	
+	$ofh->close;
+	LogDbg('DXUser',"DXUser::writeoutjson $count records written to $ofn in ". _diffms($ta) . " mS");
+	return $count;
 }
 1;
 __END__