X-Git-Url: http://dxcluster.net/gitweb/gitweb.cgi?a=blobdiff_plain;f=perl%2FDXUser.pm;h=116b69e62eebec01552ec5bb2a9c47066d9d4f61;hb=49d7cd28623df3c4accf72d48f74b4b00238a81a;hp=f78c8120181bfd39889e4be48a7b90ca60c4e9c6;hpb=fa27020ac9e6e3e8b64a2e82351eb00487b04bfb;p=spider.git

diff --git a/perl/DXUser.pm b/perl/DXUser.pm
index f78c8120..116b69e6 100644
--- a/perl/DXUser.pm
+++ b/perl/DXUser.pm
@@ -20,18 +20,21 @@ use File::Copy;
 use Data::Structure::Util qw(unbless);
 use Time::HiRes qw(gettimeofday tv_interval);
 use IO::File;
+use DXChannel;
+use DXJSON;
 
 use strict;
 
-use vars qw(%u $dbm $filename %valid $lastoperinterval $lasttime $lru $lrusize $tooold $v3);
+use vars qw(%u $dbm $filename %valid $lastoperinterval $lasttime $lru $lrusize $tooold $veryold $v3);
 
 %u = ();
 $dbm = undef;
 $filename = undef;
 $lastoperinterval = 60*24*60*60;
 $lasttime = 0;
-$lrusize = 10000;
-$tooold = 86400 * 365;		# this marks an old user who hasn't given enough info to be useful
+$lrusize = 5000;
+$tooold = 86400 * 365 * 2;		# this marks an old user who hasn't given enough info to be useful
+$veryold = $tooold * 6;	        # Ancient default 12 years
 $v3 = 0;
 our $maxconnlist = 3;			# remember this many connection time (duration) [start, end] pairs
 
@@ -39,73 +42,79 @@ my $json;
 
 # hash of valid elements and a simple prompt
 %valid = (
-		  call => '0,Callsign',
-		  alias => '0,Real Callsign',
-		  name => '0,Name',
-		  qth => '0,Home QTH',
-		  lat => '0,Latitude,slat',
-		  long => '0,Longitude,slong',
-		  qra => '0,Locator',
-		  email => '0,E-mail Address,parray',
-		  priv => '9,Privilege Level',
-		  lastin => '0,Last Time in,cldatetime',
-		  lastseen => '0,Last Seen,cldatetime',
-		  passwd => '9,Password,yesno',
-		  passphrase => '9,Pass Phrase,yesno',
-		  addr => '0,Full Address',
 		  'sort' => '0,Type of User', # A - ak1a, U - User, S - spider cluster, B - BBS
-		  xpert => '0,Expert Status,yesno',
+		  addr => '0,Full Address',
+		  alias => '0,Real Callsign',
+		  annok => '9,Accept Announces?,yesno', # accept his announces?
 		  bbs => '0,Home BBS',
-		  node => '0,Last Node',
-		  homenode => '0,Home Node',
-		  lockout => '9,Locked out?,yesno',	# won't let them in at all
+		  believe => '1,Believable nodes,parray',
+		  buddies => '0,Buddies,parray',
+		  build => '1,Build',
+		  call => '0,Callsign',
+		  clientoutput => '0,User OUT Format',
+		  clientinput => '0,User IN Format',
+		  connlist => '1,Connections,parraydifft',
 		  dxok => '9,Accept DX Spots?,yesno', # accept his dx spots?
-		  annok => '9,Accept Announces?,yesno', # accept his announces?
-		  lang => '0,Language',
-		  hmsgno => '0,Highest Msgno',
+		  email => '0,E-mail Address,parray',
 		  group => '0,Group,parray',	# used to create a group of users/nodes for some purpose or other
-		  buddies => '0,Buddies,parray',
+		  hmsgno => '0,Highest Msgno',
+		  homenode => '0,Home Node',
 		  isolate => '9,Isolate network,yesno',
-		  wantbeep => '0,Req Beep,yesno',
-		  wantann => '0,Req Announce,yesno',
-		  wantwwv => '0,Req WWV,yesno',
-		  wantwcy => '0,Req WCY,yesno',
-		  wantecho => '0,Req Echo,yesno',
-		  wanttalk => '0,Req Talk,yesno',
-		  wantwx => '0,Req WX,yesno',
-		  wantdx => '0,Req DX Spots,yesno',
-		  wantemail => '0,Req Msgs as Email,yesno',
+		  K => '9,Seen on PC92 K,yesno',
+		  lang => '0,Language',
+		  lastin => '0,Last Time in,cldatetime',
+		  lastoper => '9,Last for/oper,cldatetime',
+		  lastping => '1,Last Ping at,ptimelist',
+		  lastseen => '0,Last Seen,cldatetime',
+		  lat => '0,Latitude,slat',
+		  lockout => '9,Locked out?,yesno',	# won't let them in at all
+		  long => '0,Longitude,slong',
+		  maxconnect => '1,Max Connections',
+		  name => '0,Name',
+		  node => '0,Last Node',
+		  nopings => '9,Ping Obs Count',
+		  nothere => '0,Not Here Text',
 		  pagelth => '0,Current Pagelth',
+		  passphrase => '9,Pass Phrase,yesno',
+		  passwd => '9,Password,yesno',
 		  pingint => '9,Node Ping interval',
-		  nopings => '9,Ping Obs Count',
-		  wantlogininfo => '0,Login Info Req,yesno',
-          wantgrid => '0,Show DX Grid,yesno',
+		  priv => '9,Privilege Level',
+		  prompt => '0,Required Prompt',
+		  qra => '0,Locator',
+		  qth => '0,Home QTH',
+		  rbnseeme => '0,RBN See Me,yesno',
+		  registered => '9,Registered?,yesno',
+		  startt => '0,Start Time,cldatetime',
+		  user_interval => '0,Prompt IdleTime',
+		  version => '1,Version',
+		  wantann => '0,Req Announce,yesno',
 		  wantann_talk => '0,Talklike Anns,yesno',
-		  wantpc16 => '9,Want Users from node,yesno',
-		  wantsendpc16 => '9,Send PC16,yesno',
-		  wantroutepc19 => '9,Route PC19,yesno',
-		  wantusstate => '0,Show US State,yesno',
+		  wantbeacon => '0,Want RBN Beacon,yesno',
+		  wantbeep => '0,Req Beep,yesno',
+		  wantcw => '0,Want RBN CW,yesno',
+		  wantdx => '0,Req DX Spots,yesno',
 		  wantdxcq => '0,Show CQ Zone,yesno',
 		  wantdxitu => '0,Show ITU Zone,yesno',
+		  wantecho => '0,Req Echo,yesno',
+		  wantemail => '0,Req Msgs as Email,yesno',
+		  wantft => '0,Want RBN FT4/8,yesno',
 		  wantgtk => '0,Want GTK interface,yesno',
+		  wantlogininfo => '0,Login Info Req,yesno',
+		  wantpc16 => '9,Want Users from node,yesno',
 		  wantpc9x => '0,Want PC9X interface,yesno',
+		  wantpsk => '0,Want RBN PSK,yesno',
 		  wantrbn => '0,Want RBN spots,yesno',
-		  wantft => '0,Want RBN FT4/8,yesno',
-		  wantcw => '0,Want RBN CW,yesno',
+		  wantroutepc19 => '9,Route PC19,yesno',
 		  wantrtty => '0,Want RBN RTTY,yesno',
-		  wantpsk => '0,Want RBN PSK,yesno',
-		  wantbeacon => '0,Want (RBN) Beacon,yesno',
-		  lastoper => '9,Last for/oper,cldatetime',
-		  nothere => '0,Not Here Text',
-		  registered => '9,Registered?,yesno',
-		  prompt => '0,Required Prompt',
-		  version => '1,Version',
-		  build => '1,Build',
-		  believe => '1,Believable nodes,parray',
-		  lastping => '1,Last Ping at,ptimelist',
-		  maxconnect => '1,Max Connections',
-		  startt => '0,Start Time,cldatetime',
-		  connlist => '1,Connections,parraydifft',
+		  wantsendpc16 => '9,Send PC16,yesno',
+		  wanttalk => '0,Req Talk,yesno',
+		  wantusstate => '0,Show US State,yesno',
+		  wantwcy => '0,Req WCY,yesno',
+		  wantwwv => '0,Req WWV,yesno',
+		  wantwx => '0,Req WX,yesno',
+		  width => '0,Preferred Width',
+		  xpert => '0,Expert Status,yesno',
+          wantgrid => '0,Show DX Grid,yesno',
 		 );
 
 #no strict;
@@ -124,6 +133,8 @@ sub AUTOLOAD
        goto &$AUTOLOAD;
 }
 
+my $readonly;
+
 #use strict;
 
 #
@@ -133,17 +144,19 @@ sub init
 {
 	my $mode = shift;
   
-   $json = JSON->new->canonical(1);
+	$json = DXJSON->new->canonical(1);
 	my $fn = "users";
 	$filename = localdata("$fn.v3j");
-	unless (-e $filename || $mode == 2) {
-		LogDbg('DXUser', "New User File version $filename does not exist, running conversion from users.v3 or v2, please wait");
-		system('/spider/perl/convert-users-v3-to-v3j.pl');
-		init(1);
-		export();
-		return;
+	unless (-e $filename || $mode == 2 ) {
+		if (-e localdata("$fn.v3") || -e localdata("$fn.v2")) {
+			LogDbg('DXUser', "New User File version $filename does not exist, running conversion from users.v3 or v2, please wait");
+			system('/spider/perl/convert-users-v3-to-v3j.pl');
+			init(1);
+			export();
+			return;
+		}
 	}
-	if (-e $filename || $mode == 2) {
+	if (-e $filename || $mode) {
 		$lru = LRU->newbase("DXUser", $lrusize);
 		if ($mode) {
 			$dbm = tie (%u, 'DB_File', $filename, O_CREAT|O_RDWR, 0666, $DB_BTREE) or confess "can't open user file: $fn ($!) [rebuild it from user_json?]";
@@ -151,6 +164,8 @@ sub init
 			$dbm = tie (%u, 'DB_File', $filename, O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn ($!) [rebuild it from user_json?]";
 		}
 	}
+	$readonly = !$mode;
+	
 	die "Cannot open $filename ($!)\n" unless $dbm || $mode == 2;
 	return;
 }
@@ -180,6 +195,7 @@ sub process
 
 sub finish
 {
+	dbg('DXUser finished') unless $readonly;
 	$dbm->sync;
 	undef $dbm;
 	untie %u;
@@ -206,7 +222,6 @@ sub new
 #	confess "can't create existing call $call in User\n!" if $u{$call};
 
 	my $self = $pkg->alloc($call);
-	$self->{lastseen} = $main::systime;
 	$self->put;
 	return $self;
 }
@@ -224,14 +239,12 @@ sub get
 	# is it in the LRU cache?
 	my $ref = $lru->get($call);
 	if ($ref && ref $ref eq 'DXUser') {
-		$ref->{lastseen} = $main::systime;
 		return $ref;
 	}
 	
 	# search for it
 	unless ($dbm->get($call, $data)) {
 		eval { $ref = decode($data); };
-		
 		if ($ref) {
 			if (!UNIVERSAL::isa($ref, 'DXUser')) {
 				dbg("DXUser::get: got strange answer from decode of $call". ref $ref. " ignoring");
@@ -246,7 +259,6 @@ sub get
 			}
 			return undef;
 		}
-		$ref->{lastseen} = $main::systime;
 		$lru->put($call, $ref);
 		return $ref;
 	}
@@ -295,43 +307,27 @@ sub put
 	my $call = $self->{call};
 
 	$dbm->del($call);
-	delete $self->{annok} if $self->{annok};
-	delete $self->{dxok} if $self->{dxok};
-
+	delete $self->{annok};
+	delete $self->{dxok};
+	$self->{lastseen} = $main::systime;
 	$lru->put($call, $self);
 	my $ref = $self->encode;
 	$dbm->put($call, $ref);
+	DXChannel::refresh_user($call, $ref);
+	return $ref;
 }
 
 
 # thaw the user
 sub decode
 {
-    my $s = shift;
-    my $ref;
-    eval { $ref = $json->decode($s) };
-    if ($ref && !$@) {
-        return bless $ref, 'DXUser';
-    } else {
-        LogDbg('DXUser', "DXUser::json_decode: on '$s' $@");
-    }
-    return undef;
+	return $json->decode(shift, __PACKAGE__);
 }
 
 # freeze the user
 sub encode
 {
-    my $ref = shift;
-    unbless($ref);
-    my $s;
-	
-	eval {$s = $json->encode($ref) };
-	if ($s && !$@) {
-		bless $ref, 'DXUser';
-		return $s;
-	} else {
-		LogDbg('DXUser', "DXUser::json_encode $ref->{call}, $@");
-	}
+	return $json->encode(shift);
 }
 
 
@@ -356,9 +352,9 @@ sub close
 	my $self = shift;
 	my $startt = shift;
 	my $ip = shift;
-	$self->{lastseen} = $self->{lastin} = $main::systime;
 	# add a record to the connect list
-	my $ref = [$startt || $self->{startt}, $main::systime];
+	$self->{lastin} = $main::systime;
+	my $ref = [$self->{startt} || $startt, $main::systime];
 	push @$ref, $ip if $ip;
 	push @{$self->{connlist}}, $ref;
 	shift @{$self->{connlist}} if @{$self->{connlist}} > $maxconnlist;
@@ -384,147 +380,6 @@ sub fields
 }
 
 
-#
-# export the database to an ascii file
-#
-
-sub export
-{
-	my $name = shift || 'user_json';
-	my $basic_info_only = shift;
-
-	my $fn = $name ne 'user_json' ? $name : "$main::local_data/$name";                       # force use of local
-	
-	# save old ones
-	move "$fn.oooo", "$fn.ooooo" if -e "$fn.oooo";
-	move "$fn.ooo", "$fn.oooo" if -e "$fn.ooo";
-	move "$fn.oo", "$fn.ooo" if -e "$fn.oo";
-	move "$fn.o", "$fn.oo" if -e "$fn.o";
-	move "$fn", "$fn.o" if -e "$fn";
-
-	my $ta = [gettimeofday];
-	my $count = 0;
-	my $err = 0;
-	my $del = 0;
-	my $fh = new IO::File ">$fn" or return "cannot open $fn ($!)";
-	if ($fh) {
-		my $key = 0;
-		my $val = undef;
-		my $action;
-		my $t = scalar localtime;
-		print $fh q{#!/usr/bin/perl
-#
-# The exported userfile for a DXSpider System
-#
-# Input file: $filename
-#       Time: $t
-#
-			
-package main;
-			
-# search local then perl directories
-BEGIN {
-	umask 002;
-				
-	# root of directory tree for this system
-	$root = "/spider"; 
-	$root = $ENV{'DXSPIDER_ROOT'} if $ENV{'DXSPIDER_ROOT'};
-	
-	unshift @INC, "$root/perl";	# this IS the right way round!
-	unshift @INC, "$root/local";
-	
-	# try to detect a lockfile (this isn't atomic but 
-	# should do for now
-	$lockfn = "$root/local_data/cluster.lck";       # lock file name
-	if (-e $lockfn) {
-		open(CLLOCK, "$lockfn") or die "Can't open Lockfile ($lockfn) $!";
-		my $pid = <CLLOCK>;
-		chomp $pid;
-		die "Lockfile ($lockfn) and process $pid exists - cluster must be stopped first\n" if kill 0, $pid;
-		close CLLOCK;
-	}
-}
-
-use SysVar;
-use DXUtil;
-use DXUser;
-use JSON;
-use Time::HiRes qw(gettimeofday tv_interval);
-package DXUser;
-
-our $json = JSON->new->canonical(1);
-
-my $ta = [gettimeofday];
-our $filename = "$main::local_data/users.v3j";
-my $exists = -e $filename ? "OVERWRITING" : "CREATING"; 
-print "perl user_json $exists $filename\n";
-
-del_file();
-init(2);
-%u = ();
-my $count = 0;
-my $err = 0;
-while (<DATA>) {
-	chomp;
-	my @f = split /\t/;
-	my $ref = decode($f[1]);
-	if ($ref) {
-		$ref->put();
-		$count++;
-	} else {
-		print "# Error: $f[0]\t$f[1]\n";
-		$err++
-	}
-}
-DXUser::sync(); DXUser::finish();
-my $diff = _diffms($ta);
-print "There are $count user records and $err errors in $diff mS\n";
-};
-		print $fh "__DATA__\n";
-
-        for ($action = R_FIRST; !$dbm->seq($key, $val, $action); $action = R_NEXT) {
-			if (!is_callsign($key) || $key =~ /^0/) {
-				my $eval = $val;
-				my $ekey = $key;
-				$eval =~ s/([\%\x00-\x1f\x7f-\xff])/sprintf("%%%02X", ord($1))/eg; 
-				$ekey =~ s/([\%\x00-\x1f\x7f-\xff])/sprintf("%%%02X", ord($1))/eg; 
-				LogDbg('DXCommand', "Export Error1: invalid callsign($ekey) => '$eval'");
-				eval {$dbm->del($key)};
-				dbg(carp("Export Error1: delete call $ekey => '$eval' $@")) if $@;
-				++$err;
-				next;
-			}
-			my $ref;
-			eval {$ref = decode($val); };
-			if ($ref) {
-				my $t = $ref->{lastin} || 0;
-				if ($ref->is_user && !$ref->{priv} && $main::systime > $t + $tooold) {
-					unless ($ref->{lat} && $ref->{long} || $ref->{qth} || $ref->{qra}) {
-						eval {$dbm->del($key)};
-						dbg(carp("Export Error2: delete $key => '$val' $@")) if $@;
-						LogDbg('DXCommand', "$ref->{call} deleted, too old");
-						$del++;
-						next;
-					}
-				}
-				# only store users that are reasonably active or have useful information
-				print $fh "$key\t" . encode($ref) . "\n";
-				++$count;
-			} else {
-				LogDbg('DXCommand', "Export Error3: $key\t" . carp($val) ."\n$@");
-				eval {$dbm->del($key)};
-				dbg(carp("Export Error3: $key\t$val\n$@")) if $@;
-				++$err;
-			}
-		} 
-        $fh->close;
-    }
-	my $diff = _diffms($ta);
-	my $s = qq{Exported users to $fn - $count Users $del Deleted $err Errors in $diff mS ('sh/log Export' for details)};
-	LogDbg('command', $s);
-	return $s;
-}
-
 #
 # group handling
 #
@@ -676,7 +531,7 @@ sub wanttalk
 
 sub wantgrid
 {
-	return _want('grid', @_);
+	return _wantnot('grid', @_);
 }
 
 sub wantemail
@@ -711,12 +566,12 @@ sub wantusstate
 
 sub wantdxcq
 {
-	return _want('dxcq', @_);
+	return _wantnot('dxcq', @_);
 }
 
 sub wantdxitu
 {
-	return _want('dxitu', @_);
+	return _wantnot('dxitu', @_);
 }
 
 sub wantgtk
@@ -849,6 +704,294 @@ sub lastping
 	$b->{$call} = shift if @_;
 	return $b->{$call};	
 }
+
+
+#
+# export the database to an ascii file
+#
+
+sub export
+{
+	my $name = shift || 'user_json';
+
+	my $fn = $name ne 'user_json' ? $name : "$main::local_data/$name";                       # force use of local
+	
+	# save old ones
+	copy $fn, "$fn.keep" unless -e "$fn.keep";
+	copy "$fn.ooooo", "$fn.backstop" unless -e "$fn,backstop";
+
+	move "$fn.oooo", "$fn.ooooo" if -e "$fn.oooo";
+	move "$fn.ooo", "$fn.oooo" if -e "$fn.ooo";
+	move "$fn.oo", "$fn.ooo" if -e "$fn.oo";
+	move "$fn.o", "$fn.oo" if -e "$fn.o";
+	move "$fn", "$fn.o" if -e "$fn";
+
+	
+	my $ta = [gettimeofday];
+	my $count = 0;
+	my $err = 0;
+	my $del = 0;
+	my $spurious = 0;
+	my $unlocked = 0;
+	my $old =  0;
+	my $ancient =  0;
+	my $nodes = 0;
+	my $renamed = 0;
+
+	my %del;
+	
+	my $fh = new IO::File ">$fn" or return "cannot open $fn ($!)";
+	if ($fh) {
+		my $key = 0;
+		my $val = undef;
+		my $action;
+		my $t = scalar localtime;
+		print $fh export_preamble();
+		
+
+        for ($action = R_FIRST; !$dbm->seq($key, $val, $action); $action = R_NEXT) {
+			if (!is_callsign($key) || $key =~ /^0/) {
+				my $eval = $val;
+				my $ekey = $key;
+				$eval =~ s/([\%\x00-\x1f\x7f-\xff])/sprintf("%%%02X", ord($1))/eg; 
+				$ekey =~ s/([\%\x00-\x1f\x7f-\xff])/sprintf("%%%02X", ord($1))/eg;
+				LogDbg('DXCommand', "Export Error1: invalid call '$key' => '$val'");
+
+				$del{$key} = $val;
+				++$err;
+				next;
+			}
+			my $ref;
+			eval {$ref = decode($val); };
+			if ($ref) {
+				my $t = $ref->{lastseen} if exists $ref->{lastseen};
+				$t ||= $ref->{lastin} if exists $ref->{lastin};
+				$t ||= $ref->{lastoper} if exists $ref->{lastoper};
+				$t //= 0;
+				
+				if ($ref->is_user) {
+					if (!$ref->{priv} && $main::systime > $t + $tooold) {
+						unless (($ref->{lat} && $ref->{long}) || $ref->{qth} || $ref->{name} || $ref->{qra}) {
+							LogDbg('DXCommand', sprintf("$ref->{call} deleted, empty and too Old at %s", difft($t, ' ')));
+							++$del;
+							++$old;
+							$del{$key} = $val;
+							next;
+						}
+					}
+					if ($main::systime > $t + $veryold) {
+						LogDbg('DXCommand', sprintf("$ref->{call} deleted, POSITIVELY ANCIENT at %s", difft($t, ' ')));
+						++$del;
+						++$ancient;
+						$del{$key} = $val;
+						next;
+					}
+					if (exists $ref->{lockout} && $ref->{lockout} == 1 && exists $ref->{priv} && $ref->{priv} == 1) {
+						LogDbg('DXCommand', "$ref->{call} depriv'd and unlocked");
+						$ref->{lockout} = $ref->{priv} = 0;
+						$ref->put;
+						++$unlocked;
+					}
+					if ($ref->is_node && $main::systime > $t + $veryold) {
+						LogDbg('DXCommand', sprintf("NODE $ref->{call} deleted (%s) old", difft($t, ' ')));
+						++$del;
+						++$nodes;
+						$del{$key} = $val;
+						next;
+					}
+					
+					my $normcall = normalise_call($key);
+					if ($normcall ne $key) {
+						# if the normalised call does not exist, create it from the duff call.
+						my $nref = DXUser::get_current($normcall);
+						unless ($nref) {
+							$ref->{call} = $normcall;
+							$ref->put;
+							LogDbg('DXCommand', "DXProt: spurious call $key normalises to $normcall renaming $key -> $normcall");
+							++$renamed;
+						} 
+						LogDbg('DXCommand', "DXProt: spurious call $key (should be $normcall), removing");
+						$del{$key} = $val;
+						++$spurious;
+						++$del;
+						next;
+					}
+				}
+			} else {
+				LogDbg('DXCommand', "Export Error3: '$key'\t" . carp($val) ."\n$@");
+				$del{$key} = $val;
+				++$err;
+				next;
+			}
+			
+			# only store users that are reasonably active or have useful information
+			print $fh "$key\t" . encode($ref) . "\n";
+			++$count;
+		}
+	} 
+	$fh->close;
+	
+	while (my ($k, $v) = each %del) {
+		eval {$dbm->del($k)};
+		LogDbg('DXCommand', "Error deleting key: $k value: $v error: $@") if $@;
+	}
+
+	my $diff = _diffms($ta);
+	my $s = qq{Exported users to $fn - $count Users,  $del Deleted ($old empty \& too old, $ancient ancient, $nodes nodes, $spurious spurious), $renamed renamed, $unlocked Unlocked, $err Errors in $diff mS ('sh/log Export' for details)};
+	LogDbg('command', $s);
+	return ($s);
+}
+
+sub export_preamble
+{
+	return q{#!/usr/bin/perl
+#
+# The exported userfile for a DXSpider System
+#
+# Input file: $filename
+#       Time: $t
+#
+			
+package main;
+			
+# search local then perl directories
+BEGIN {
+	umask 002;
+				
+	# root of directory tree for this system
+	$root = "/spider"; 
+	$root = $ENV{'DXSPIDER_ROOT'} if $ENV{'DXSPIDER_ROOT'};
+	
+	unshift @INC, "$root/perl";	# this IS the right way round!
+	unshift @INC, "$root/local";
+	
+	# try to detect a lockfile (this isn't atomic but 
+	# should do for now
+	$lockfn = "$root/local_data/cluster.lck";       # lock file name
+	if (-e $lockfn) {
+		open(CLLOCK, "$lockfn") or die "Can't open Lockfile ($lockfn) $!";
+		my $pid = <CLLOCK>;
+		chomp $pid;
+		die "Lockfile ($lockfn) and process $pid exists - cluster must be stopped first\n" if kill 0, $pid;
+		close CLLOCK;
+	}
+}
+
+use SysVar;
+use DXUtil;
+use DXUser;
+use DXChannel;
+use JSON;
+use Time::HiRes qw(gettimeofday tv_interval);
+package DXUser;
+
+our $json = JSON->new->canonical(1);
+
+my $ta = [gettimeofday];
+our $filename = "$main::local_data/users.v3j";
+my $exists = -e $filename ? "OVERWRITING" : "CREATING"; 
+print "perl user_json $exists $filename\n";
+
+del_file();
+init(2);
+%u = ();
+my $count = 0;
+my $err = 0;
+
+while (<DATA>) {
+	chomp;
+	my @f = split /\t/;
+	my $ref = decode($f[1]);
+	if ($ref) {
+		$ref->put();
+		$count++;
+	} else {
+		print "# Error: $f[0]\t$f[1]\n";
+		$err++
+	}
+}
+DXUser::sync(); DXUser::finish();
+my $diff = _diffms($ta);
+print "There are $count user records and $err errors in $diff mS\n";
+
+exit $err ? -1 : 0;
+
+__DATA__
+};
+
+}
+
+sub recover
+{
+	my $name = shift || 'recover_json';
+
+	my $fn = $name ne 'recover_json' ? $name : "$main::local_data/$name";                       # force use of local
+	
+	# save old ones
+	move "$fn.oooo", "$fn.ooooo" if -e "$fn.oooo";
+	move "$fn.ooo", "$fn.oooo" if -e "$fn.ooo";
+	move "$fn.oo", "$fn.ooo" if -e "$fn.oo";
+	move "$fn.o", "$fn.oo" if -e "$fn.o";
+	move "$fn", "$fn.o" if -e "$fn";
+
+	my $ta = [gettimeofday];
+	my $count = 0;
+	my $errs = 0;
+	my $total = 0;
+		
+	my $strings = "strings $filename";
+	my $ifh = new IO::File "$strings |" or return "cannot open input $filename ($!)";
+	my $fh = new IO::File ">$fn" or return "cannot open output $fn ($!)";
+	if ($ifh && $fh) {
+		my $key = 0;
+		my $val = undef;
+		my $action;
+		my $t = scalar localtime;
+		print $fh export_preamble();
+
+		my $call;
+		my $l;
+
+		my $last = '';
+		while (defined ($l = $ifh->getline)) {
+			next unless  $l =~ /^{"call":"[-\d\w\/]+"/;
+			dbg("recover: $l");
+			$l =~ s/[^}]+$//;
+			my $data = $l;
+			if ($data) {
+				my $v;
+				
+				eval{ $v = decode($data); };
+				if ($@) {
+					++$errs;
+					++$total;
+				} else {
+					next if $data eq $last;
+					print $fh  "$v->{call}\t$l\n";
+					++$count;
+					++$total;
+					$last = $l;
+				}
+			}
+		}
+	}
+	$fh->close;
+	$ifh->close;
+
+	my $diff = _diffms($ta);
+	my $s = qq{Recovered users to $fn - $count Users, $errs errors $total possible records read in $diff mS ('sh/log recover' for details)};
+	LogDbg('command', $s);
+	return ($s);
+}
+
+sub END
+{
+	if ($dbm) {
+		print "DXUser Ended\n" unless $readonly;
+		finish();
+	}
+}
+
 1;
 __END__