X-Git-Url: http://dxcluster.net/gitweb/gitweb.cgi?a=blobdiff_plain;f=perl%2FDXUser.pm;h=d2082d68954f0fa48923564dbf3755a8fabb6d45;hb=974aca6b8ccf37405098dd3c465a938da5f85eac;hp=3497fa8af71e4eb41b3d8bba230dc26e5449ff86;hpb=5e704493de7958adcef0e839c8ed6de599bc21f3;p=spider.git

diff --git a/perl/DXUser.pm b/perl/DXUser.pm
index 3497fa8a..d2082d68 100644
--- a/perl/DXUser.pm
+++ b/perl/DXUser.pm
@@ -13,14 +13,17 @@ use DB_File;
 use Data::Dumper;
 use Fcntl;
 use IO::File;
-use DXDebug;
 use DXUtil;
 use LRU;
 use File::Copy;
+use JSON;
+use DXDebug;
+use Data::Structure::Util qw(unbless);
+	
 
 use strict;
 
-use vars qw(%u $dbm $filename %valid $lastoperinterval $lasttime $lru $lrusize $tooold $v3 $noips);
+use vars qw(%u $dbm $filename %valid $lastoperinterval $lasttime $lru $lrusize $tooold $v3 $v4);
 
 %u = ();
 $dbm = undef;
@@ -30,7 +33,10 @@ $lasttime = 0;
 $lrusize = 2000;
 $tooold = 86400 * 365;		# this marks an old user who hasn't given enough info to be useful
 $v3 = 0;
-$noips = 4;
+$v4 = 0;
+my $json;
+
+our $maxconnlist = 3;			# remember this many connection time (duration) [start, end] pairs
 
 # hash of valid elements and a simple prompt
 %valid = (
@@ -92,7 +98,8 @@ $noips = 4;
 		  believe => '1,Believable nodes,parray',
 		  lastping => '1,Last Ping at,ptimelist',
 		  maxconnect => '1,Max Connections',
-		  ip => '1,IP address',
+		  startt => '0,Start Time,cldatetime',
+		  connlist => '1,Connections,parraydifft',
 		 );
 
 #no strict;
@@ -123,30 +130,36 @@ sub init
 	my $ufn;
 	my $convert;
 	
-	eval {
-		require Storable;
-	};
-
 	my $fn = "users";
-	
-	if ($@) {
-		$ufn = localdata("users.v2");
-		$v3 = $convert = 0;
-		dbg("the module Storable appears to be missing!!");
-		dbg("trying to continue in compatibility mode (this may fail)");
-		dbg("please install Storable from CPAN as soon as possible");
-	} else {
-		import Storable qw(nfreeze thaw);
 
-		$ufn = localdata("users.v3");
-		$v3 = 1;
-		$convert++ if -e localdata("users.v2") && !-e $ufn;
+	if ($mode == 4 || -e localdata("users.v4")) {
+		$ufn = localdata("users.v4");
+		$v4 = 1;
+		$json = JSON->new();
+		$json->canonical(1);
+	} else {
+		eval {
+			require Storable;
+		};
+		if ($@) {
+			$ufn = localdata("users.v2");
+			$v3 = $convert = 0;
+			dbg("the module Storable appears to be missing!!");
+			dbg("trying to continue in compatibility mode (this may fail)");
+			dbg("please install Storable from CPAN as soon as possible");
+		}
+		else {
+			import Storable qw(nfreeze thaw);
+			$ufn = localdata("users.v3");
+			$v3 = 1;
+			$convert++ if -e localdata("users.v2") && !-e $ufn;
+		}
 	}
 	
 	if ($mode) {
-		$dbm = tie (%u, 'DB_File', $ufn, O_CREAT|O_RDWR, 0666, $DB_BTREE) or confess "can't open user file: $fn ($!) [rebuild it from user_asc?]";
+		$dbm = tie (%u, 'DB_File', $ufn, O_CREAT|O_RDWR, 0666, $DB_BTREE) or confess "can't open user file: $ufn ($!) [rebuild it from user_asc?]";
 	} else {
-		$dbm = tie (%u, 'DB_File', $ufn, O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn ($!) [rebuild it from user_asc?]";
+		$dbm = tie (%u, 'DB_File', $ufn, O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $ufn ($!) [rebuild it from user_asc?]";
 	}
 
 	die "Cannot open $ufn ($!)\n" unless $dbm;
@@ -158,7 +171,7 @@ sub init
 		my ($key, $val, $action, $count, $err) = ('','',0,0,0);
 		
 		my %oldu;
-		dbg("Converting the User File to V3 ");
+		dbg("Converting the User File to V$convert ");
 		dbg("This will take a while, I suggest you go and have cup of strong tea");
 		my $odbm = tie (%oldu, 'DB_File', localdata("users.v2"), O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn.v2 ($!) [rebuild it from user_asc?]";
         for ($action = R_FIRST; !$odbm->seq($key, $val, $action); $action = R_NEXT) {
@@ -185,8 +198,14 @@ sub init
 sub del_file
 {
 	# with extreme prejudice
-	unlink "$main::data/users.v3";
-	unlink "$main::local_data/users.v3";
+	if ($v3) {
+		unlink "$main::data/users.v3";
+		unlink "$main::local_data/users.v3";
+	}
+	if ($v4) {
+		unlink "$main::data/users.v4";
+		unlink "$main::local_data/users.v4";
+	}
 }
 
 #
@@ -326,6 +345,7 @@ sub put
 # freeze the user
 sub encode
 {
+	goto &json_encode if $v4;
 	goto &asc_encode unless $v3;
 	my $self = shift;
 	return nfreeze($self);
@@ -334,6 +354,7 @@ sub encode
 # thaw the user
 sub decode
 {
+	goto &json_decode if $v4;
 	goto &asc_decode unless $v3;
 	my $ref;
 	$ref = thaw(shift);
@@ -378,6 +399,28 @@ sub asc_decode
 	return $ref;
 }
 
+sub json_decode
+{
+	my $s = shift;
+    my $ref;
+	eval { $ref = $json->decode($s) };
+	if ($ref && !$@) {
+        return bless $ref, 'DXUser';
+	} else {
+		LogDbg('err', "DXUser::json_decode: on '$s' $@");
+	}
+	return undef;
+}
+
+sub json_encode
+{
+	my $ref = shift;
+	unbless($ref);
+    my $s = $json->encode($ref);
+	bless $ref, 'DXUser';
+	return $s;
+}
+	
 #
 # del - delete a user
 #
@@ -397,7 +440,14 @@ sub del
 sub close
 {
 	my $self = shift;
-	$self->{lastin} = time;
+	my $startt = shift;
+	my $ip = shift;
+	$self->{lastin} = $main::systime;
+	# add a record to the connect list
+	my $ref = [$startt || $self->{startt}, $main::systime];
+	push @$ref, $ip if $ip;
+	push @{$self->{connlist}}, $ref;
+	shift @{$self->{connlist}} if @{$self->{connlist}} > $maxconnlist;
 	$self->put();
 }
 
@@ -555,6 +605,147 @@ print "There are $count user records and $err errors\n";
 	return $s;
 }
 
+sub export_json
+{
+	my $name = shift || 'user_json';
+	my $basic_info_only = shift;
+
+	my $fn = $name ne 'user_json' ? $name : "$main::local_data/$name";                       # force use of local
+	
+	# save old ones
+	move "$fn.oooo", "$fn.ooooo" if -e "$fn.oooo";
+	move "$fn.ooo", "$fn.oooo" if -e "$fn.ooo";
+	move "$fn.oo", "$fn.ooo" if -e "$fn.oo";
+	move "$fn.o", "$fn.oo" if -e "$fn.o";
+	move "$fn", "$fn.o" if -e "$fn";
+
+	my $json = JSON->new;
+	$json->canonical(1);
+	$json->allow_blessed(1);
+	
+	my $count = 0;
+	my $err = 0;
+	my $del = 0;
+	my $fh = new IO::File ">$fn" or return "cannot open $fn ($!)";
+	if ($fh) {
+		my $key = 0;
+		my $val = undef;
+		my $action;
+		my $t = scalar localtime;
+		print $fh q{#!/usr/bin/perl
+#
+# The exported userfile for a DXSpider System
+#
+# Input file: $filename
+#       Time: $t
+#
+			
+package main;
+			
+# search local then perl directories
+BEGIN {
+	umask 002;
+				
+	# root of directory tree for this system
+	$root = "/spider"; 
+	$root = $ENV{'DXSPIDER_ROOT'} if $ENV{'DXSPIDER_ROOT'};
+	
+	unshift @INC, "$root/perl";	# this IS the right way round!
+	unshift @INC, "$root/local";
+	
+	# try to detect a lockfile (this isn't atomic but 
+	# should do for now
+	$lockfn = "$root/local_data/cluster.lck";       # lock file name
+	if (-e $lockfn) {
+		open(CLLOCK, "$lockfn") or die "Can't open Lockfile ($lockfn) $!";
+		my $pid = <CLLOCK>;
+		chomp $pid;
+		die "Lockfile ($lockfn) and process $pid exists - cluster must be stopped first\n" if kill 0, $pid;
+		close CLLOCK;
+	}
+}
+
+use SysVar;
+use DXUser;
+
+if (@ARGV) {
+	$main::userfn = shift @ARGV;
+	print "user filename now $userfn\n";
+}
+
+package DXUser;
+
+use JSON;
+my $json = JSON->new;
+
+del_file();
+init(4);
+%u = ();
+my $count = 0;
+my $err = 0;
+while (<DATA>) {
+	chomp;
+	my @f = split /\t/;
+	my $ref;
+    eval { $ref = $json->decode($f[1]); };
+	if ($ref && !$@) {
+        $ref = bless $ref, 'DXUser';
+		$ref->put();
+		$count++;
+        DXUser::sync() unless $count % 10000;
+	} else {
+		print "# Error: $f[0]\t$f[1]\n";
+		$err++
+	}
+}
+DXUser::sync(); DXUser::finish();
+print "There are $count user records and $err errors\n";
+};
+		print $fh "__DATA__\n";
+
+        for ($action = R_FIRST; !$dbm->seq($key, $val, $action); $action = R_NEXT) {
+			if (!is_callsign($key) || $key =~ /^0/) {
+				my $eval = $val;
+				my $ekey = $key;
+				$eval =~ s/([\%\x00-\x1f\x7f-\xff])/sprintf("%%%02X", ord($1))/eg; 
+				$ekey =~ s/([\%\x00-\x1f\x7f-\xff])/sprintf("%%%02X", ord($1))/eg; 
+				LogDbg('DXCommand', "Export Error1: $ekey\t$eval");
+				eval {$dbm->del($key)};
+				dbg(carp("Export Error1: $ekey\t$eval\n$@")) if $@;
+				++$err;
+				next;
+			}
+			my $ref;
+			eval {$ref = decode($val); };
+			if ($ref) {
+				my $t = $ref->{lastin} || 0;
+				if ($ref->is_user && !$ref->{priv} && $main::systime > $t + $tooold) {
+					unless ($ref->{lat} && $ref->{long} || $ref->{qth} || $ref->{qra}) {
+						eval {$dbm->del($key)};
+						dbg(carp("Export Error2: $key\t$val\n$@")) if $@;
+						LogDbg('DXCommand', "$ref->{call} deleted, too old");
+						$del++;
+						next;
+					}
+				}
+				# only store users that are reasonably active or have useful information
+				unbless($ref);
+				print $fh "$key\t" . $json->encode($ref) . "\n";
+				++$count;
+			} else {
+				LogDbg('DXCommand', "Export Error3: $key\t" . carp($val) ."\n$@");
+				eval {$dbm->del($key)};
+				dbg(carp("Export Error3: $key\t$val\n$@")) if $@;
+				++$err;
+			}
+		} 
+        $fh->close;
+    }
+	my $s = qq{Exported users to $fn - $count Users $del Deleted $err Errors ('sh/log Export' for details)};
+	LogDbg('command', $s);
+	return $s;
+}
+
 #
 # group handling
 #