X-Git-Url: http://dxcluster.net/gitweb/gitweb.cgi?a=blobdiff_plain;f=perl%2FDXDupe.pm;h=012039ee6a614eb8035b98fc27d3b6a87dd01a88;hb=5756741d9682667ae5b0442c4e6f609bd481b6eb;hp=6595ef879c7cdd4446106e2a5183758d62fd2af7;hpb=6fa95fc3ce0732180e59f7d3d1c68b10c887b9b6;p=spider.git diff --git a/perl/DXDupe.pm b/perl/DXDupe.pm index 6595ef87..012039ee 100644 --- a/perl/DXDupe.pm +++ b/perl/DXDupe.pm @@ -17,31 +17,23 @@ use vars qw{$lasttime $dbm %d $default $fn}; $default = 48*24*60*60; $lasttime = 0; -$fn = "$main::data/dupefile"; - -use vars qw($VERSION $BRANCH); -$VERSION = sprintf( "%d.%03d", q$Revision$ =~ /(\d+)\.(\d+)/ ); -$BRANCH = sprintf( "%d.%03d", q$Revision$ =~ /\d+\.\d+\.(\d+)\.(\d+)/ || (0,0)); -$main::build += $VERSION; -$main::branch += $BRANCH; +localdata_mv("dupefile"); +$fn = localdata("dupefile"); sub init { + unlink $fn; $dbm = tie (%d, 'DB_File', $fn); - unless ($dbm) { - eval { untie %d }; - dbg("Dupefile $fn corrupted, removing..."); - unlink $fn; - $dbm = tie (%d, 'DB_File', $fn) or confess "can't open dupe file: $fn ($!)"; - confess "cannot open $fn $!" unless $dbm; - } + confess "cannot open $fn $!" unless $dbm; } sub finish { + dbg("DXDupe finishing"); undef $dbm; untie %d; undef %d; + unlink $fn; } sub check @@ -54,6 +46,7 @@ sub check sub find { + return 0 unless $_[0]; return $d{$_[0]}; } @@ -61,12 +54,19 @@ sub add { my $s = shift; my $t = shift || $main::systime + $default; + return unless $s; + $d{$s} = $t; + dbg("DXDupe::add key: $s time: " . ztime($t)) if isdbg('dxdupe'); } sub del { my $s = shift; + return unless $s; + + my $t = $d{$s}; + dbg("DXDupe::del key: $s time: " . ztime($t)) if isdbg('dxdupe'); delete $d{$s}; } @@ -74,9 +74,11 @@ sub process { # once an hour if ($main::systime - $lasttime >= 3600) { + my @del; while (($k, $v) = each %d) { - delete $d{$k} if $main::systime >= $v; + push @del, $k if $main::systime >= $v; } + del($k) for @del; $lasttime = $main::systime; } } @@ -107,4 +109,12 @@ sub listdups } return @out; } + +sub END +{ + if ($dbm) { + dbg("DXDupe ENDing"); + finish(); + } +} 1;