WIP spot deduping

[spider.git] / perl / Spot.pm

diff --git a/perl/Spot.pm b/perl/Spot.pm
index 792aa2e79e122c0ea8de7d777881d2994c0bbcee..3012c9e968f4c36b958763b78baa15e6d511b4d5 100644 (file)
--- a/perl/Spot.pm
+++ b/perl/Spot.pm
@@ -33,7 +33,7 @@ $maxspots = 100;                                      # maximum spots to return
 $defaultspots = 10;                            # normal number of spots to return
 $maxdays = 100;                                # normal maximum no of days to go back
 $dirprefix = "spots";
-$duplth = 20;                                  # the length of text to use in the deduping
+$duplth = 15;                                  # the length of text to use in the deduping
 $dupage = 1*3600;               # the length of time to hold spot dups
 $maxcalllth = 12;                               # the max length of call to take into account for dupes
 $filterdef = bless ([
@@ -74,8 +74,8 @@ our $spotcachedays = 2;                       # default 2 days worth
 our $minselfspotqrg = 1240000; # minimum freq above which self spotting is allowed
 
 our $readback = $main::is_win ? 0 : 1; # don't read spot files backwards if it's windows
-our $qrggranularity = 100000;  # normalise the qrg to this number of hz (default: 100khz), so tough luck if you have a fumble fingers moment
-our $timegranularity = 600;            # ditto to the nearest second 
+our $qrggranularity = 1000;    # normalise the qrg to this number of hz (default: 100khz), so tough luck if you have a fumble fingers moment
+our $timegranularity = 600;            # ditto to the nearest 100 seconds 
 our $oldstyle = 0;                             # revert to traditional dupe key format
 
 
@@ -193,7 +193,6 @@ sub init
        if ($spotcachedays > 0) {
                my $t0 = [gettimeofday];
                $spotcachedays = 2 if $spotcachedays < 2;
-               dbg "Spot::init - reading in $spotcachedays days of spots into cache"; 
                for (my $i = 0; $i < $spotcachedays; ++$i) {
                        my $now = $today->sub($i);
                        my $fh = $fp->open($now);
@@ -480,46 +479,81 @@ sub formatl
 # enter the spot for dup checking and return true if it is already a dup
 sub dup
 {
-       my ($freq, $call, $d, $text, $by, $node, $just_find) = @_; 
+       my ($freq, $call, $d, $text, $by, $node, $just_find) = @_;
+
+       dbg("Spot::dup: freq=$freq call=$call d=$d text='$text' by=$by node=$node" . ($just_find ? " jf=$just_find" : "")) if isdbg('spotdup');
 
        # dump if too old
        return 2 if $d < $main::systime - $dupage;
 
-       my $nd = nearest_floor($d, $timegranularity);
-
        # turn the time into minutes (should be already but...)
        $d = int ($d / 60);
        $d *= 60;
 
+       my $nd = nearest($timegranularity, $d);
+
        # remove SSID or area
        $by =~ s|[-/]\d+$||;
        
 #      $freq = sprintf "%.1f", $freq;       # normalise frequency
        $freq = int $freq;       # normalise frequency
-       my $qrg = nearest_floor($freq, $qrggranularity); # to the nearest however many hz
+
+       my $qrg = nearest($qrggranularity, $freq); # to the nearest however many hz
+       
        $call = substr($call, 0, $maxcalllth) if length $call > $maxcalllth;
 
+       my $dtext ;
+       
+       my $l = length $text;
+       $dtext = qq{original:'$text'($l)} if isdbg('spottext');
+
        chomp $text;
+       
        $text =~ s/\%([0-9A-F][0-9A-F])/chr(hex($1))/eg;
        $text = uc unpad($text);
+
+       $l = length $text;
+       $dtext .= qq{->afterhex: '$text'($l)} if isdbg('spottext');
+       my @dubious;
+       if (isdbg('spottext')) {
+               (@dubious) = $text =~ /([?\x00-\x08\x0a-\x1F\x7B-\xFF]+)+/;
+               $dtext .= sprintf q{DUBIOUS '%s'}, join '', @dubious if @dubious;
+       }
+
        my $otext = $text;
 #      $text = Encode::encode("iso-8859-1", $text) if $main::can_encode && Encode::is_utf8($text, 1);
        $text =~ s/^\+\w+\s*//;                 # remove leading LoTW callsign
-       $text =~ s/\s{2,}[\dA-Z]?[A-Z]\d?$// if length $text > 24;
+       $text =~ s/\s{2,}[\dA-Z]?[A-Z]\d?$//g if length $text > 24;
+       $text =~ s/\x09+//g;
        $text =~ s/[\W\x00-\x2F\x7B-\xFF]//g; # tautology, just to make quite sure!
-       $text = substr($text, 0, $duplth) if length $text > $duplth; 
-       my $ldupkey = $oldstyle ? "X|$call|$by|$node|$freq|$d|$text" : "X|$call|$by|$qrg|$nd|$text";
-
-       my $t = DXDupe::find($ldupkey);
-       return 1 if $t && $t - $main::systime > 0;
+       $text = substr($text, 0, $duplth) if length $text > $duplth;
+
+       $l = length $text;
+       $dtext .= qq{->final:'$text'($l)} if isdbg('spottext');
+               
+       my $ldupkey = $oldstyle ? "X|$call|$by|$node|$freq|$d|$text" : "X|$call|$by|$node|$qrg|$nd|$text";
+
+       my $t = 0;
+       $t = DXDupe::find($ldupkey);
+       dbg("Spot::dup ldupkey $ldupkey t '$t'") if isdbg('spotdup');
+       $dtext .= ' DUPE' if $t;
+       dbg("text transforms: $dtext") if length $text && isdbg('spottext');
+       return 1 if $t > 0;     
        
        DXDupe::add($ldupkey, $main::systime+$dupage) unless $just_find;
+
        $otext = substr($otext, 0, $duplth) if length $otext > $duplth; 
        $otext =~ s/\s+$//;
        if (length $otext && $otext ne $text) {
                $ldupkey = $oldstyle ? "X|$freq|$call|$by|$otext" : "X|$qrg|$call|$by|$otext";
                $t = DXDupe::find($ldupkey);
-               return 1 if $t && $t - $main::systime > 0;
+               dbg("Spot::dup (OTEXT) ldupkey $ldupkey t '$t'") if isdbg('spotdup');
+               if (isdbg('spottext')) {
+                       $dtext .= sprintf q{DUBIOUS '%s'}, join '', @dubious if @dubious;
+                       $dtext .= ' DUPE (OTEXT)' if $t;
+                       dbg("text transforms: $dtext") if length $text;
+               }
+               return 1 if $t > 0;
                DXDupe::add($ldupkey, $main::systime+$dupage) unless $just_find;
        }
        return undef;