fix rbn basecall a bit more
[spider.git] / perl / RBN.pm
index c968d4c8d56cc4f6f0c01b4ee5c8567837a01e35..3a906dbe7e0b5b5d816f83d59b85b93097245211 100644 (file)
@@ -75,19 +75,29 @@ our $startup_delay = 5*60;          # don't send anything out until this timer has expir
                                 # this is to allow the feed to "warm up" with duplicates
                                 # so that the "big rush" doesn't happen.
 
-our $minspottime = 30*60;              # the time between respots of a callsign - if a call is
+our $respottime = 30*60;               # the time between respots of a callsign - if a call is
                                 # still being spotted (on the same freq) and it has been
                                 # spotted before, it's spotted again after this time
-                                # until the next minspottime has passed.
+                                # until the next respottime has passed.
+
 
 our $beacontime = 5*60;                        # same as minspottime, but for beacons (and shorter)
 
 our $dwelltime = 10;                   # the amount of time to wait for duplicates before issuing
                                 # a spot to the user (no doubt waiting with bated breath).
 
+our $limbotime = 5*60;                         # if there are fewer than $minqual candidates and $dwelltime
+                                # has expired then allow this spot to live a bit longer. It may
+                                # simply be that it is not in standard spot coverage. (ask G4PIQ
+                                # about this).
+
+our $cachetime = 60*60;                        # The length of time spot data is cached
+
 our $filterdef = $Spot::filterdef; # we use the same filter as the Spot system. Can't think why :-).
 
 my $spots;                                             # the GLOBAL spot cache
+my $qrg;                                               # the GlOBAL (ephemeral) qrg cache (generated on re-read of cache)
+
 
 my %runtime;                                   # how long each channel has been running
 
@@ -96,14 +106,19 @@ our $cache_valid = 4*60;           # The cache file is considered valid if it is not more
 
 our $maxqrgdiff = 10;                  # the maximum
 our $minqual = 2;                              # the minimum quality we will accept for output
+our $maxqual = 9;                              # if there is enough quality, then short circuit any remaining dwelltime.
 
 my $json;
 my $noinrush = 0;                              # override the inrushpreventor if set
 our $maxdeviants = 5;                  # the number of deviant QRGs to record for skimmer records
 
+our %seeme;                                    # the list of users that want to see themselves
+
+
 sub init
 {
        $json = DXJSON->new;
+       $json->canonical(0);
        if (check_cache()) {
                $noinrush = 1;
        } else {
@@ -139,7 +154,7 @@ sub new
        $self->{norawhour} = 0;
        $self->{sort} = 'N';
        $self->{lasttime} = $main::systime;
-       $self->{minspottime} = $minspottime;
+       $self->{respottime} = $respottime;
        $self->{beacontime} = $beacontime;
        $self->{showstats} = 0;
        $self->{pingint} = 0;
@@ -215,7 +230,7 @@ sub normal
        my $self = shift;
        my $line = shift;
        my @ans;
-#      my $spots = $self->{spot};
+       my $dbgrbn = isdbg('rbn');
        
        # remove leading and trailing spaces
        chomp $line;
@@ -244,13 +259,30 @@ sub normal
                return;
        }
 
-       $origin =~ s/\-(?:\d{1,2}\-)?\#$//; # get rid of all the crap we aren't interested in
+       # remove all extraneous crap from the origin - just leave the base callsign
+       my $norigin = basecall($origin);
+       unless ($norigin) {
+               dbg("RBN: ERROR '$origin' is an invalid callsign, dumped");
+               return;
+       }
+       $origin = $norigin;
 
+       # is this callsign in badspotter list?
+       if ($DXProt::badspotter->in($origin) || $DXProt::badnode->in($origin)) {
+               dbg("RBN: ERROR $origin is a bad spotter/node, dumped");
+               return;
+       }
+       
+       # is the qrg valid
+       unless ($qrg =~ /^\d+\.\d{1,3}$/) {
+               dbg("RBN: ERROR qrg $qrg from $origin invalid, dumped");
+               return;
+       }
 
        $sort ||= '';
        $tx ||= '';
        $qra ||= '';
-    dbg qq{RBN:input decode or:$origin qr:$qrg ca:$call mo:$mode s:$s m:$m sp:$spd u:$u sort:$sort t:$t tx:$tx qra:$qra} if isdbg('rbn');
+    dbg qq{RBN:input decode or:$origin qr:$qrg ca:$call mo:$mode s:$s m:$m sp:$spd u:$u sort:$sort t:$t tx:$tx qra:$qra} if $dbgrbn && isdbg('rbn');
 
        ++$self->{noraw};
        ++$self->{noraw10};
@@ -307,6 +339,32 @@ sub normal
                my $nqrg = nearest(1, $qrg * 10);  # normalised to nearest Khz
                my $sp = "$call|$nqrg";           # hopefully the skimmers will be calibrated at least this well!
 
+               # deal with the unix time
+               my ($hh,$mm) = $t =~ /(\d\d)(\d\d)Z$/;
+               my $utz = $hh*3600 + $mm*60 + $main::systime_daystart; # possible issue with late spot from previous day
+               $utz -= 86400 if $utz > $now+3600;                                         # too far ahead, drag it back one day
+
+               #
+               # But before we do anything, if this call is in the seeme hash then just send the spot to them
+               #
+               if (exists $seeme{$call} && (my $scall = $seeme{basecall($call)})) {
+                       my $uchan = DXChannel::get($call);
+                       if ($uchan) {
+                               if ($uchan->is_user) {
+                                       if (isdbg('seeme')) {
+                                               dbg("seeme: $line");
+                                               dbg( qq{seemme:decode or:$origin qr:$qrg ca:$call mo:$mode s:$s m:$m sp:$spd u:$u sort:$sort t:$t tx:$tx qra:$qra});
+                                       }
+                                       my @s =  Spot::prepare($qrg, $call, $utz, sprintf("%-3s %2ddB **SEEME**", $mode, $s), $origin.'-#');
+                                       my $buf = $uchan->format_dx_spot(@s);
+                                       dbg("seeme: result '$buf'") if isdbg('seeme');
+                                       $uchan->local_send('S', $buf) if $scall;
+                               } else {
+                                       LogDbg("RBN Someone is playing silly persons $call is not a user and cannot do 'seeme', ignored and reset");
+                                       delete $seeme{$call};
+                               }
+                       }
+               }
                # find it?
                my $cand = $spots->{$sp};
                unless ($cand) {
@@ -317,7 +375,7 @@ sub normal
                        }
                        if ($cand) {
                                my $diff = $i - $nqrg;
-                               dbg(qq{RBN: QRG Diff using $new (+$diff) for $sp for qrg $qrg}) if (isdbg('rbnqrg') || isdbg('rbn'));
+                               dbg(qq{RBN: QRG Diff using $new (+$diff) for $sp for qrg $qrg}) if (isdbg('rbnqrg') || ($dbgrbn && isdbg('rbn')));
                                $sp = $new;
                        }
                }
@@ -329,7 +387,7 @@ sub normal
                        }
                        if ($cand) {
                                my $diff = $nqrg - $i;
-                               dbg(qq{RBN: QRG Diff using $new (-$diff) for $sp for qrg $qrg}) if (isdbg('rbnqrg') || isdbg('rbn'));
+                               dbg(qq{RBN: QRG Diff using $new (-$diff) for $sp for qrg $qrg}) if (isdbg('rbnqrg') || ($dbgrbn && isdbg('rbn')));
                                $sp = $new;
                        }
                }
@@ -338,12 +396,12 @@ sub normal
                my $respot = 0;
                if ($cand && ref $cand) {
                        if (@$cand <= CData) {
-                               unless ($self->{minspottime} > 0 && $now - $cand->[CTime] >= $self->{minspottime}) {
-                                       dbg("RBN: key: '$sp' call: $call qrg: $qrg DUPE \@ ". atime(int $cand->[CTime])) if isdbg('rbn');
+                               if ($self->{respottime} > 0 && $now - $cand->[CTime] < $self->{respottime}) {
+                                       dbg("RBN: key: '$sp' call: $call qrg: $qrg DUPE \@ ". atime(int $cand->[CTime])) if $dbgrbn && isdbg('rbn');
                                        return;
                                }
                                
-                               dbg("RBN: key: '$sp' RESPOTTING call: $call qrg: $qrg last seen \@ ". atime(int $cand->[CTime])) if isdbg('rbn');
+                               dbg("RBN: key: '$sp' RESPOTTING call: $call qrg: $qrg last seen \@ ". atime(int $cand->[CTime])) if $dbgrbn && isdbg('rbn');
                                $cand->[CTime] = $now;
                                ++$respot;
                        }
@@ -352,23 +410,16 @@ sub normal
                } elsif ($cand) {
                        dbg("RBN: key '$sp' = '$cand' not ref");
                        return;
-               }
-
-               # here we either have an existing spot record buildup on the go, or we need to create the first one
-               unless ($cand) {
+               } else {
+                       # new spot / frequency
                        $spots->{$sp} = $cand = [$now, 0];
-                       dbg("RBN: key: '$sp' call: $call qrg: $qrg NEW" . ($respot ? ' RESPOT' : '')) if isdbg('rbn');
+                       dbg("RBN: key: '$sp' call: $call qrg: $qrg NEW" . ($respot ? ' RESPOT' : '')) if $dbgrbn && isdbg('rbn');
                }
 
                # add me to the display queue unless we are waiting for initial in rush to finish
                return unless $noinrush || $self->{inrushpreventor} < $main::systime;
 
                # build up a new record and store it in the buildup
-               # deal with the unix time
-               my ($hh,$mm) = $t =~ /(\d\d)(\d\d)Z$/;
-               my $utz = $hh*3600 + $mm*60 + $main::systime_daystart; # possible issue with late spot from previous day
-               $utz -= 86400 if $utz > $now+3600;                                         # too far ahead, drag it back one day
-
                # create record and add into the buildup
                my $r = [$origin, nearest(.1, $qrg), $call, $mode, $s, $t, $utz, $respot, $u];
                my @s =  Spot::prepare($r->[RQrg], $r->[RCall], $r->[RUtz], '', $r->[ROrigin]);
@@ -385,12 +436,12 @@ sub normal
 
                ++$self->{queue}->{$sp};# unless @$cand>= CData; # queue the KEY (not the record)
 
-               dbg("RBN: key: '$sp' ADD RECORD call: $call qrg: $qrg origin: $origin") if isdbg('rbn');
+               dbg("RBN: key: '$sp' ADD RECORD call: $call qrg: $qrg origin: $origin respot: $respot") if $dbgrbn && isdbg('rbn');
 
                push @$cand, $r;
 
        } else {
-               dbg "RBN:DATA,$line" if isdbg('rbn');
+               dbg "RBN:DATA,$line" if $dbgrbn && isdbg('rbn');
        }
 }
 
@@ -469,9 +520,9 @@ sub dx_spot
                $comment = sprintf "%-3s %2ddB $quality", $r->[RMode], $r->[RStrength];
                my $s = $r->[RSpotData];                # the prepared spot
                $s->[SComment] = $comment;              # apply new generated comment
-               
+
                ++$zone{$s->[SZone]};           # save the spotter's zone
+
                # save the lowest strength one
                if ($r->[RStrength] < $strength) {
                        $strength = $r->[RStrength];
@@ -484,7 +535,6 @@ sub dx_spot
                        dbg("RBN: FILTERING for $call spot: $s->[SCall] qrg: $s->[SQrg] origin: $s->[SOrigin] dB: $r->[RStrength] com: '$s->[SComment]' want: " . ($want ? 'YES':'NO')) if isdbg 'rbnll';
                        next unless $want;
                        $filtered = $s;
-#                      last;
                }
        }
 
@@ -501,22 +551,8 @@ sub dx_spot
                # alter spot data accordingly
                $saver->[SComment] .= " Z:$z" if $z;
                
-               dbg("RBN: SENDING to $call spot: $saver->[SCall] qrg: $saver->[SQrg] origin: $saver->[SOrigin] $saver->[SComment]") if isdbg 'rbnll';
-               if ($dxchan->{ve7cc}) {
-                       my $call = $saver->[SOrigin];
-                       $saver->[SOrigin] .= '-#';
-                       $buf = VE7CC::dx_spot($dxchan, @$saver);
-                       $saver->[SOrigin] = $call;
-               } else {
-                       my $call = $saver->[SOrigin];
-                       $saver->[SOrigin] = substr($call, 0, 6);
-                       $saver->[SOrigin] .= '-#';
-                       $buf = $dxchan->format_dx_spot(@$saver);
-                       $saver->[SOrigin] = $call;
-               }
-#              $buf =~ s/^DX/RB/;
-               $dxchan->local_send('N', $buf);
-
+               send_final($dxchan, $saver);
+               
                ++$self->{nospot};
                ++$self->{nospot10};
                ++$self->{nospothour};
@@ -526,15 +562,41 @@ sub dx_spot
                        unless ($user->qra && is_qra($user->qra)) {
                                $user->qra($qra);
                                dbg("RBN: update qra on $saver->[SCall] to $qra");
-                               $user->put;
                        }
+                       # update lastseen if nothing else
+                       $user->put;
                }
        }
 }
 
+sub send_final
+{
+       my $dxchan = shift;
+       my $saver = shift;
+       my $call = $dxchan->{call};
+       my $buf;
+       
+       dbg("RBN: SENDING to $call spot: $saver->[SCall] qrg: $saver->[SQrg] origin: $saver->[SOrigin] $saver->[SComment]") if isdbg 'rbnll';
+       if ($dxchan->{ve7cc}) {
+               my $call = $saver->[SOrigin];
+               $saver->[SOrigin] .= '-#';
+               $buf = VE7CC::dx_spot($dxchan, @$saver);
+               $saver->[SOrigin] = $call;
+       } else {
+               my $call = $saver->[SOrigin];
+               $saver->[SOrigin] = substr($call, 0, 6);
+               $saver->[SOrigin] .= '-#';
+               $buf = $dxchan->format_dx_spot(@$saver);
+               $saver->[SOrigin] = $call;
+       }
+       $dxchan->local_send('R', $buf);
+}
+
 # per second
 sub process
 {
+       my $rbnskim = isdbg('rbnskim');
+       
        foreach my $dxchan (DXChannel::get_all()) {
                next unless $dxchan->is_rbn;
 
@@ -547,26 +609,36 @@ sub process
                foreach my $sp (keys %{$dxchan->{queue}}) {
                        my $cand = $spots->{$sp};
                        ++$items;
+                       
                        unless ($cand && $cand->[CTime]) {
                                dbg "RBN Cand $sp " . ($cand ? 'def' : 'undef') . " [CTime] " . ($cand->[CTime] ? 'def' : 'undef') . " dwell $dwelltime";
+                               delete $spots->{$sp};
+                               delete $dxchan->{queue}->{$sp};    # remove
                                next;
-                       } 
-                       if ($now >= $cand->[CTime] + $dwelltime ) {
+                       }
+                       
+                       my $ctime = $cand->[CTime];
+                       my $quality = @$cand - CData;
+                       my $dwellsecs =  $now - $ctime;
+                       if ($quality >= $maxqual || $dwellsecs >= $dwelltime || $dwellsecs >= $limbotime) {
                                # we have a candidate, create qualitee value(s);
                                unless (@$cand > CData) {
                                        dbg "RBN: QUEUE key '$sp' MISSING RECORDS, IGNORED" . dd($cand) if isdbg 'rbnqueue';
+                                       delete $spots->{$sp}; # don't remember it either - this means that a spot HAS to come in with sufficient spotters to be processed.
+                                       delete $dxchan->{queue}->{$sp};
                                        next;
                                }
                                dbg "RBN: QUEUE PROCESSING key: '$sp' $now >= $cand->[CTime]" if isdbg 'rbnqueue'; 
-                               my $quality = @$cand - CData;
                                my $spotters = $quality;
 
-                               # dump it and remove it from the queue if it is of unadequate quality
-                               if ($quality < $minqual) {
-                                       if (isdbg('rbnskim')) {
-                                               my $r = $cand->[CData];
+                               # dump it and remove it from the queue if it is of unadequate quality, but only if it is no longer in Limbo and can be reasonably passed on to its demise
+                               my $r = $cand->[CData];
+                               if ($dwellsecs > $limbotime && $quality < $minqual) {
+                                       if ( $rbnskim && isdbg('rbnskim')) {
+                                               $r = $cand->[CData];
                                                if ($r) {
-                                                       my $s = "RBN: SPOT IGNORED(Q:$quality < Q:$minqual) key: '$sp' = $r->[RCall] on $r->[RQrg] by $r->[ROrigin] \@ $r->[RTime] route: $dxchan->{call}";
+                                                       my $lastin = difft($ctime, $now, 2);
+                                                       my $s = "RBN:SKIM time in Limbo exceeded DUMPED (lastin: $lastin Q:$quality < Q:$minqual) key: '$sp' = $r->[RCall] on $r->[RQrg] by $r->[ROrigin] \@ $r->[RTime] route: $dxchan->{call}";
                                                        dbg($s);
                                                }
                                        }
@@ -575,49 +647,124 @@ sub process
                                        next;
                                }
 
+                               # we have a possible removal from Limbo, check for more than one skimmer and reset the quality if required
+                               # DOES THIS TEST CAUSE RACES?
+                               if (!$r->[Respot] && $quality >= $minqual && $dwellsecs > $dwelltime+1) {
+
+                                       # because we don't need to check for repeats by the same skimmer in the normal case, we do here
+                                       my %seen;
+                                       my @origin;
+                                       foreach my $wr (@$cand) {
+                                               next unless ref $wr;
+                                               push @origin, $wr->[ROrigin];
+                                               if (exists $seen{$wr->[ROrigin]}) {
+                                                       next;
+                                               }
+                                               $seen{$wr->[ROrigin]} = $wr;
+                                       }
+                                       # reset the quality to ignore dupes
+                                       my $oq = $quality;
+                                       $quality = keys %seen;
+                                       if ($quality >= $minqual) {
+                                               if ( $rbnskim && isdbg('rbnskim')) {
+                                                       my $lastin = difft($ctime, $now, 2);
+                                                       my $sk = join ' ', keys %seen;
+                                                       my $or = join ' ', @origin;
+                                                       my $s = "RBN:SKIM promoted from Limbo - key: '$sp' (lastin: $lastin Q now: $quality was $oq skimmers now: $sk";
+                                                       $s .= " was $or" if $or ne $sk;
+                                                       $s .= ')';
+                                                       dbg($s);
+                                               } 
+                                       } elsif ($oq != $quality) {
+                                               if ( $rbnskim && isdbg('rbnskim')) {
+                                                       my $lastin = difft($ctime, $now, 2);
+                                                       my $sk = join ' ', keys %seen;
+                                                       my $or = join ' ', @origin;
+                                                       my $s = "RBN:SKIM quality reset key: '$sp' (lastin: $lastin Q now: $quality was $oq skimmers now: $sk was: $or)";
+                                                       dbg($s);
+                                               }
+                                               # remove the excess
+                                               my @ncand = (@$cand[CTime, CQual], values %seen);
+                                               $spots->{$sp} = \@ncand;
+                                       }
+                               }
+
+                               # we now kick this spot into Limbo 
+                               if ($quality < $minqual) {
+                                       next;
+                               }
+
                                $quality = 9 if $quality > 9;
                                $cand->[CQual] = $quality if $quality > $cand->[CQual];
 
-                               my $r;
-                               my %qrg;
+                               # this scores each candidate according to its skimmer's QRG score (i.e. how often it agrees with its peers)
+                               # what happens is hash of all QRGs in candidates are incremented by that skimmer's reputation for "accuracy"
+                               # or, more exactly, past agreement with the consensus. This score can be from -5 -> +5. 
+                               my %qrg = ();
                                my $skimmer;
                                my $sk;
                                my $band;
-                               my %seen;
+                               my %seen = ();
                                foreach $r (@$cand) {
                                        next unless ref $r;
                                        if (exists $seen{$r->[ROrigin]}) {
-                                               undef $r;
+                                               $r = 0;
                                                next;
                                        }
                                        $seen{$r->[ROrigin]} = 1;
                                        $band ||= int $r->[RQrg] / 1000;
-                                       $sk = "SKIM|$r->[ROrigin]|$band";
+                                       $sk = "SKIM|$r->[ROrigin]|$band"; # thus only once per set of candidates
                                        $skimmer = $spots->{$sk};
                                        unless ($skimmer) {
-                                               $skimmer = $spots->{$sk} = [0+0, 0+0, 0+0, $now, []]; # this stupid incantation is to make sure than there are no JSON nulls!
-                                               dbg("RBN:SKIM new slot $sk " . $json->encode($skimmer)) if isdbg('rbnskim');
+                                               $skimmer = $spots->{$sk} = [1, 0, 0, $now, []]; # this first time, this new skimmer gets the benefit of the doubt on frequency.
+                                               dbg("RBN:SKIM new slot $sk " . $json->encode($skimmer)) if  $rbnskim && isdbg('rbnskim');
                                        }
                                        $qrg{$r->[RQrg]} += ($skimmer->[DScore] || 1);
                                }
                                
-                               # determine the most likely qrg and then set it
+                               # determine the most likely qrg and then set it - NOTE (-)ve votes, generated by the skimmer scoring system above, are ignored
                                my @deviant;
                                my $c = 0;
                                my $mv = 0;
-                               my $qrg;
+                               my $qrg = 0;
                                while (my ($k, $votes) = each %qrg) {
-                                       $qrg = $k, $mv = $votes if $votes >= $mv;
+                                       if ($votes >= $mv) {
+                                               $qrg = $k;
+                                               $mv = $votes;
+                                       }
                                        ++$c;
                                }
-                               # spit out the deviants
+
+                               # Ignore possible spots with 0 QRG score - as determined by the skimmer scoring system above -  as they are likely to be wrong 
+                               unless ($qrg > 0) {
+                                       if ( $rbnskim && isdbg('rbnskim')) {
+                                               my $keys;
+                                               while (my ($k, $v) = (each %qrg)) {
+                                                       $keys .= "$k=>$v, ";
+                                               }
+                                               $keys =~ /,\s*$/;
+                                               my $i = 0;
+                                               foreach $r (@$cand) {
+                                                       next unless $r && ref $r;
+                                                       dbg "RBN:SKIM cand $i QRG likely wrong from '$sp' = $r->[RCall] on $r->[RQrg] by $r->[ROrigin] \@ $r->[RTime] (qrgs: $keys c: $c) route: $dxchan->{call}, ignored";
+                                                       ++$i;
+                                               }
+                                       }
+                                       delete $spots->{$sp}; # get rid
+                                       delete $dxchan->{queue}->{$sp};
+                                       next;
+                               }
+
+                               # detemine and spit out the deviants. Then adjust the scores according to whether it is a deviant or good
+                               # NOTE: deviant nodes can become good (or less bad), and good nodes bad (or less good) on each spot that
+                               # they generate. This is based solely on each skimmer's agreement (or not) with the "consensus" score generated
+                               # above ($qrg). The resultant score + good + bad is stored per band and will be used the next time a spot
+                               # appears on this band from each skimmer.
                                foreach $r (@$cand) {
                                        next unless $r && ref $r;
                                        my $diff = $c > 1 ? nearest(.1, $r->[RQrg] - $qrg) : 0;
                                        $sk = "SKIM|$r->[ROrigin]|$band";
                                        $skimmer = $spots->{$sk};
-                                       $skimmer->[DBad] ||= 0+0; # stop JSON nulls?
-                                       $skimmer->[DEviants] ||= []; # ditto
                                        if ($diff) {
                                                ++$skimmer->[DBad] if $skimmer->[DBad] < $maxdeviants;
                                                --$skimmer->[DGood] if $skimmer->[DGood] > 0;
@@ -630,8 +777,12 @@ sub process
                                                shift @{$skimmer->[DEviants]};
                                        }
                                        $skimmer->[DScore] = $skimmer->[DGood] - $skimmer->[DBad];
-                                       $skimmer->[DScore] ||= 0.2; # minimun score
-                                       dbg("RBN:SKIM key $sp slot $sk $r->[RQrg] - $qrg = $diff " . $json->encode($skimmer)) if isdbg('rbnskim'); 
+                                       if ($rbnskim && isdbg('rbnskim')) {
+                                               my $lastin = difft($skimmer->[DLastin], $now, 2);
+                                               my $difflist = join(', ', @{$skimmer->[DEviants]});
+                                               $difflist = " band qrg diffs: $difflist" if $difflist;
+                                               dbg("RBN:SKIM key $sp slot $sk $r->[RQrg] - $qrg = $diff Skimmer score: $skimmer->[DGood] - $skimmer->[DBad] = $skimmer->[DScore] lastseen:$lastin ago$difflist"); 
+                                       }
                                        $skimmer->[DLastin] = $now;
                                        $r->[RSpotData]->[SQrg] = $qrg if $qrg && $c > 1; # set all the QRGs to the agreed value
                                }
@@ -644,10 +795,11 @@ sub process
                                $squality .= '+' if $r->[Respot];
 
                                if (isdbg('progress')) {
-                                       my $s = "RBN: SPOT key: '$sp' = $r->[RCall] on $r->[RQrg] by $r->[ROrigin] \@ $r->[RTime] $squality route: $dxchan->{call}";
+                                       my $rt = difft($ctime, $now, 2);
+                                       my $s = "RBN: SPOT key: '$sp' = $r->[RCall] on $r->[RQrg] by $r->[ROrigin] \@ $r->[RTime] $squality route: $dxchan->{call} dwell:$rt";
                                        my $td = @deviant;
-                                       $s .= " QRGScore $mv Deviants ($td/$spotters): ";
-                                       $s .= join(', ', sort @deviant) if $td;
+                                       $s .= " QRGScore: $mv Deviants: $td/$spotters";
+                                       $s .= ' (' . join(', ', sort @deviant) . ')' if $td;
                                        dbg($s);
                                }
 
@@ -657,7 +809,6 @@ sub process
                                # clear out the data and make this now just "spotted", but no further action required until respot time
                                dbg "RBN: QUEUE key '$sp' cleared" if isdbg 'rbn';
 
-                               delete $spots->{$sp};
                                delete $dxchan->{queue}->{$sp};
 
                                # calculate new sp (which will be 70% likely the same as the old one)
@@ -666,8 +817,11 @@ sub process
                                my $nqrg = nearest(1, $qrg * 10);  # normalised to nearest Khz
                                my $nsp = "$r->[RCall]|$nqrg";
                                if ($sp ne $nsp) {
-                                       dbg("RBN:SKIM CHANGE KEY sp '$sp' -> '$nsp' for storage") if isdbg('rbnskim');
+                                       dbg("RBN:SKIM CHANGE KEY sp '$sp' -> '$nsp' for storage") if  $rbnskim && isdbg('rbnskim');
+                                       delete $spots->{$sp};
                                        $spots->{$nsp} = [$now, $cand->[CQual]];
+                               } else {
+                                       $spots->{$sp} = [$now, $cand->[CQual]];
                                }
                        }
                        else {
@@ -707,7 +861,7 @@ sub per_10_minute
                next if $k =~ /^O\|/;
                next if $k =~ /^SKIM\|/;
                
-               if ($main::systime - $cand->[CTime] > $minspottime*2) {
+               if ($main::systime - $cand->[CTime] > $cachetime) {
                        delete $spots->{$k};
                        ++$removed;
                }
@@ -744,7 +898,7 @@ sub finish
 sub write_cache
 {
        my $ta = [ gettimeofday ];
-       $json->indent(1);
+       $json->indent(1)->canonical(1) if isdbg 'rbncache';
        my $s = eval {$json->encode($spots)};
        if ($s) {
                my $fh = IO::File->new(">$cachefn") or confess("writing $cachefn $!");
@@ -752,10 +906,12 @@ sub write_cache
                $fh->close;
        } else {
                dbg("RBN:Write_cache error '$@'");
+               return;
        }
-       $json->indent(0);
+       $json->indent(0)->canonical(0);
        my $diff = _diffms($ta);
-       dbg("RBN:WRITE_CACHE time to write: $diff mS");
+       my $size = sprintf('%.3fKB', (length($s) / 1000));
+       dbg("RBN:WRITE_CACHE size: $size time to write: $diff mS");
 }
 
 sub check_cache
@@ -781,7 +937,7 @@ sub check_cache
                                eval {$spots = $json->decode($s)};
                                if ($spots && ref $spots) {     
                                        if (exists $spots->{VERSION} && $spots->{VERSION} == $DATA_VERSION) {
-                                               # now clean out anything that is current
+                                               # now clean out anything that has spot build ups in progress
                                                while (my ($k, $cand) = each %$spots) {
                                                        next if $k eq 'VERSION';
                                                        next if $k =~ /^O\|/;
@@ -807,4 +963,15 @@ sub check_cache
        return undef;
 }
 
+sub add_seeme
+{
+       my $call = shift;
+       $seeme{basecall($call)} = 1;
+}
+
+sub del_seeme
+{
+       my $call = shift;
+       delete $seeme{basecall($call)};
+}
 1;