X-Git-Url: http://dxcluster.net/gitweb/gitweb.cgi?a=blobdiff_plain;f=perl%2FAnnTalk.pm;h=71bd272f2736dc6e28642035f92a86311607260c;hb=766014807e2521cc60a219c6eda5b49d13ffc70f;hp=0090eb04824581d567903654a87fb90a87b005bb;hpb=d2580480d24be5e8f1beb4aeeb40980db0dedde0;p=spider.git

diff --git a/perl/AnnTalk.pm b/perl/AnnTalk.pm
index 0090eb04..71bd272f 100644
--- a/perl/AnnTalk.pm
+++ b/perl/AnnTalk.pm
@@ -3,22 +3,25 @@
 #
 # Copyright (c) 2000 Dirk Koopman
 #
-# $Id$
+#
 #
 
 package AnnTalk;
 
 use strict;
 
+use DXVars;
 use DXUtil;
 use DXDebug;
 use DXDupe;
-use DXVars;
+use DXLog;
+use DXLogPrint;
+use Time::HiRes qw(gettimeofday tv_interval);
 
 use vars qw(%dup $duplth $dupage $filterdef);
 
-$duplth = 60;					# the length of text to use in the deduping
-$dupage = 5*24*3600;			# the length of time to hold spot dups
+$duplth = 30;					# the length of text to use in the deduping
+$dupage = 18*3600;				# the length of time to hold ann dups
 $filterdef = bless ([
 			  # tag, sort, field, priv, special parser 
 			  ['by', 'c', 0],
@@ -36,7 +39,26 @@ $filterdef = bless ([
 			  ['origin_zone', 'nz', 12],
 			  ['by_state', 'nz', 13],
 			  ['origin_state', 'nz', 14],
-			 ], 'Filter::Cmd');
+		   ], 'Filter::Cmd');
+
+our $maxcache = 130;
+our @anncache;
+
+sub init
+{
+	my $t0 = [gettimeofday];
+	dbg("AnnTalk: loading up to $maxcache announcements into cache");
+	@anncache = DXLog::search(0, $maxcache, $main::systime, 'ann');
+	shift @anncache while @anncache > $maxcache;
+	my $l = @anncache;
+	dbg("AnnTalk: loaded last $l announcements into cache in " . _diffms($t0) . "mS");
+}
+
+sub add_anncache
+{
+	push @anncache, [ $main::systime, @_ ];
+	shift @anncache while @anncache > $maxcache;
+}
 
 # enter the spot for dup checking and return true if it is already a dup
 sub dup
@@ -47,11 +69,10 @@ sub dup
 	chomp $text;
 	unpad($text);
 	$text =~ s/\%([0-9A-F][0-9A-F])/chr(hex($1))/eg;
-	$text = Encode::encode("iso-8859-1", $text) if $main::can_encode && Encode::is_utf8($text, 1);
-	$text = pack("C*", map {$_ & 127} unpack("C*", $text));
+#	$text = Encode::encode("iso-8859-1", $text) if $main::can_encode && Encode::is_utf8($text, 1);
 	$text =~ s/[^\#a-zA-Z0-9]//g;
 	$text = substr($text, 0, $duplth) if length $text > $duplth; 
-	my $dupkey = "A$to|\L$text";
+	my $dupkey = "A$call|$to|\L$text";
 	return DXDupe::check($dupkey, $t);
 }
 
@@ -67,6 +88,7 @@ sub is_talk_candidate
 {
 	my ($from, $text) = @_;
 	my $call;
+
 	($call) = $text =~ /^\s*(?:[Xx]|[Tt][Oo]?:?)\s+([\w-]+)/;
 	($call) = $text =~ /^\s*>\s*([\w-]+)\b/ unless $call;
 	($call) = $text =~ /^\s*([\w-]+):?\b/ unless $call;