take only alphanum chars for dupe checking

[spider.git] / perl / AnnTalk.pm

diff --git a/perl/AnnTalk.pm b/perl/AnnTalk.pm
index 987b885d05bb06403559192714c947cc8ef4ac27..84aa55ed8b4158c92536c25178baf8d170125897 100644 (file)
--- a/perl/AnnTalk.pm
+++ b/perl/AnnTalk.pm
@@ -26,6 +26,7 @@ sub dup
 
        chomp $text;
        unpad($text);
+       $text =~ s/[^ a-zA-Z0-9]//g;
        $text = substr($text, 0, $duplth) if length $text > $duplth; 
        my $dupkey = "A$to|$text";
        return DXDupe::check($dupkey, $main::systime + $dupage);