fix badword, ipv6 address detect, add CTY

[spider.git] / perl / BadWords.pm

diff --git a/perl/BadWords.pm b/perl/BadWords.pm
index ff9dd04c8b4e3b70a634828862420fcac1ccaa8a..312a04082c705fac550f9b0fcb065388cca2e564 100644 (file)
--- a/perl/BadWords.pm
+++ b/perl/BadWords.pm
@@ -3,7 +3,7 @@
 #
 # Copyright (c) 2000 Dirk Koopman
 #
-# $Id$
+#
 #
 
 package BadWords;
@@ -12,43 +12,85 @@ use strict;
 
 use DXUtil;
 use DXVars;
+use DXHash;
+use DXDebug;
+
 use IO::File;
 
-use vars qw(%badwords $fn);
+use vars qw($badword $regexcode);
 
-$fn = "$main::data/badwords";
-%badwords = ();
+our $regex;
 
 # load the badwords file
 sub load
 {
+       my $bwfn = localdata("badword");
+       filecopy("$main::data.issue", $bwfn) unless -e $bwfn;
+       
+       my @out;
+
+       $badword = new DXHash "badword";
+       
+       push @out, create_regex(); 
+       return @out;
+}
+
+sub create_regex
+{
+       $regex = localdata("badw_regex");
+       filecopy("$regex.gb.issue", $regex) unless -e $regex;
+       
        my @out;
-       return unless -e $fn;
-       my $fh = new IO::File $fn;
+       my $fh = new IO::File $regex;
        
        if ($fh) {
-               %badwords = ();
+               my $s = "sub { my \$str = shift; my \@out; \n";
                while (<$fh>) {
                        chomp;
                        next if /^\s*\#/;
                        my @list = split " ";
                        for (@list) {
-                               $badwords{lc $_}++;
+                               # create a closure for each word so that it matches stuff with spaces/punctuation
+                               # and repeated characters in it
+                               my $w = uc $_;
+                               my @l = split //, $w;
+                               my $e = join '+[\s\W]*', @l;
+                               $s .= qq{push \@out, \$1 if \$str =~ m|\\b($e+)|;\n};
                        }
                }
+               $s .= "return \@out;\n}";
+               $regexcode = eval $s;
+               dbg($s) if isdbg('badword');
+               if ($@) {
+                       @out = ($@);
+                       dbg($@);
+                       return @out;
+               }
                $fh->close;
        } else {
-               my $l = "can't open $fn $!";
-               dbg('err', $l);
+               my $l = "can't open $regex $!";
+               dbg($l);
                push @out, $l;
        }
+       
        return @out;
 }
 
 # check the text against the badwords list
 sub check
 {
-       return grep { $badwords{$_} } split(/\b/, lc shift);
+       my $s = uc shift;
+       my @out;
+
+       push @out, &$regexcode($s) if $regexcode;
+       
+       return @out if @out;
+       
+       for (split(/\b/, $s)) {
+               push @out, $_ if $badword->in($_);
+       }
+
+       return @out;
 }
 
 1;