fix badword, ipv6 address detect, add CTY

[spider.git] / perl / BadWords.pm

diff --git a/perl/BadWords.pm b/perl/BadWords.pm
index db33d7a1c4ebaeb34127c79bb1f5c1463dc35463..312a04082c705fac550f9b0fcb065388cca2e564 100644 (file)
--- a/perl/BadWords.pm
+++ b/perl/BadWords.pm
@@ -3,7 +3,7 @@
 #
 # Copyright (c) 2000 Dirk Koopman
 #
-# $Id$
+#
 #
 
 package BadWords;
@@ -17,55 +17,34 @@ use DXDebug;
 
 use IO::File;
 
-use vars qw($badword @regex);
-
-my $oldfn = "$main::data/badwords";
-my $regex = "$main::data/badw_regex";
-my $bwfn = "$main::data/badword";
-
-# copy issue ones across
-filecopy("$regex.issue", $regex) unless -e $regex;
-filecopy("$bwfn.issue", $bwfn) unless -e $bwfn;
+use vars qw($badword $regexcode);
 
-$badword = new DXHash "badword";
-
-use vars qw($VERSION $BRANCH);
-$VERSION = sprintf( "%d.%03d", q$Revision$ =~ /(\d+)\.(\d+)/ );
-$BRANCH = sprintf( "%d.%03d", q$Revision$ =~ /\d+\.\d+\.(\d+)\.(\d+)/ ) || 0;
-$main::build += $VERSION;
-$main::branch += $BRANCH;
+our $regex;
 
 # load the badwords file
 sub load
 {
+       my $bwfn = localdata("badword");
+       filecopy("$main::data.issue", $bwfn) unless -e $bwfn;
+       
        my @out;
-       my $fh = new IO::File $oldfn;
+
+       $badword = new DXHash "badword";
        
-       if ($fh) {
-               while (<$fh>) {
-                       chomp;
-                       next if /^\s*\#/;
-                       my @list = split " ";
-                       for (@list) {
-                               $badword->add($_);
-                       }
-               }
-               $fh->close;
-               $badword->put;
-               unlink $oldfn;
-       }
        push @out, create_regex(); 
        return @out;
 }
 
 sub create_regex
 {
-       my @out;
-       @regex = ();
+       $regex = localdata("badw_regex");
+       filecopy("$regex.gb.issue", $regex) unless -e $regex;
        
+       my @out;
        my $fh = new IO::File $regex;
        
        if ($fh) {
+               my $s = "sub { my \$str = shift; my \@out; \n";
                while (<$fh>) {
                        chomp;
                        next if /^\s*\#/;
@@ -75,12 +54,18 @@ sub create_regex
                                # and repeated characters in it
                                my $w = uc $_;
                                my @l = split //, $w;
-                               my $e = join '+[\s\W]+', @l;
-                               my $s = eval qq{sub { return \$_[0] =~ /$e+/ ? '$w' : () } };
-                               push @regex, $s unless $@;
-                               dbg("create_regex: $@") if $@;
+                               my $e = join '+[\s\W]*', @l;
+                               $s .= qq{push \@out, \$1 if \$str =~ m|\\b($e+)|;\n};
                        }
                }
+               $s .= "return \@out;\n}";
+               $regexcode = eval $s;
+               dbg($s) if isdbg('badword');
+               if ($@) {
+                       @out = ($@);
+                       dbg($@);
+                       return @out;
+               }
                $fh->close;
        } else {
                my $l = "can't open $regex $!";
@@ -96,15 +81,12 @@ sub check
 {
        my $s = uc shift;
        my @out;
-       
-       for (@regex) {
-               push @out, &$_($s);
-       }
+
+       push @out, &$regexcode($s) if $regexcode;
        
        return @out if @out;
        
-       for (split(/\s+/, $s)) {
-               s/\'?S$//;
+       for (split(/\b/, $s)) {
                push @out, $_ if $badword->in($_);
        }