From: Dirk Koopman <djk@tobit.co.uk>
Date: Tue, 28 Feb 2023 10:48:24 +0000 (+0000)
Subject: remove dupe IPs in create_master_badip_files.pl
X-Git-Url: http://dxcluster.net/gitweb/gitweb.cgi?p=spider.git;a=commitdiff_plain;h=a3070f2678d8be7d44dbf2148cee77cd6a212243

remove dupe IPs in create_master_badip_files.pl
---

diff --git a/Changes b/Changes
index a6815aec..802b0cc9 100644
--- a/Changes
+++ b/Changes
@@ -3,6 +3,8 @@
 2. Add CTY-3308 prefixes.
 3. Fix gen_usdb_data.pl (add missing use DXUtil), update current location of
    official FCC databases.
+4. Remove the duplicate IP addresses across exits and relays from badip lists
+   during generation by create_master_badip_files.pl.
 14Feb23=======================================================================
 1. Attempt to be more M$ Windows compatible. This basically is to do with the
    unfortunate fact that most of the Windows perl cannot do, or simulate 
diff --git a/perl/create_master_badip_files.pl b/perl/create_master_badip_files.pl
index 37e1747a..f7b25be4 100755
--- a/perl/create_master_badip_files.pl
+++ b/perl/create_master_badip_files.pl
@@ -81,24 +81,24 @@ foreach my $e (@{$data->{relays}}) {
 	my $seen = str2time($e->{last_seen});
 	next unless $seen >= $now - $last_seen_window;
 	
-	my @or = clean_addr(@{$e->{or_addresses}}) if exists $e->{or_addresses};
 	my @exit = clean_addr(@{$e->{exit_addresses}}) if exists $e->{exit_addresses} ;
+	my @or = clean_addr(@{$e->{or_addresses}}) if exists $e->{or_addresses};
 	my $ors = join ', ', @or;
 	my $es = join ', ', @exit;
 	dbg "$0: $e->{nickname} $e->{last_seen} relays: [$ors] exits: [$es]" if $debug;
-	for (@or) {
+	for (@exit) {
 		if (is_ipaddr($_)) {
-			print RELAY "$_\n";
-			++$rcount;
+			print EXIT "$_\n";
+			++$ecount;
 		} else {
 			print STDERR "$_\n";
 			++$error;
 		}
 	}
-	for (@exit) {
+	for (@or) {
 		if (is_ipaddr($_)) {
-			print EXIT "$_\n";
-			++$ecount;
+			print RELAY "$_\n";
+			++$rcount;
 		} else {
 			print STDERR "$_\n";
 			++$error;
@@ -117,17 +117,25 @@ unlink "$exitfn.$rand";
 
 exit $error;
 
+my %addr;
+
 sub clean_addr
 {
 	my @out;
 	foreach (@_) {
 		my ($ipv4) = /^((?:\d+\.){3}\d+)/;
 		if ($ipv4) {
+			next if exists $addr{$ipv4};
 			push @out, $ipv4;
+			$addr{$ipv4}++;
 			next;
 		}
 		my ($ipv6) = /^\[([:a-f\d]+)\]/;
-		push @out, $ipv6 if $ipv6;
+		if ($ipv6) {
+			next if exists $addr{$ipv6};
+			push @out, $ipv6;
+			$addr{$ipv6}++;
+		}
 	}
 	return @out;
 }