added first tranche of USDB stuff

diff --git a/perl/USDB.pm b/perl/USDB.pm
new file mode 100644 (file)
index 0000000..3b62fa5
--- /dev/null
+++ b/perl/USDB.pm
@@ -0,0 +1,113 @@
+#
+# Package to handle US Callsign -> City, State translations
+#
+# Copyright (c) 2002 Dirk Koopman G1TLH
+#
+# 
+
+use strict;
+
+use DXVars;
+use DB_File;
+use File::Copy;
+use DXDebug;
+use Compress::Zlib;
+
+use vars qw($VERSION $BRANCH);
+$VERSION = sprintf( "%d.%03d", q$Revision$ =~ /(\d+)\.(\d+)/ );
+$BRANCH = sprintf( "%d.%03d", q$Revision$ =~ /\d+\.\d+\.(\d+)\.(\d+)/  || (0,0));
+$main::build += $VERSION;
+$main::branch += $BRANCH;
+
+use vars qw(%db $present);
+
+my $dbfn = "$main::data/usdb.v1";
+
+sub init
+{
+       end();
+       tie %db, 'DB_File', $dbfn and $present = 1;
+}
+
+sub end
+{
+       return unless $present;
+       untie %db;
+       undef $present;
+}
+
+sub get
+{
+       return () unless $present;
+       my $ctyn = $db{$_[0]};
+       my @s = split /\|/, $db{$ctyn} if $ctyn;
+       return @s;
+}
+
+sub getstate
+{
+       return () unless $present;
+       my @s = get($_[0]);
+       return @s ? $s[1] : undef;
+}
+
+sub getcity
+{
+       return () unless $present;
+       my @s = get($_[0]);
+       return @s ? $s[0] : undef;
+}
+
+#
+# load in / update an existing DB with a standard format (GZIPPED)
+# "raw" file.
+#
+# Note that this removes and overwrites the existing DB file
+# You will need to init again after doing this
+#
+
+sub load
+{
+       # create the new output file
+       my $a = new DB_File::BTREEINFO;
+       $a->{psize} = 4096 * 2;
+       my $s;
+       if ($s = -s $dbfn && $s > 1024 * 1024) {
+               $a->{cachesize} = int(($s / (1024*1024)) / 2) * 1024 * 1024;
+       }
+       my %dbn;
+       if (-e $dbfn ) {
+               syscopy($dbfn, "$dbfn.new") or return "cannot copy $dbfn -> $dbfn.new $!";
+       }
+       
+       tie %dbn, 'DB_File', "$dbfn.new", O_RDWR|O_CREAT, 0664, $a or return "cannot tie $dbfn.new $!";
+       
+       # now write away all the files
+       for (@_) {
+               my $fn = shift;
+               my $f = gzopen($fn, "r") or return "Cannot open $fn $!";
+               while ($f->gzreadline) {
+                       chomp;
+                       my ($call, $city, $state) = split /\|/;
+                       
+                       # lookup the city 
+                       my $s = "$city|$state";
+                       my $ctyn = $dbn{$s};
+                       unless ($ctyn) {
+                               my $no = $dbn{'##'} || 1;
+                               $ctyn = "#$no";
+                               $dbn{$s} = $ctyn;
+                               $dbn{$ctyn} = $s; 
+                               $no++;
+                               $dbn{'##'} = "$no";
+                       }
+                       $dbn{$call} = $ctyn; 
+               }
+               $f->gzclose;
+       }
+       
+       untie %dbn;
+       rename "$dbfn.new", $dbfn;
+}
+
+1;

diff --git a/perl/create_usdb.pl b/perl/create_usdb.pl
new file mode 100755 (executable)
index 0000000..a0da7ff
--- /dev/null
+++ b/perl/create_usdb.pl
@@ -0,0 +1,36 @@
+#!/usr/bin/perl
+#
+# create a USDB file from a standard raw file (which is GZIPPED BTW)
+#
+# This will overwrite and remove any existing usdb file, but it will 
+# copy the old one first and update that.
+#
+
+use strict;
+
+# make sure that modules are searched in the order local then perl
+BEGIN {
+       # root of directory tree for this system
+       use vars qw($root);
+       
+       $root = "/spider"; 
+       $root = $ENV{'DXSPIDER_ROOT'} if $ENV{'DXSPIDER_ROOT'};
+
+       unshift @INC, "$root/local";
+}
+
+use vars qw($VERSION $BRANCH);
+$VERSION = sprintf( "%d.%03d", q$Revision$ =~ /(\d+)\.(\d+)/ );
+$BRANCH = sprintf( "%d.%03d", q$Revision$ =~ /\d+\.\d+\.(\d+)\.(\d+)/  || (0,0));
+$main::build += $VERSION;
+$main::branch += $BRANCH;
+
+use DXVars;
+use USDB;
+
+die "no input (usdbraw?) files specified\n" unless @ARGV;
+
+USDB::load(@ARGV);
+exit(0);
+
+

diff --git a/perl/gen_usdb_data.pl b/perl/gen_usdb_data.pl
new file mode 100755 (executable)
index 0000000..153a560
--- /dev/null
+++ b/perl/gen_usdb_data.pl
@@ -0,0 +1,130 @@
+#!/usr/bin/perl
+#
+# Something to create my subset of the US call book data,
+# in my flat file form, either from the main data base or
+# else the daily updates. 
+#
+# You can get the main database from: 
+#
+#   http://wireless.fcc.gov/uls/data/complete/l_amat.zip
+#
+# The daily data bases are available as a set of seven from here:-
+#
+#   http://wireless.fcc.gov/uls/data/daily/l_am_sat.zip 
+#   http://wireless.fcc.gov/uls/data/daily/l_am_sun.zip 
+#   http://wireless.fcc.gov/uls/data/daily/l_am_mon.zip 
+#   http://wireless.fcc.gov/uls/data/daily/l_am_tue.zip 
+#   http://wireless.fcc.gov/uls/data/daily/l_am_wed.zip 
+#   http://wireless.fcc.gov/uls/data/daily/l_am_thu.zip 
+#   http://wireless.fcc.gov/uls/data/daily/l_am_fri.zip
+# 
+# this program expects one or more zip files containing the call book
+# data as arguments.
+#
+# Copyright (c) 2002 Dirk Koopman G1TLH
+#
+# $Id$
+#
+
+use strict;
+
+# make sure that modules are searched in the order local then perl
+BEGIN {
+       # root of directory tree for this system
+       use vars qw($root);
+       
+       $root = "/spider"; 
+       $root = $ENV{'DXSPIDER_ROOT'} if $ENV{'DXSPIDER_ROOT'};
+
+       unshift @INC, "$root/local";
+}
+
+use vars qw($VERSION $BRANCH);
+$VERSION = sprintf( "%d.%03d", q$Revision$ =~ /(\d+)\.(\d+)/ );
+$BRANCH = sprintf( "%d.%03d", q$Revision$ =~ /\d+\.\d+\.(\d+)\.(\d+)/  || (0,0));
+$main::build += $VERSION;
+$main::branch += $BRANCH;
+
+use DXVars;
+use Archive::Zip qw(:ERROR_CODES);
+use Archive::Zip::MemberRead;
+use IO::File;
+use Compress::Zlib;
+
+my $blksize = 1024 * 1024;
+
+STDOUT->autoflush(1);
+
+my $dbrawfn = "$main::data/usdbraw";
+
+rename "$dbrawfn.oo", "$dbrawfn.ooo";
+rename "$dbrawfn.o", "$dbrawfn.oo";
+rename "$dbrawfn", "$dbrawfn.o";
+my $gzfh = gzopen($dbrawfn, "wb9") or die "Cannot open $dbrawfn $!";
+
+my $ctycount;
+  
+foreach my $argv (@ARGV) {
+       my $zip = new Archive::Zip($argv) or die "Cannot open $argv $!\n";
+       print "Doing $argv\n";
+       handleEN($zip, $argv);
+       handleAM($zip, $argv);
+       handleHS($zip, $argv);
+}
+
+$gzfh->gzclose;
+print "$ctycount Cities found\n";
+
+exit(0);
+
+sub handleEN
+{
+       my ($zip, $argv) = @_;
+       my $mname = "EN.dat";
+       my $ofn = "$main::data/$mname";
+       print "  Handling EN records, unzipping";
+       if ($zip->extractMember($mname, $ofn) == AZ_OK) {
+               my $fh = new IO::File "$ofn" or die "Cannot open $ofn $!";
+               if ($fh) {
+                       
+                       print ", storing";
+                       
+                       my $count;
+                       my $buf;
+                       
+                       while (my $l = $fh->getline) {
+                               $l =~ s/[\r\n]+$//;
+                               my ($rt,$usi,$ulsfn,$ebfno,$call,$type,$lid,$name,$first,$middle,$last,$suffix,
+                                       $phone,$fax,$email,$street,$city,$state,$zip,$pobox,$attl,$sgin,$frn) = split /\|/, $l;
+                               
+                               my $rec = uc join '|', $call,$city,$state if $city && $state;
+                               $buf .= "$rec\n";
+                               if (length $buf > $blksize) {
+                                       $gzfh->gzwrite($buf);
+                                       undef $buf;
+                               }
+                               my $c = uc "$city|$state";
+                               $count++;
+                       }
+                       if (length $buf > $blksize) {
+                               $gzfh->gzwrite($buf);
+                       }
+                       print ", $count records\n";
+                       $fh->close;
+               }
+               unlink $ofn;
+       } else {
+               print "EN missing in $argv\n";
+               return;
+       }
+}
+
+sub handleAM
+{
+
+}
+
+sub handleHS
+{
+
+}