From 29e86370c5f331ae3d2c6f85e7001a7d2e758137 Mon Sep 17 00:00:00 2001 From: Dirk Koopman Date: Wed, 8 Jul 2020 23:01:00 +0100 Subject: [PATCH] The last revision before merge back to mojo? See Changes file for details --- Changes | 13 ++ UPGRADE.mojo | 25 +++- cmd/Aliases | 303 +++++++++++++++++++++++--------------------- cmd/Commands_en.hlp | 172 +++++++++++++++++++++++++ cmd/set/wantrbn.pl | 102 ++++++++++++++- perl/DXUser.pm | 2 +- perl/DXUtil.pm | 2 +- perl/Messages | 2 + perl/RBN.pm | 122 ++++++++++++++++-- perl/cluster.pl | 4 + 10 files changed, 577 insertions(+), 170 deletions(-) diff --git a/Changes b/Changes index 3627a929..3ff74402 100644 --- a/Changes +++ b/Changes @@ -1,3 +1,16 @@ +08Jul20======================================================================= +1. "Finish" the RBN system :-) +2. This includes enabling the coarse selection of spot modes using set/wantrbn + with arguments like 'set/wantrbn cw beacon'. This limits your output to + just CW, BCN and DXF modes. +3. The RBN spot is now cached. With a following wind, this means that even a + node restart, done in a timely fashion (within a few minutes) will not + cause a "cache warmup" delay for users on a restart. +4. Added the "full fat" set/wantrbn command and aliased it to 'set/skimmer'. + I use both terms (whenever I remembered) in the help text. +5. Help text has been written. +6. The UPGRADE.mojo file has been tweeked to point out the users file format + change. 07Jul20======================================================================= 1. Fix show/node command. 2. Fix show/cluster command to take into account the presence of skimmer nodes diff --git a/UPGRADE.mojo b/UPGRADE.mojo index 8fd254cb..baecd479 100644 --- a/UPGRADE.mojo +++ b/UPGRADE.mojo @@ -1,4 +1,8 @@ -There are the notes for upgrading to the mojo branch. +8th July 2020 +------------- + +There are the notes for upgrading to the mojo branch. PLEASE NOTE THERE HAVE BEEN CHANGES +FOR all MOJO BRANCH USERS. See APPENDIX(i) at the end of this document. There is NO POINT in doing this at the moment unless you are running a node with many (>50) users. It is the future, but at the moment I am testing larger and larger installations to @@ -184,6 +188,25 @@ I try very hard not to leave it in a broken state... Dirk G1TLH +APPENDIX(i) + +With this revrsion of the code, the users.v3 file will be replaced with users.v3j. This is a reversable +change. Simply revert to the previous revision, and email me, should anything go wrong. On restarting +the node, the users.v3j file will be generated from the users.v3 file. The users.v3 file is not changed. +The process of generation will take up to 30 seconds depending on the number of users in your file, +the speed of your disk(s) and the CPU speed (probably in that order. On my machine, it takes about 5 +seconds, on an RPi??? + +Part of this process may clear out some old records or suggest that there might errors. DO NOT BE +ALARM. This is completely normal. + +This change not only should make the rebuilding of the users file (much) less likely, but tests suggest +that access to the users file is about 2.5 times quicker. How much difference this makes in practise +remains to be seen. + +When you done this, in another shell, run /spider/perl/create_dxsql.pl. This will convert the DXQSL +system to dxqsl.v1j (for the sh/dxqsl command). When this is finished, run 'load/dxqsl' in +a console (or restart the node, but it isn't necessary). diff --git a/cmd/Aliases b/cmd/Aliases index e9029f18..f776d430 100644 --- a/cmd/Aliases +++ b/cmd/Aliases @@ -22,151 +22,160 @@ package CmdAlias; %alias = ( - '?' => [ - '^\?', 'apropos', 'apropos', - ], - 'a' => [ - '^a$', 'announce', 'announce', - '^acc?e?p?t?$', 'apropos accept', 'apropos', - '^ann?o?u?n?c?e?/full', 'announce full', 'announce', - '^ann?o?u?n?c?e?/sysop', 'announce sysop', 'announce', - '^ann?o?u?n?c?e?/(.*)$', 'announce $1', 'announce', - ], - 'b' => [ - '^b$', 'bye', 'bye', - ], - 'c' => [ - '^cle?a?r?$', 'apropos clear', 'apropos', - '^cre?a?t?e?$', 'apropos create', 'apropos', - ], - 'd' => [ - '^dele?t?e?/fu', 'kill full', 'kill', - '^dele?t?e?$', 'kill', 'kill', - '^dir?e?c?t?o?r?y?/a\w*', 'directory all', 'directory', - '^dir?e?c?t?o?r?y?/b\w*', 'directory bulletins', 'directory', - '^dir?e?c?t?o?r?y?/n\w*', 'directory new', 'directory', - '^dir?e?c?t?o?r?y?/o\w*', 'directory own', 'directory', - '^dir?e?c?t?o?r?y?/s\w*', 'directory subject', 'directory', - '^dir?e?c?t?o?r?y?/t\w*', 'directory to', 'directory', - '^dir?e?c?t?o?r?y?/f\w*', 'directory from', 'directory', - '^dir?e?c?t?o?r?y?/(\d+)-(\d+)', 'directory $1-$2', 'directory', - '^dir?e?c?t?o?r?y?/(\d+)', 'directory $1', 'directory', - ], - 'e' => [ - '^exi?t?$', 'bye', 'bye', - '^export_u', 'export_users', 'export_users', - '^expor?', 'export', 'export', - '^expun?g?e?$', 'kill expunge', 'kill expunge', - ], - 'f' => [ - '^for?w?a?r?d?$', 'apropos forward', 'apropos', - ], - 'g' => [ - ], - 'h' => [ - ], - 'i' => [ - ], - 'j' => [ - ], - 'k' => [ - '^ki?l?l?/ex', 'kill expunge', 'kill', - ], - 'l' => [ - '^loa?d?$', 'apropos load', 'apropos', - '^l$', 'directory', 'directory', - '^ll$', 'directory', 'directory', - '^ll/(\d+)', 'directory $1', 'directory', - '^lm$', 'directory own', 'directory', - '^l>$', 'directory to', 'directory', - '^l<$', 'directory from', 'directory', - ], - 'm' => [ - ], - 'n' => [ - ], - 'o' => [ - ], - 'p' => [ - ], - 'q' => [ - '^qu?i?t?$', 'bye', 'bye', - ], - 'r' => [ - '^r$', 'read', 'read', - '^reje?c?t?$', 'apropos reject', 'apropos', - '^rcmd/(\S+)', 'rcmd $1', 'rcmd', - ], - 's' => [ - '^s$', 'send', 'send', - '^s/p$', 'send', 'send', - '^sb$', 'send noprivate', 'send', - '^set/home$', 'set/homenode', 'set/homenode', - '^set/nobe', 'unset/beep', 'unset/beep', - '^set/nohe', 'unset/here', 'unset/here', - '^set/noan', 'unset/announce', 'unset/announce', - '^set/nodxg', 'unset/dxgrid', 'unset/dxgrid', - '^set/nodx', 'unset/dx', 'unset/dx', - '^set/noe', 'unset/echo', 'unset/echo', - '^set/nota', 'unset/talk', 'unset/talk', - '^set/noww', 'unset/wwv', 'unset/wwv', - '^set/nowx', 'unset/wx', 'unset/wx', - '^set$', 'apropos set', 'apropos', - '^sho?w?/u$', 'show/user', 'show/user', - '^sho?w?/bul', 'show/files bulletins', 'show/files', - '^sho?w?/co?n?\w*/a', 'show/configuration all', 'show/configuration', - '^sho?w?/co?n?\w*/n', 'show/configuration nodes', 'show/configuration', - '^sho?w?/c$', 'show/configuration', 'show/configuration', - '^sho?w?/com', 'dbavail', 'dbavail', - '^sho?w?/dxcc', 'show/dx dxcc', 'show/dx', - '^sho?w?/dx/(\d+)-(\d+)', 'show/dx $1-$2', 'show/dx', - '^sho?w?/dx/(\d+)', 'show/dx $1', 'show/dx', - '^sho?w?/dx/d(\d+)', 'show/dx from $1', 'show/dx', - '^sho?w?/fdx/(\d+)-(\d+)', 'show/dx real $1-$2', 'show/fdx', - '^sho?w?/fdx/(\d+)', 'show/dx real $1', 'show/fdx', - '^sho?w?/fdx/d(\d+)', 'show/dx real from $1', 'show/fdx', - '^sho?w?/fdx', 'show/dx real', 'show/fdx', - '^sho?w?/grou?p?s?', 'show/groups', 'show/groups', - '^sho?w?/gr[ae]?y?l?i?n?e?', 'show/grayline', 'show/grayline', - '^sho?w?/myfd?x?/(\d+)-(\d+)', 'show/dx filter real $1-$2', 'show/mydx', - '^sho?w?/myfd?x?/(\d+)', 'show/dx filter real $1', 'show/mydx', - '^sho?w?/myfd?x?/d(\d+)', 'show/dx filter real from $1', 'show/mydx', - '^sho?w?/myfd?x?', 'show/dx filter real', 'show/mydx', - '^sho?w?/myd?x?/(\d+)-(\d+)', 'show/dx filter $1-$2', 'show/mydx', - '^sho?w?/myd?x?/(\d+)', 'show/dx filter $1', 'show/mydx', - '^sho?w?/myd?x?/d(\d+)', 'show/dx filter from $1', 'show/mydx', - '^sho?w?/myd?x?', 'show/dx filter', 'show/mydx', - '^sho?w?/newco?n?\w*/n', 'show/newconfiguration node', 'show/newconfiguration', - '^sho?w?/sta?$', 'show/station', 'show/station', - '^sho?w?/tnc', 'who', 'who', - '^sho?w?/up', 'show/cluster', 'show/cluster', - '^sho?w?/ww?v?/(\d+)-(\d+)', 'show/wwv $1-$2', 'show/wwv', - '^sho?w?/ww?v?/(\d+)', 'show/wwv $1', 'show/wwv', - '^sho?w?$', 'apropos show', 'apropos', - '^shutd?\w*$', 'shutdown', 'shutdown', - '^sp$', 'send', 'send', - '^sta?t?$', 'apropos stat', 'apropos', + '?' => [ + '^\?', 'apropos', 'apropos', + ], + 'a' => [ + '^a$', 'announce', 'announce', + '^acc?e?p?t?$', 'apropos accept', 'apropos', + '^ann?o?u?n?c?e?/full', 'announce full', 'announce', + '^ann?o?u?n?c?e?/sysop', 'announce sysop', 'announce', + '^ann?o?u?n?c?e?/(.*)$', 'announce $1', 'announce', + ], + 'b' => [ + '^b$', 'bye', 'bye', + ], + 'c' => [ + '^cle?a?r?$', 'apropos clear', 'apropos', + '^cre?a?t?e?$', 'apropos create', 'apropos', + ], + 'd' => [ + '^dele?t?e?/fu', 'kill full', 'kill', + '^dele?t?e?$', 'kill', 'kill', + '^dir?e?c?t?o?r?y?/a\w*', 'directory all', 'directory', + '^dir?e?c?t?o?r?y?/b\w*', 'directory bulletins', 'directory', + '^dir?e?c?t?o?r?y?/n\w*', 'directory new', 'directory', + '^dir?e?c?t?o?r?y?/o\w*', 'directory own', 'directory', + '^dir?e?c?t?o?r?y?/s\w*', 'directory subject', 'directory', + '^dir?e?c?t?o?r?y?/t\w*', 'directory to', 'directory', + '^dir?e?c?t?o?r?y?/f\w*', 'directory from', 'directory', + '^dir?e?c?t?o?r?y?/(\d+)-(\d+)', 'directory $1-$2', 'directory', + '^dir?e?c?t?o?r?y?/(\d+)', 'directory $1', 'directory', + ], + 'e' => [ + '^exi?t?$', 'bye', 'bye', + '^export_u', 'export_users', 'export_users', + '^expor?', 'export', 'export', + '^expun?g?e?$', 'kill expunge', 'kill expunge', + ], + 'f' => [ + '^for?w?a?r?d?$', 'apropos forward', 'apropos', + ], + 'g' => [ + ], + 'h' => [ + ], + 'i' => [ + ], + 'j' => [ + ], + 'k' => [ + '^ki?l?l?/ex', 'kill expunge', 'kill', + ], + 'l' => [ + '^loa?d?$', 'apropos load', 'apropos', + '^l$', 'directory', 'directory', + '^ll$', 'directory', 'directory', + '^ll/(\d+)', 'directory $1', 'directory', + '^lm$', 'directory own', 'directory', + '^l>$', 'directory to', 'directory', + '^l<$', 'directory from', 'directory', + ], + 'm' => [ + ], + 'n' => [ + ], + 'o' => [ + ], + 'p' => [ + ], + 'q' => [ + '^qu?i?t?$', 'bye', 'bye', + ], + 'r' => [ + '^r$', 'read', 'read', + '^rbn$', 'apropos rbn', 'apropos', + '^reje?c?t?$', 'apropos reject', 'apropos', + '^rcmd/(\S+)', 'rcmd $1', 'rcmd', + ], + 's' => [ + '^s$', 'send', 'send', + '^s/p$', 'send', 'send', + '^sb$', 'send noprivate', 'send', + '^set/dbg$', 'set/debug', 'set/debug', + '^set/home$', 'set/homenode', 'set/homenode', + '^set/nobe', 'unset/beep', 'unset/beep', + '^set/nohe', 'unset/here', 'unset/here', + '^set/noan', 'unset/announce', 'unset/announce', + '^set/nodxg', 'unset/dxgrid', 'unset/dxgrid', + '^set/nodx', 'unset/dx', 'unset/dx', + '^set/noe', 'unset/echo', 'unset/echo', + '^set/nota', 'unset/talk', 'unset/talk', + '^set/noww', 'unset/wwv', 'unset/wwv', + '^set/nowx', 'unset/wx', 'unset/wx', + '^set/nosk', 'set/wantrbn none', 'set/wantrbn', + '^set/sk', 'set/wantrbn', 'set/wantrbn', + '^set$', 'apropos set', 'apropos', + '^sho?w?/u$', 'show/user', 'show/user', + '^sho?w?/bul', 'show/files bulletins', 'show/files', + '^sho?w?/co?n?\w*/a', 'show/configuration all', 'show/configuration', + '^sho?w?/co?n?\w*/n', 'show/configuration nodes', 'show/configuration', + '^sho?w?/c$', 'show/configuration', 'show/configuration', + '^sho?w?/com', 'dbavail', 'dbavail', + '^sho?w?/dbg', 'show/debug', 'show/debug', + '^sho?w?/dxcc', 'show/dx dxcc', 'show/dx', + '^sho?w?/dx/(\d+)-(\d+)', 'show/dx $1-$2', 'show/dx', + '^sho?w?/dx/(\d+)', 'show/dx $1', 'show/dx', + '^sho?w?/dx/d(\d+)', 'show/dx from $1', 'show/dx', + '^sho?w?/fdx/(\d+)-(\d+)', 'show/dx real $1-$2', 'show/fdx', + '^sho?w?/fdx/(\d+)', 'show/dx real $1', 'show/fdx', + '^sho?w?/fdx/d(\d+)', 'show/dx real from $1', 'show/fdx', + '^sho?w?/fdx', 'show/dx real', 'show/fdx', + '^sho?w?/grou?p?s?', 'show/groups', 'show/groups', + '^sho?w?/gr[ae]?y?l?i?n?e?', 'show/grayline', 'show/grayline', + '^sho?w?/myfd?x?/(\d+)-(\d+)', 'show/dx filter real $1-$2', 'show/mydx', + '^sho?w?/myfd?x?/(\d+)', 'show/dx filter real $1', 'show/mydx', + '^sho?w?/myfd?x?/d(\d+)', 'show/dx filter real from $1', 'show/mydx', + '^sho?w?/myfd?x?', 'show/dx filter real', 'show/mydx', + '^sho?w?/myd?x?/(\d+)-(\d+)', 'show/dx filter $1-$2', 'show/mydx', + '^sho?w?/myd?x?/(\d+)', 'show/dx filter $1', 'show/mydx', + '^sho?w?/myd?x?/d(\d+)', 'show/dx filter from $1', 'show/mydx', + '^sho?w?/myd?x?', 'show/dx filter', 'show/mydx', + '^sho?w?/newco?n?\w*/n', 'show/newconfiguration node', 'show/newconfiguration', + '^sho?w?/sta?$', 'show/station', 'show/station', + '^sho?w?/tnc', 'who', 'who', + '^sho?w?/u$', 'show/user', 'show/user', + '^sho?w?/up', 'show/cluster', 'show/cluster', + '^sho?w?/ww?v?/(\d+)-(\d+)', 'show/wwv $1-$2', 'show/wwv', + '^sho?w?/ww?v?/(\d+)', 'show/wwv $1', 'show/wwv', + '^sho?w?$', 'apropos show', 'apropos', + '^shutd?\w*$', 'shutdown', 'shutdown', + '^sp$', 'send', 'send', + '^sta?t?$', 'apropos stat', 'apropos', - ], - 't' => [ - '^ta$', 'talk', 'talk', - '^t$', 'talk', 'talk', - ], - 'u' => [ - '^uns?e?t?$', 'apropos unset', 'apropos', - '^uns?e?t?/node$', 'set/user', 'set/user', - ], - 'v' => [ - ], - 'w' => [ - '^w$', 'who', 'who', - '^wx/full', 'wx full', 'wx', - '^wx/sysop', 'wx sysop', 'wx', - ], - 'x' => [ - ], - 'y' => [ - ], - 'z' => [ - ], -) + ], + 't' => [ + '^ta$', 'talk', 'talk', + '^t$', 'talk', 'talk', + ], + 'u' => [ + '^uns?e?t?$', 'apropos unset', 'apropos', + '^uns?e?t?/dbg$', 'unset/debug', 'unset/debug', + '^uns?e?t?/node$', 'set/user', 'set/user', + '^uns?e?t?/sk', 'set/wantrbn none', 'set/wantrbn', + ], + 'v' => [ + ], + 'w' => [ + '^w$', 'who', 'who', + '^wx/full', 'wx full', 'wx', + '^wx/sysop', 'wx sysop', 'wx', + ], + 'x' => [ + ], + 'y' => [ + ], + 'z' => [ + ], + ); + diff --git a/cmd/Commands_en.hlp b/cmd/Commands_en.hlp index 244688d9..a44426a3 100644 --- a/cmd/Commands_en.hlp +++ b/cmd/Commands_en.hlp @@ -107,6 +107,7 @@ You can use the tag 'all' to accept everything eg: === 0^ACCEPT/SPOTS [0-9] ^Set an 'accept' filter line for spots +=== 0^ACCEPT/RBN [0-9] ^Set an 'accept' filter line for RBN spots Create an 'accept this spot' line for a filter. An accept filter line means that if the spot matches this filter it is @@ -1119,6 +1120,102 @@ is a good indication of the quality of the link. The actual time it takes is output to the console in seconds. Any visible cluster node can be PINGed. +=== 9^RBN^The Reverse Beacon or Skimmer System +Please read the document /spider/RBN.mojo. This has the latest information +about RBN/Skimmer setup. + +=== 0^RBN^The Reverse Beacon or Skimmer System +DXSpider now has the ability to show spots from the Reverse Beacon Network +or "Skimmers", if your sysop has enabled the feed(s) (and has the bandwidth +to both receive the feeds and also to pass them on to you. + +Currently there are two RBN/Skimmer feeds available which, at busy +times can send up to 50,000 spots/hour EACH. Somewhere in the low +1000s is more normal. Clearly this is not much use to the average user +and so DXSpider "curates" them by removing duplicates and checking for +invalid callsigns or prefixes, as well as using some algorithms to fix +the rather variable frequencies that some skimmers produce +(particularly for CW spots). + +This means that the format of the spot that you see is completely +different to the spots that the RBN feeds supply and, as a result of +the "curation" reduces the volume of spots to you by between 8 and 11 +times. + +See SET/SKIMMER (or SET/WANTRBN) for more information on enabling +RBN/Skimmer spots and also on selecting particular categories (e.g CW +or FT8/FT4) - which has the side benefit of reducing the volume of +spots that you receive even more! + +Here are some examples of the output: + +DX de LZ4UX-#: 14015.5 ON7TQ CW 6dB Q:9 Z:5,14,15,40 14 0646Z 20 +DX de VE7CC-#: 3573.0 N8ADO FT8 -14dB Q:4 Z:4,5 4 0647Z 3 +DX de DM7EE-#: 14027.5 R1AC CW 9dB Q:9* Z:5,15,17,20 16 0643Z 14 +DX de WE9V-#: 7074.0 EA7ALL FT8 -9dB Q:2+ Z:5 14 0641Z 4 + +Note that UNSET/DXGRID, UNSET/DXITU and SET/DXCQ are in operation in +these examples. This is completely optional. + +The comment field has been completely changed in order provide as much +information, in as smaller space, as possible. All the irrelevant +information has been removed. + +You can use the Category (CW and FT8 in these examples) to with +SET/SKIMMER (or SET/WANTRBN) to, rather coarsely, select which spots +you require. You can refine this further by the use of Filtering. See +SET/SKIMMER or SET/WANTRBN for more information. But the short answer +is that these are spots and are filtered like any other spot, unless +you want to filter these spots differently, in which case you can use +REJECT/RBN and ACCEPT/RBN in exactly the same way as ACCEPT/SPOT and +REJECT/SPOT. If you don't use RBN filters then these spots will be +filter by any spot filters that you may have. + +The next field (6dB, -14dB etc) is the LOWEST reported signal that was +heard. + +The Q: field is the number of skimmers that heard this spot (up to 9 +shown, but it could easily be many more). If Q: is > 1 (especially on +CW) then you can be reasonably certain that the callsign is accurate, +especially on CW. 'Q' stands for "Qualitee" :-) + +If there is a '*', it means that there was a disagreement about +frequency. In fact, particularly for CW spots, I have see +disagreements of 600Hz. Which is a worry. The frequency that is shown +is the majority view of all the skimmers spotting this call. You may +have to fossick about the airwaves to find the actual frequency :-) + +There are stations that are permanently on, like Beacons, and also +others that have long sessions on the same frequency and do a lot of +CQing. If they have been on for a certain length of time and they +reappear before their cache entry expires (about 2 hours), then they +are respotted. This is indicated by the '+'. NOTE - if they change +frequency, this will generate new spots. Each callsign/frequency pair +could respotted separately for as long as any individual +callsign/frequency pair remain in the cache. + +The Z: field is present then that indicates the other CQ zones that +heard this spot - not including the skimmer that is shown. I show as +many as there are in whatever space is left in the comment +field. Note: if you have any of the optional flags around the time +then they may overwrite part of this field. + +If there is NO filter in operation, then the skimmer spot with the +LOWEST signal strength will be shown. This implies that if any extra +Z: zones are shown, then the signal will be higher in those zones. + +If you have a filter (for instance: ACCEPT/SPOT by_zone 14 and not +zone 14 or zone 14 and not by_zone 14) where '14' is your QTH CQ +zone. You will, instead be served with the lowest signal strength spot +that satisfies that filter. Incidentally, this particular style of +filter is quite useful for RBN spots, as it reduces the volume and is +likely to be more relevant for casual use. If this filter is too broad +(or narrow) for your normal spotting requirements, then you can use +ACCEPT/RBN with the same filter specification and it will only apply +to RBN spots. You can also replace '14' with a list like '14,15' if +you want to broaden it out. You will still get the same Z: list (if +any) whether you filter or not. + === 1^RCMD ^Send a command to another DX Cluster This command allows you to send nearly any command to another DX Cluster node that is connected to the system. @@ -1185,6 +1282,7 @@ default for nodes and users eg:- reject/ann user_default by G,M,2 === 0^REJECT/SPOTS [0-9] ^Set a 'reject' filter line for spots +=== 0^REJECT/RBN [0-9] ^Set a 'reject' filter line for RBN spots Create a 'reject this spot' line for a filter. A reject filter line means that if the spot matches this filter it is @@ -1890,6 +1988,16 @@ correctly (assuming your locator is correct ;-). For example:- Tell the system where you are. For example:- SET/QTH East Dereham, Norfolk +=== 9^SET/RBN ...^Mark this call as an RBN node +This will mark this callsign as a Reverse Beacon +Network client. It's not a node in the normal sense of that word +in DXSpider. But it will generate spots from the RBN/Skimmers and +will act like a specialised node just for RBN spots. + +You will need to use this command to create your skimmer node +connections. Normally one per RBN port (7000, 7001) but, in principle +you could connect to any skimmer that uses the same spot format. + === 9^SET/REGISTER ...^Mark a user as registered === 9^UNSET/REGISTER ...^Mark a user as not registered Registration is a concept that you can switch on by executing the @@ -1958,6 +2066,70 @@ Conflicts with: SET/DXCQ, SET/DXITU Do a STAT/USER to see which flags you have set if you are confused. +=== 9^SET/WANTRBN^ [category ..]^Allow (some) RBN/Skimmer spots +=== 9^SET/SKIMMER^ [category ..]^Allow (some) RBN/Skimmer spots +This sysop only command allows you to set a user's RBN/Skimmer for them. + +It's also good for resetting a user's flags if they get into a muddle. + +=== 0^SET/WANTRBN^[category ..]^Allow (some) RBN/Skimmer spots +=== 0^SET/SKIMMER^[category ..]^Allow (some) RBN/Skimmer spotsT +=== 0^UNSET/WANTRBN^Stop all RBN/Skimmer spots +=== 0^UNSET/SKIMMER^Stop all RBN/Skimmer spots +This command allows curated Reverse Beacon Spots to come out on your +terminal (or not). + +If you want everything just type: + + set/wantrbn +or + set/skimmer + +Either command will do. + +If you want it all to just stop type: + + unset/skimmer (or unset/wantrbn) +or + set/skimmer none + +There five categories (or modes) of RBN/Skimmer spot available and one +can limit the spots to one or more of these categories/modes: + + CW BEACON PSK RTTY FT + +together with a load of synonyms + + BEACON BCN DXF + PSK FSK MSK + FT FT8 FT4 + +if you use + + set/skimmer psk ft8 + +you will get psk, fsk, msk, ft4 and ft8 spots. if you want to break +that down, then you will need to set filters accordingly - but your +filter will only be offered spots from the categories that you have +selected. + +If you get into a muddle with this you can simply reset 'all on' +with SET/SKIMMER or 'all off' with UNSET/SKIMMER. + +By default any filters that you have for "manual" spots will be +automatically applied to your RBN/Skimmer feed. However it is possible +to filter RBN/Skimmer spots differently by use ACCEPT/RBN and/or +REJECT/RBN filters. + +The RBN filters completely override any spot filters for these +spots. But the spot filters will continue to filter "manual" spots as +before. + +Please see HELP RBN for an explanation of the spot format. It is NOT +the same as one would get directly from the RBN/Skimmers. But it is +recommended that you SET/DXCQ and UNSET/DXITU and UNSET/DXGRID (unless +latter in more important to you with, for example, FT4/8 spots). + === 0^SET/WCY^Allow WCY messages to come out on your terminal === 0^UNSET/WCY^Stop WCY messages coming out on your terminal diff --git a/cmd/set/wantrbn.pl b/cmd/set/wantrbn.pl index f4aa86e2..e4528c33 100644 --- a/cmd/set/wantrbn.pl +++ b/cmd/set/wantrbn.pl @@ -7,21 +7,111 @@ # my ($self, $line) = @_; -my @args = split /\s+/, $line; +my @args = split /\s+/, uc $line; my $call; my @out; -@args = $self->call if (!@args || $self->priv < 9); +my @calls; +my @want; -foreach $call (@args) { +dbg('set/skimmer @args = "' . join(', ', @args) . '"') if isdbg('set/skim'); + +while (@args) { + my $a = shift @args; + dbg("set/skimmer \$a = $a") if isdbg('set/skim');; + if ($a !~ /^(?:FT|BCN|BEA|DXF|CW|PSK|MSK|FSK|RTT|NO)/ && is_callsign($a)) { + return (1, $self->msg('e5')) if $a ne $self->call && $self->priv < 9; + push @calls, $a; + next; + } + last unless $a; + + dbg("set/skimmer \$a = $a") if isdbg('set/skim');; + + my ($want) = $a =~ /^(FT|BCN|BEA|DXF|CW|PSK|MSK|FSK|RTT|NO)/; + return (1, $self->msg('e39', $a)) unless $want; + push @want, $want; +} + +dbg('set/skimmer @calls = "' . join(', ', @calls) . '"') if isdbg('set/skim'); +dbg('set/skimmer @want = "' . join(', ', @want) . '"') if isdbg('set/skim'); + +my $s = ''; + +push @calls, $self->call unless @calls; + +foreach $call (@calls) { $call = uc $call; my $user = DXUser::get_current($call); if ($user) { + + dbg(sprintf("set/skimmer before rbn:%d ft:%d bcn:%d cw:%d psk:%d rtty:%d", + $user->wantrbn, + $user->wantft, + $user->wantbeacon, + $user->wantcw, + $user->wantpsk, + $user->wantrtty, + )) if isdbg('set/skim'); + $user->wantrbn(1); + if (@want) { + $user->wantft(0); + $user->wantbeacon(0); + $user->wantcw(0); + $user->wantpsk(0); + $user->wantrtty(0); + for (@want) { + $user->wantrbn(0) if /^NO/; + $user->wantft(1) if /^FT/; + $user->wantbeacon(1) if /^BCN|BEA|DXF/; + $user->wantcw(1) if /^CW/; + $user->wantpsk(1) if /^PSK|MSK|FSK/; + $user->wantrtty(1) if /^RT/; + } + } elsif ($user->wantrbn) { + $user->wantft(1); + $user->wantbeacon(1); + $user->wantcw(1); + $user->wantpsk(1); + $user->wantrtty(1); + } else { + $user->wantft(0); + $user->wantbeacon(0); + $user->wantcw(0); + $user->wantpsk(0); + $user->wantrtty(0); + } + + dbg(sprintf("set/skimmer after rbn:%d ft:%d bcn:%d cw:%d psk:%d rtty:%d", + $user->wantrbn, + $user->wantft, + $user->wantbeacon, + $user->wantcw, + $user->wantpsk, + $user->wantrtty, + )) if isdbg('set/skim'); + + my $s = ''; + if (@want) { + @want = (); # variable reuse!! + push @want, 'CW' if $user->wantcw; + push @want, 'BEACONS' if $user->wantbeacon; + push @want, 'PSK, FSK' if $user->wantpsk; + push @want, 'RTTY' if $user->wantrtty; + push @want, 'FT8 & FT4' if $user->wantft; + $s = join(', ', @want) if @want && $user->wantrbn; + } + + dbg("set/skimmer \$s = $s") if isdbg('set/skim');; + dbg('set/skimmer @want NOW = "' . join(', ', @want) . '"') if isdbg('set/skim'); + + $s ||= $user->wantrbn ? 'ALL MODES' : 'NONE'; $user->put; - push @out, $self->msg('wante', 'RBN', $call); - } else { - push @out, $self->msg('e3', "Set wantrbn", $call); + push @out, $self->msg('skims', $call, $s); + } + else { + push @out, $self->msg('e3', "Set Skimmer", $call); } } return (1, @out); diff --git a/perl/DXUser.pm b/perl/DXUser.pm index bfc06b75..267c68ed 100644 --- a/perl/DXUser.pm +++ b/perl/DXUser.pm @@ -95,7 +95,7 @@ my $json; wantcw => '0,Want RBN CW,yesno', wantrtty => '0,Want RBN RTTY,yesno', wantpsk => '0,Want RBN PSK,yesno', - wantbeacon => '0,Want (RBN) Beacon,yesno', + wantbeacon => '0,Want RBN Beacon,yesno', lastoper => '9,Last for/oper,cldatetime', nothere => '0,Not Here Text', registered => '9,Registered?,yesno', diff --git a/perl/DXUtil.pm b/perl/DXUtil.pm index d23cb92e..f7e52c9a 100644 --- a/perl/DXUtil.pm +++ b/perl/DXUtil.pm @@ -385,7 +385,7 @@ sub is_callsign return $_[0] =~ m!^ (?:\d?[A-Z]{1,2}\d{0,2}/)? # out of area prefix / (?:\d?[A-Z]{1,2}\d{1,5}) # main prefix one (required) - lengthened for special calls - [A-Z]{1,5} # callsign letters (required) + [A-Z]{1,8} # callsign letters (required) (?:-(?:\d{1,2}))? # - nn possibly (eg G8BPQ-8) (?:/[0-9A-Z]{1,7})? # / another prefix, callsign or special label (including /MM, /P as well as /EURO or /LGT) possibly $!x; diff --git a/perl/Messages b/perl/Messages index 08a79251..d79eec73 100644 --- a/perl/Messages +++ b/perl/Messages @@ -112,6 +112,7 @@ package DXM; e36 => 'You can only do this in normal user prompt state', e37 => 'Need at least a callsign', e38 => 'This is not a valid regex', + e39 => 'Sorry $_[0] is not a valid argument', echoon => 'Echoing enabled', echooff => 'Echoing disabled', @@ -302,6 +303,7 @@ package DXM; showconf => 'Node Callsigns', shu => '\"SHU\" is not enough! you need to type at least \"SHUT\" to shutdown the node', shutting => '$main::mycall shutting down...', + skims => 'RBN/Skimming set to $_[1] for $_[0]', sloc => 'Cluster lat $_[0] long $_[1], DON\'T FORGET TO CHANGE YOUR DXVars.pm', snode1 => 'Node Call Sort Version', snode2 => '$_[0] $_[1] $_[2]', diff --git a/perl/RBN.pm b/perl/RBN.pm index 829f11f7..418c1cb3 100644 --- a/perl/RBN.pm +++ b/perl/RBN.pm @@ -11,8 +11,8 @@ package RBN; use 5.10.1; -use DXUtil; use DXDebug; +use DXUtil; use DXLog; use DXUser; use DXChannel; @@ -20,6 +20,8 @@ use Math::Round qw(nearest); use Date::Parse; use Time::HiRes qw(clock_gettime CLOCK_REALTIME); use Spot; +use JSON; +use IO::File; our @ISA = qw(DXChannel); @@ -34,7 +36,7 @@ our $minspottime = 60*60; # the time between respots of a callsign - if a call our $beacontime = 5*60; # same as minspottime, but for beacons (and shorter) -our $dwelltime = 8; # the amount of time to wait for duplicates before issuing +our $dwelltime = 10; # the amount of time to wait for duplicates before issuing # a spot to the user (no doubt waiting with bated breath). our $filterdef = $Spot::filterdef; # we use the same filter as the Spot system. Can't think why :-). @@ -43,6 +45,25 @@ my $spots; # the GLOBAL spot cache my %runtime; # how long each channel has been running +our $cachefn = localdata('rbn_cache'); +our $cache_valid = 4*60; # The cache file is considered valid if it is not more than this old + +my $json; +my $noinrush = 0; # override the inrushpreventor if set + +sub init +{ + $json = JSON->new; + $spots = {}; + if (check_cache()) { + $noinrush = 1; + } + if (defined $DB::VERSION) { + $noinrush = 1; + $json->indent(1); + } +} + sub new { my $self = DXChannel::alloc(@_); @@ -51,7 +72,6 @@ sub new my $pkg = shift; my $call = shift; - $spots ||= {}; $self->{last} = 0; $self->{noraw} = 0; $self->{nospot} = 0; @@ -130,7 +150,9 @@ sub start } # if we have been running and stopped for a while - $self->{inrushpreventor} = exists $runtime{$call} && $runtime{$call} > $startup_delay ? 0 : $main::systime + $startup_delay; + # if the cache is warm enough don't operate the inrush preventor + $self->{inrushpreventor} = exists $runtime{$call} && $runtime{$call} > $startup_delay || $noinrush ? 0 : $main::systime + $startup_delay; + dbg("RBN: noinrush: $noinrush, setting inrushpreventor on $self->{call} to $self->{inrushpreventor}"); } my @queue; # the queue of spots ready to send @@ -166,13 +188,19 @@ sub normal my $qra = $spd, $spd = '' if is_qra($spd); $u = $qra if $qra; + # is this anything like a callsign? + unless (is_callsign($call)) { + dbg("RBN: ERROR $call from $origin on $qrg is invalid, dumped"); + return; + } + $origin =~ s/\-(?:\d{1,2}\-)?\#$//; # get rid of all the crap we aren't interested in $sort ||= ''; $tx ||= ''; $qra ||= ''; - dbg qq{or:$origin qr:$qrg ca:$call mo:$mode s:$s m:$m sp:$spd u:$u sort:$sort t:$t tx:$tx qra:$qra} if isdbg('rbn'); + dbg qq{RBN:input decode or:$origin qr:$qrg ca:$call mo:$mode s:$s m:$m sp:$spd u:$u sort:$sort t:$t tx:$tx qra:$qra} if isdbg('rbn'); ++$self->{noraw}; ++$self->{noraw10}; @@ -259,12 +287,11 @@ sub normal # here we either have an existing spot record buildup on the go, or we need to create the first one unless ($spot) { $spots->{$sp} = $spot = [clock_gettime(CLOCK_REALTIME)];; - dbg("RBN: key: '$sp' call: $call qrg: $qrg NEW" . $respot ? ' RESPOT' : '') if isdbg('rbn'); + dbg("RBN: key: '$sp' call: $call qrg: $qrg NEW" . ($respot ? ' RESPOT' : '')) if isdbg('rbn'); } # add me to the display queue unless we are waiting for initial in rush to finish - return unless $self->{inrushpreventor} < $main::systime; - push @{$self->{queue}}, $sp if @$spot == 1; # queue the KEY (not the record) + return unless $noinrush || $self->{inrushpreventor} < $main::systime; # build up a new record and store it in the buildup # deal with the unix time @@ -274,14 +301,22 @@ sub normal # create record and add into the buildup my $r = [$origin, nearest(.1, $qrg), $call, $mode, $s, $t, $utz, $respot, $u]; - dbg("RBN: key: '$sp' ADD RECORD call: $call qrg: $qrg origin: $origin") if isdbg('rbn'); my @s = Spot::prepare($r->[1], $r->[2], $r->[6], '', $r->[0]); + if ($s[5] == 666) { + dbg("RBN: ERROR invalid prefix/callsign $call from $origin-# on $qrg, dumped"); + return; + } + if ($self->{inrbnfilter}) { my ($want, undef) = $self->{inrbnfilter}->it($s); - next unless $want; + return unless $want; } $r->[9] = \@s; + push @{$self->{queue}}, $sp if @$spot == 1; # queue the KEY (not the record) + + dbg("RBN: key: '$sp' ADD RECORD call: $call qrg: $qrg origin: $origin") if isdbg('rbn'); + push @$spot, $r; # At this point we run the queue to see if anything can be sent onwards to the punter @@ -340,6 +375,9 @@ sub per_minute $dxchan->{noraw} = $dxchan->{norbn} = $dxchan->{nospot} = 0; $dxchan->{nousers} = {}; $runtime{$dxchan->{call}} += 60; } + + # save the spot cache + write_cache() unless $main::systime + $startup_delay < $main::systime;; } sub per_10_minute @@ -399,12 +437,17 @@ sub send_dx_spot ++$want if $user->wantbeacon && $mode =~ /^BCN|DXF/; ++$want if $user->wantcw && $mode =~ /^CW/; ++$want if $user->wantrtty && $mode =~ /^RTT/; - ++$want if $user->wantpsk && $mode =~ /^PSK/; - ++$want if $user->wantcw && $mode =~ /^CW/; + ++$want if $user->wantpsk && $mode =~ /^PSK|FSK|MSK/; ++$want if $user->wantft && $mode =~ /^FT/; - ++$want unless $want; # send everything if nothing is selected. - next unless $want; + dbg(sprintf("RBN: spot selection for $dxchan->{call} mode: '$mode' want: $want flags rbn:%d ft:%d bcn:%d cw:%d psk:%d rtty:%d", + $user->wantrbn, + $user->wantft, + $user->wantbeacon, + $user->wantcw, + $user->wantpsk, + $user->wantrtty, + )) if isdbg('rbnll'); # send one spot to one user out of the ones that we have $self->dx_spot($dxchan, $quality, $spot) if $want; @@ -516,4 +559,55 @@ sub dx_spot } } +sub finish +{ + write_cache(); +} + +sub write_cache +{ + my $fh = IO::File->new(">$cachefn") or confess("writing $cachefn $!"); + my $s = $json->encode($spots); + $fh->print($s); + $fh->close; +} + +sub check_cache +{ + if (-e $cachefn) { + my $mt = (stat($cachefn))[9]; + my $t = $main::systime - $mt || 1; + my $p = difft($mt); + if ($t < $cache_valid) { + dbg("RBN:check_cache '$cachefn' spot cache exists, created $p ago and not too old"); + my $fh = IO::File->new($cachefn); + my $s; + if ($fh) { + local $/ = undef; + $s = <$fh>; + dbg("RBN:check_cache cache read size " . length $s); + $fh->close; + } else { + dbg("RBN:check_cache file read error $!"); + return undef; + } + if ($s) { + eval {$spots = $json->decode($s)}; + if ($spots && ref $spots) { + dbg("RBN:check_cache spot cache restored"); + return 1; + } + } + dbg("RBN::checkcache error decoding $@"); + } else { + my $d = difft($main::systime-$cache_valid); + dbg("RBN::checkcache '$cachefn' created $p ago is too old (> $d), ignored"); + } + } else { + dbg("RBN:check_cache '$cachefn' spot cache not present"); + } + + return undef; +} + 1; diff --git a/perl/cluster.pl b/perl/cluster.pl index bd8de8c3..2f1baf46 100755 --- a/perl/cluster.pl +++ b/perl/cluster.pl @@ -412,6 +412,7 @@ sub cease UDPMsg::finish(); # end everything else + RBN::finish(); DXUser::finish(); DXDupe::finish(); @@ -683,6 +684,9 @@ sub setup_start dbg("reading database descriptors ..."); DXDb::load(); + dbg("starting RBN ..."); + RBN::init(); + # starting local stuff dbg("doing local initialisation ..."); QSL::init(1); -- 2.34.1