diff --git a/CHANGELOG.md b/CHANGELOG.md index 92eebfa5d..2613d7e96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ which provides certain common STOMP-bot-specific operations, factored out from `intelmq.bots.collectors.stomp.collector` and `intelmq.bots.outputs.stomp.output` (PR#2408 and PR#2414 by Jan Kaliszewski). +- `intelmq.lib.upgrades`: Replace deprecated instances of `url2fqdn` experts by the new `url` expert in runtime configuration (PR#2432 by Sebastian Wagner). ### Development - Makefile: Add codespell and test commands (PR#2425 by Sebastian Wagner). diff --git a/intelmq/etc/runtime.yaml b/intelmq/etc/runtime.yaml index 170f0cbfd..c39c7b248 100644 --- a/intelmq/etc/runtime.yaml +++ b/intelmq/etc/runtime.yaml @@ -156,16 +156,16 @@ taxonomy-expert: parameters: destination_queues: _default: - - url2fqdn-expert-queue + - url-expert-queue run_mode: continuous -url2fqdn-expert: - bot_id: url2fqdn-expert - description: url2fqdn is the bot responsible to parsing the fqdn from the url. +url-expert: + bot_id: url-expert + description: Extract additional information for the URL enabled: true group: Expert groupname: experts - module: intelmq.bots.experts.url2fqdn.expert - name: URL2FQDN + module: intelmq.bots.experts.url.expert + name: url parameters: destination_queues: _default: diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index a920d4be8..1972a628a 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -39,6 +39,7 @@ 'v310_feed_changes', 'v310_shadowserver_feednames', 'v320_update_turris_greylist_url', + 'v322_url_replacement', ] @@ -879,6 +880,24 @@ def v320_update_turris_greylist_url(configuration, harmonization, dry_run, **kwa return ' '.join(messages) if messages else None, configuration, harmonization +def v322_url_replacement(configuration, harmonization, dry_run, **kwargs): + """ + Replace deprecated url2fqdn expert with url expert. + """ + changed = False + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] == "intelmq.bots.experts.url2fqdn.expert": + configuration[bot_id]["module"] = "intelmq.bots.experts.url.expert" + if "parameters" not in configuration[bot_id]: + configuration[bot_id]["parameters"] = {} + # skip all fields except the fqdn field for backwards compatibility + configuration[bot_id]["parameters"]["skip_fields"] = ["source.ip", "source.port", "source.urlpath", "source.account", "destination.ip", "destination.port", "destination.urlpath", "destination.account", "protocol.application", "protocol.transport"] + changed = True + return changed, configuration, harmonization + + UPGRADES = OrderedDict([ ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax,)), ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)), @@ -905,6 +924,7 @@ def v320_update_turris_greylist_url(configuration, harmonization, dry_run, **kwa ((3, 0, 2), ()), ((3, 1, 0), (v310_feed_changes, v310_shadowserver_feednames)), ((3, 2, 0), (v320_update_turris_greylist_url,)), + ((3, 2, 2), (v322_url_replacement, )), ]) ALWAYS = (harmonization,) diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index 151bfb161..dfdfdca0a 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -550,6 +550,29 @@ "parameters": {} } } +V322_URL2FQN_IN = { + "global": {}, + "url2fqdn-expert": { + "module": "intelmq.bots.experts.url2fqdn.expert", + "parameters": { + } + }, +} +V322_URL2FQN_IN_1 = { + "global": {}, + "url2fqdn-expert": { + "module": "intelmq.bots.experts.url2fqdn.expert", + }, +} +V322_URL2FQN_OUT = { + "global": {}, + "url2fqdn-expert": { + "module": "intelmq.bots.experts.url.expert", + "parameters": { + "skip_fields": ["source.ip", "source.port", "source.urlpath", "source.account", "destination.ip", "destination.port", "destination.urlpath", "destination.account", "protocol.application", "protocol.transport"] + }, + }, +} def generate_function(function): @@ -762,6 +785,16 @@ def test_v310_feed_changes(self): result[0]) self.assertEqual(V310_FEED_CHANGES, result[1]) + def test_v322_url_replacement(self): + """ Test v322_url_replacement """ + result = upgrades.v322_url_replacement(V322_URL2FQN_IN, {}, False) + self.assertTrue(result[0]) + self.assertEqual(V322_URL2FQN_OUT, result[1]) + + result = upgrades.v322_url_replacement(V322_URL2FQN_IN_1, {}, False) + self.assertTrue(result[0]) + self.assertEqual(V322_URL2FQN_OUT, result[1]) + for name in upgrades.__all__: setattr(TestUpgradeLib, 'test_function_%s' % name,