From b775fcdb65635068831f54618f13e6d442f598fe Mon Sep 17 00:00:00 2001
From: Sebastian Wagner <sebix@sebix.at>
Date: Wed, 22 Nov 2023 20:15:38 +0100
Subject: [PATCH] ENH: upgrade: replace url2fqn by url expert

the url2fqn expert is deprecated and will be removed in version 4
remove the bot from the default runtime config
add an upgrade function that replaces the bot in installations by the
url expert and parameters so that the behavior will stay the same

fixes certtools/intelmq#2430
---
 CHANGELOG.md                       |  1 +
 intelmq/etc/runtime.yaml           | 12 +++++------
 intelmq/lib/upgrades.py            | 20 ++++++++++++++++++
 intelmq/tests/lib/test_upgrades.py | 33 ++++++++++++++++++++++++++++++
 4 files changed, 60 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2aae7fd66..13d85d575 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -32,6 +32,7 @@
   which provides certain common STOMP-bot-specific operations, factored out from
   `intelmq.bots.collectors.stomp.collector` and `intelmq.bots.outputs.stomp.output`
   (PR#2408 and PR#2414 by Jan Kaliszewski).
+- `intelmq.lib.upgrades`: Replace deprecated instances of `url2fqdn` experts by the new `url` expert in runtime configuration (PR#2432 by Sebastian Wagner).
 
 ### Development
 - Makefile: Add codespell and test commands (PR#2425 by Sebastian Wagner).
diff --git a/intelmq/etc/runtime.yaml b/intelmq/etc/runtime.yaml
index 170f0cbfd..c39c7b248 100644
--- a/intelmq/etc/runtime.yaml
+++ b/intelmq/etc/runtime.yaml
@@ -156,16 +156,16 @@ taxonomy-expert:
   parameters:
     destination_queues:
       _default:
-      - url2fqdn-expert-queue
+      - url-expert-queue
   run_mode: continuous
-url2fqdn-expert:
-  bot_id: url2fqdn-expert
-  description: url2fqdn is the bot responsible to parsing the fqdn from the url.
+url-expert:
+  bot_id: url-expert
+  description: Extract additional information for the URL
   enabled: true
   group: Expert
   groupname: experts
-  module: intelmq.bots.experts.url2fqdn.expert
-  name: URL2FQDN
+  module: intelmq.bots.experts.url.expert
+  name: url
   parameters:
     destination_queues:
       _default:
diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py
index a920d4be8..1972a628a 100644
--- a/intelmq/lib/upgrades.py
+++ b/intelmq/lib/upgrades.py
@@ -39,6 +39,7 @@
            'v310_feed_changes',
            'v310_shadowserver_feednames',
            'v320_update_turris_greylist_url',
+           'v322_url_replacement',
            ]
 
 
@@ -879,6 +880,24 @@ def v320_update_turris_greylist_url(configuration, harmonization, dry_run, **kwa
     return ' '.join(messages) if messages else None, configuration, harmonization
 
 
+def v322_url_replacement(configuration, harmonization, dry_run, **kwargs):
+    """
+    Replace deprecated url2fqdn expert with url expert.
+    """
+    changed = False
+    for bot_id, bot in configuration.items():
+        if bot_id == 'global':
+            continue
+        if bot["module"] == "intelmq.bots.experts.url2fqdn.expert":
+            configuration[bot_id]["module"] = "intelmq.bots.experts.url.expert"
+            if "parameters" not in configuration[bot_id]:
+                configuration[bot_id]["parameters"] = {}
+            # skip all fields except the fqdn field for backwards compatibility
+            configuration[bot_id]["parameters"]["skip_fields"] = ["source.ip", "source.port", "source.urlpath", "source.account", "destination.ip", "destination.port", "destination.urlpath", "destination.account", "protocol.application", "protocol.transport"]
+            changed = True
+    return changed, configuration, harmonization
+
+
 UPGRADES = OrderedDict([
     ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax,)),
     ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)),
@@ -905,6 +924,7 @@ def v320_update_turris_greylist_url(configuration, harmonization, dry_run, **kwa
     ((3, 0, 2), ()),
     ((3, 1, 0), (v310_feed_changes, v310_shadowserver_feednames)),
     ((3, 2, 0), (v320_update_turris_greylist_url,)),
+    ((3, 2, 2), (v322_url_replacement, )),
 ])
 
 ALWAYS = (harmonization,)
diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py
index 151bfb161..dfdfdca0a 100644
--- a/intelmq/tests/lib/test_upgrades.py
+++ b/intelmq/tests/lib/test_upgrades.py
@@ -550,6 +550,29 @@
         "parameters": {}
     }
 }
+V322_URL2FQN_IN = {
+    "global": {},
+    "url2fqdn-expert": {
+        "module": "intelmq.bots.experts.url2fqdn.expert",
+        "parameters": {
+        }
+    },
+}
+V322_URL2FQN_IN_1 = {
+    "global": {},
+    "url2fqdn-expert": {
+        "module": "intelmq.bots.experts.url2fqdn.expert",
+    },
+}
+V322_URL2FQN_OUT = {
+    "global": {},
+    "url2fqdn-expert": {
+        "module": "intelmq.bots.experts.url.expert",
+        "parameters": {
+            "skip_fields": ["source.ip", "source.port", "source.urlpath", "source.account", "destination.ip", "destination.port", "destination.urlpath", "destination.account", "protocol.application", "protocol.transport"]
+        },
+    },
+}
 
 
 def generate_function(function):
@@ -762,6 +785,16 @@ def test_v310_feed_changes(self):
                          result[0])
         self.assertEqual(V310_FEED_CHANGES, result[1])
 
+    def test_v322_url_replacement(self):
+        """ Test v322_url_replacement """
+        result = upgrades.v322_url_replacement(V322_URL2FQN_IN, {}, False)
+        self.assertTrue(result[0])
+        self.assertEqual(V322_URL2FQN_OUT, result[1])
+
+        result = upgrades.v322_url_replacement(V322_URL2FQN_IN_1, {}, False)
+        self.assertTrue(result[0])
+        self.assertEqual(V322_URL2FQN_OUT, result[1])
+
 
 for name in upgrades.__all__:
     setattr(TestUpgradeLib, 'test_function_%s' % name,