From a2deddd20ad38c61b591b9c7a4fbb33c071e27c2 Mon Sep 17 00:00:00 2001 From: gethvi Date: Thu, 14 Dec 2023 15:47:54 +0100 Subject: [PATCH] FIX: Removes discontinued feeds and bots. --- CHANGELOG.md | 3 + docs/user/feeds.md | 161 ---- intelmq/bots/parsers/netlab_360/__init__.py | 0 intelmq/bots/parsers/netlab_360/parser.py | 70 -- intelmq/bots/parsers/sucuri/__init__.py | 0 intelmq/bots/parsers/sucuri/parser.py | 71 -- intelmq/bots/parsers/webinspektor/__init__.py | 0 intelmq/bots/parsers/webinspektor/parser.py | 60 -- intelmq/etc/feeds.yaml | 91 --- intelmq/lib/upgrades.py | 56 +- .../tests/bots/parsers/netlab_360/__init__.py | 0 intelmq/tests/bots/parsers/netlab_360/dga.txt | 9 - .../bots/parsers/netlab_360/dga.txt.license | 2 - .../tests/bots/parsers/netlab_360/hajime.txt | 1 - .../parsers/netlab_360/hajime.txt.license | 2 - .../bots/parsers/netlab_360/magnitude.txt | 4 - .../parsers/netlab_360/magnitude.txt.license | 2 - .../tests/bots/parsers/netlab_360/mirai.txt | 1 - .../bots/parsers/netlab_360/mirai.txt.license | 2 - .../bots/parsers/netlab_360/test_parser.py | 148 ---- intelmq/tests/bots/parsers/sucuri/__init__.py | 0 .../bots/parsers/sucuri/test_sucuri.data | 764 ------------------ .../parsers/sucuri/test_sucuri.data.license | 2 - .../tests/bots/parsers/sucuri/test_sucuri.py | 66 -- .../bots/parsers/webinspektor/__init__.py | 0 .../webinspektor/test_webinspektor.data | 582 ------------- .../test_webinspektor.data.license | 2 - .../parsers/webinspektor/test_webinspektor.py | 69 -- intelmq/tests/lib/test_upgrades.py | 29 + 29 files changed, 87 insertions(+), 2110 deletions(-) delete mode 100644 intelmq/bots/parsers/netlab_360/__init__.py delete mode 100644 intelmq/bots/parsers/netlab_360/parser.py delete mode 100644 intelmq/bots/parsers/sucuri/__init__.py delete mode 100644 intelmq/bots/parsers/sucuri/parser.py delete mode 100644 intelmq/bots/parsers/webinspektor/__init__.py delete mode 100644 intelmq/bots/parsers/webinspektor/parser.py delete mode 100644 intelmq/tests/bots/parsers/netlab_360/__init__.py delete mode 100644 intelmq/tests/bots/parsers/netlab_360/dga.txt delete mode 100644 intelmq/tests/bots/parsers/netlab_360/dga.txt.license delete mode 100644 intelmq/tests/bots/parsers/netlab_360/hajime.txt delete mode 100644 intelmq/tests/bots/parsers/netlab_360/hajime.txt.license delete mode 100644 intelmq/tests/bots/parsers/netlab_360/magnitude.txt delete mode 100644 intelmq/tests/bots/parsers/netlab_360/magnitude.txt.license delete mode 100644 intelmq/tests/bots/parsers/netlab_360/mirai.txt delete mode 100644 intelmq/tests/bots/parsers/netlab_360/mirai.txt.license delete mode 100644 intelmq/tests/bots/parsers/netlab_360/test_parser.py delete mode 100644 intelmq/tests/bots/parsers/sucuri/__init__.py delete mode 100644 intelmq/tests/bots/parsers/sucuri/test_sucuri.data delete mode 100644 intelmq/tests/bots/parsers/sucuri/test_sucuri.data.license delete mode 100644 intelmq/tests/bots/parsers/sucuri/test_sucuri.py delete mode 100644 intelmq/tests/bots/parsers/webinspektor/__init__.py delete mode 100644 intelmq/tests/bots/parsers/webinspektor/test_webinspektor.data delete mode 100644 intelmq/tests/bots/parsers/webinspektor/test_webinspektor.data.license delete mode 100644 intelmq/tests/bots/parsers/webinspektor/test_webinspektor.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 38a140f6e..9eea08993 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,9 @@ - Minor fixes/improvements and some refactoring (see also above: *Core*...). #### Parsers + - `intelmq.bots.parsers.netlab_360.parser`: Removed as the feed is discontinued. (#2442 by Filip Pokorný) + - `intelmq.bots.parsers.webinspektor.parser`: Removed as the feed is discontinued. (#2442 by Filip Pokorný) + - `intelmq.bots.parsers.sucuri.parser`: Removed as the feed is discontinued. (#2442 by Filip Pokorný) #### Experts - `intelmq.bots.experts.jinja` (PR#2417 by Mikk Margus Möll): diff --git a/docs/user/feeds.md b/docs/user/feeds.md index 8224336d8..06c67fb46 100644 --- a/docs/user/feeds.md +++ b/docs/user/feeds.md @@ -1899,101 +1899,6 @@ module: intelmq.bots.parsers.microsoft.parser_ctip --- -## Netlab 360 - -### DGA - -This feed lists DGA family, Domain, Start and end of valid time(UTC) of a number of DGA families. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: http://data.netlab.360.com/feeds/dga/dga.txt - name: DGA - provider: Netlab 360 - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.netlab_360.parser -``` - ---- - - -### Hajime Scanner - -This feed lists IP address for know Hajime bots network. These IPs data are obtained by joining the DHT network and interacting with the Hajime node - -**Public:** yes - -**Revision:** 2019-08-01 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://data.netlab.360.com/feeds/hajime-scanner/bot.list - name: Hajime Scanner - provider: Netlab 360 - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.netlab_360.parser -``` - ---- - - -### Magnitude EK - -This feed lists FQDN and possibly the URL used by Magnitude Exploit Kit. Information also includes the IP address used for the domain and last time seen. - -**Public:** yes - -**Revision:** 2018-01-20 - -**Documentation:** - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: http://data.netlab.360.com/feeds/ek/magnitude.txt - name: Magnitude EK - provider: Netlab 360 - rate_limit: 3600 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.netlab_360.parser -``` - ---- - - ## OpenPhish ### Premium Feed @@ -2495,41 +2400,6 @@ module: intelmq.bots.parsers.github_feed --- -## Sucuri - -### Hidden IFrames - -Latest hidden iframes identified on compromised web sites. - -**Public:** yes - -**Revision:** 2018-01-28 - -**Documentation:** - -**Additional Information:** Please note that the parser only extracts the hidden iframes and the conditional redirects, not the encoded javascript. - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: http://labs.sucuri.net/?malware - name: Hidden IFrames - provider: Sucuri - rate_limit: 86400 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.sucuri.parser -``` - ---- - - ## Surbl ### Malicious Domains @@ -2917,37 +2787,6 @@ parameters: --- -## WebInspektor - -### Unsafe sites - -Latest detected unsafe sites. - -**Public:** yes - -**Revision:** 2018-03-09 - - -**Collector configuration** - -```yaml -module: intelmq.bots.collectors.http.collector_http -parameters: - http_url: https://app.webinspector.com/public/recent_detections/ - name: Unsafe sites - provider: WebInspektor - rate_limit: 60 -``` - -**Parser configuration** - -```yaml -module: intelmq.bots.parsers.webinspektor.parser -``` - ---- - - ## ZoneH ### Defacements diff --git a/intelmq/bots/parsers/netlab_360/__init__.py b/intelmq/bots/parsers/netlab_360/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/intelmq/bots/parsers/netlab_360/parser.py b/intelmq/bots/parsers/netlab_360/parser.py deleted file mode 100644 index 04017e942..000000000 --- a/intelmq/bots/parsers/netlab_360/parser.py +++ /dev/null @@ -1,70 +0,0 @@ -# SPDX-FileCopyrightText: 2016 jgedeon120 -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- -""" IntelMQ parser for Netlab 360 data feeds. """ - -from intelmq.lib.bot import ParserBot -from intelmq.lib.harmonization import DateTime - - -class Netlab360ParserBot(ParserBot): - """Parse the Netlab 360 DGA, Hajime, Magnitude and Mirai feeds""" - DGA_FEED = {'http://data.netlab.360.com/feeds/dga/dga.txt', - 'https://data.netlab.360.com/feeds/dga/dga.txt'} - MAGNITUDE_FEED = {'http://data.netlab.360.com/feeds/ek/magnitude.txt', - 'https://data.netlab.360.com/feeds/ek/magnitude.txt'} - MIRAI_SCANNER_FEED = {'http://data.netlab.360.com/feeds/mirai-scanner/scanner.list', - 'https://data.netlab.360.com/feeds/mirai-scanner/scanner.list'} - HAJIME_SCANNER_FEED = {'http://data.netlab.360.com/feeds/hajime-scanner/bot.list', - 'https://data.netlab.360.com/feeds/hajime-scanner/bot.list'} - - def parse_line(self, line, report): - if line.startswith('#') or not line.strip(): - self.tempdata.append(line) - - else: - value = line.split('\t') - event = self.new_event(report) - event.add('classification.identifier', value[0].lower()) - event.add('raw', line) - - if report['feed.url'] in Netlab360ParserBot.DGA_FEED: - event.add('source.fqdn', value[1]) - # DGA Feed format is - # DGA family, Domain, Start and end of valid time(UTC) - - event.add('time.source', value[2] + ' UTC') - if event['time.source'] > event['time.observation']: - event.change('time.source', event['time.observation']) - event.add('classification.type', 'c2-server') - event.add('event_description.url', 'http://data.netlab.360.com/dga') - - elif report['feed.url'] in Netlab360ParserBot.MAGNITUDE_FEED: - event.add('time.source', DateTime.from_timestamp(int(value[1]))) - event.add('source.ip', value[2]) - # ignore ips as fqdns - event.add('source.fqdn', value[3], raise_failure=False) - if value[4] != 'N/A': - event.add('source.url', value[4]) - event.add('classification.type', 'exploit') - event.add('event_description.url', 'http://data.netlab.360.com/ek') - elif report['feed.url'] in Netlab360ParserBot.MIRAI_SCANNER_FEED: - event.add('time.source', value[0] + ' UTC') - event.add('source.ip', value[1].replace('sip=', '')) - event.add('destination.port', value[2].replace('dport=', '')) - event.add('classification.type', 'scanner') - event.add('classification.identifier', 'mirai', overwrite=True) - elif report['feed.url'] in Netlab360ParserBot.HAJIME_SCANNER_FEED: - event.add('time.source', value[0] + 'T00:00:00 UTC') - event.add('source.ip', value[1].replace('ip=', '')) - event.add('classification.type', 'scanner') - event.add('classification.identifier', 'hajime', overwrite=True) - else: - raise ValueError('Unknown data feed %s.' % report['feed.url']) - - yield event - - -BOT = Netlab360ParserBot diff --git a/intelmq/bots/parsers/sucuri/__init__.py b/intelmq/bots/parsers/sucuri/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/intelmq/bots/parsers/sucuri/parser.py b/intelmq/bots/parsers/sucuri/parser.py deleted file mode 100644 index 2a2bae951..000000000 --- a/intelmq/bots/parsers/sucuri/parser.py +++ /dev/null @@ -1,71 +0,0 @@ -# SPDX-FileCopyrightText: 2018 dargen3 -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- -""" -Only parses hidden iframes and conditional redirections, not Encoded javascript. -""" -import re -from html.parser import HTMLParser - -from intelmq.lib import utils -from intelmq.lib.bot import ParserBot - - -class MyHTMLParser(HTMLParser): - - lsData = "" - - def handle_data(self, data): - self.lsData = data - - -parser = MyHTMLParser() -remove_comments = re.compile(r"", re.DOTALL) - - -class SucuriParserBot(ParserBot): - """Parse the Sucuri Malware Hidden Iframes and Conditional redirections feeds""" - def process(self): - report = self.receive_message() - raw_report = utils.base64_decode(report["raw"]) # decoding - report_list = [row.strip() for row in raw_report.splitlines()] - index = 0 - actual_line = report_list[index] - while parser.lsData != "Hidden iframes": # displacement to target table - index += 1 - actual_line = report_list[index] - parser.feed(actual_line) - while actual_line[:8] != "": # scrabing table data - index += 1 - raw_actual_line = report_list[index] - actual_line = remove_comments.sub("", raw_actual_line).replace('.', '.') - if actual_line[:2] == "