From 4536da50c65c58d625f8f3bf6cd1e1bf05e8f3d8 Mon Sep 17 00:00:00 2001 From: aaronkaplan Date: Sun, 11 Jun 2017 15:09:42 -0400 Subject: [PATCH 1/4] initial version of the domain tools expert --- intelmq/bots/experts/domaintools/README.md | 13 +++++ intelmq/bots/experts/domaintools/__init__.py | 0 intelmq/bots/experts/domaintools/expert.py | 55 +++++++++++++++++++ .../bots/experts/domaintools/test_expert.py | 48 ++++++++++++++++ 4 files changed, 116 insertions(+) create mode 100644 intelmq/bots/experts/domaintools/README.md create mode 100644 intelmq/bots/experts/domaintools/__init__.py create mode 100644 intelmq/bots/experts/domaintools/expert.py create mode 100644 intelmq/tests/bots/experts/domaintools/test_expert.py diff --git a/intelmq/bots/experts/domaintools/README.md b/intelmq/bots/experts/domaintools/README.md new file mode 100644 index 000000000..9d404c83a --- /dev/null +++ b/intelmq/bots/experts/domaintools/README.md @@ -0,0 +1,13 @@ +# Domaintools expert + +This expert bot is an example on how to query domaintools. + +It does require an API from domaintools. + +Documentation on domaintools: https://www.domaintools.com/resources/api-documentation/ +Specifically, this bot can query a domain for the reputation score in domaintools: https://www.domaintools.com/resources/api-documentation/reputation/ + +It will add the score in the extra field: ``extra.domaintools_score``. + + +Authors: Juan Ortega Valiente, Aaron Kaplan diff --git a/intelmq/bots/experts/domaintools/__init__.py b/intelmq/bots/experts/domaintools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/intelmq/bots/experts/domaintools/expert.py b/intelmq/bots/experts/domaintools/expert.py new file mode 100644 index 000000000..687e20aac --- /dev/null +++ b/intelmq/bots/experts/domaintools/expert.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +""" +domaintools expert: query domtaintools.com to get a reputation score for a domain name + +""" +from intelmq.lib.bot import Bot +try: + from domaintools import API, exceptions +except ImportError: + API = None + + +class DomaintoolsExpertBot(Bot): + + def init(self): + self.logger.info("Loading Domaintools expert.") + if (not API): + self.logger.exception("need to have the domaintools API installed. See https://github.com/domaintools/python_api") + self.stop() + if (not self.parameters.user): + self.logger.exception("need to specify user for domaintools expert in runtime.conf. Exiting") + self.stop() + if (not self.parameters.password): + self.logger.exception("need to specify password for the user for domaintools expert in runtime.conf. Exiting") + self.stop() + self.api = API(self.parameters.user, self.parameters.password) + + def domaintools_get_score(self, fqdn): + + if fqdn: + resp = self.api.reputation(fqdn, include_reason=False) # don't include a reason in the JSON response + try: + score = resp['risk_score'] + except exceptions.NotFoundException: + score = None + except exceptions.BadRequestException: + score = None + return score + + def process(self): + event = self.receive_message() + + for key in ["source.", "destination."]: + key_fqdn = key + "fqdn" + if key_fqdn not in event: + continue # can't query if we don't have a domain name + score = self.domaintools_get_score(key_fqdn) + if score: + event.add("extra.domaintools_score", score, raise_failure=False) + + self.send_message(event) + self.acknowledge_message() + + +BOT = DomaintoolsExpertBot diff --git a/intelmq/tests/bots/experts/domaintools/test_expert.py b/intelmq/tests/bots/experts/domaintools/test_expert.py new file mode 100644 index 000000000..08a8f7c5c --- /dev/null +++ b/intelmq/tests/bots/experts/domaintools/test_expert.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +""" +Testing GethostbynameExpertBot. +""" + +import unittest + +import intelmq.lib.test as test +from intelmq.bots.experts.gethostbyname.expert import DomaintoolsExpertBot + +EXAMPLE_INPUT = {"__type": "Event", + "source.fqdn": "google.com", + "time.observation": "2015-01-01T00:00:00+00:00" + } +EXAMPLE_OUTPUT = {"__type": "Event", + "source.fqdn": "example.com", + "extra.domaintools_score": 0, + "time.observation": "2015-01-01T00:00:00+00:00" + } +NONEXISTING_INPUT = {"__type": "Event", + "source.fqdn": "example.invalid", + "destination.fqdn": "example.invalid", + "time.observation": "2015-01-01T00:00:00+00:00" + } + + +@test.skip_internet() +class TestDomaintoolsExpertBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for DomaintoolsExpertBot. + """ + + @classmethod + def set_bot(self): + self.bot_reference = DomaintoolsExpertBot + + def test_existing(self): + self.input_message = EXAMPLE_INPUT + self.run_bot() + self.assertMessageEqual(0, EXAMPLE_OUTPUT) + + def test_non_existing(self): + self.input_message = NONEXISTING_INPUT + self.run_bot() + self.assertMessageEqual(0, NONEXISTING_INPUT) + +if __name__ == '__main__': # pragma: no cover + unittest.main() From 5c61f0a959aed3de3ac4255c5a728a33b9f92fd1 Mon Sep 17 00:00:00 2001 From: aaronkaplan Date: Sun, 11 Jun 2017 15:41:00 -0400 Subject: [PATCH 2/4] add requirement for domaintools_api --- intelmq/bots/experts/domaintools/REQUIREMENTS.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 intelmq/bots/experts/domaintools/REQUIREMENTS.txt diff --git a/intelmq/bots/experts/domaintools/REQUIREMENTS.txt b/intelmq/bots/experts/domaintools/REQUIREMENTS.txt new file mode 100644 index 000000000..322350615 --- /dev/null +++ b/intelmq/bots/experts/domaintools/REQUIREMENTS.txt @@ -0,0 +1 @@ +domaintools_api==0.1.7 From 5348b39cf94fe914aca7750f940492ff341f47ef Mon Sep 17 00:00:00 2001 From: aaronkaplan Date: Sun, 11 Jun 2017 15:45:29 -0400 Subject: [PATCH 3/4] add BOTS config for domaintools --- intelmq/bots/BOTS | 8 ++++++++ intelmq/bots/experts/domaintools/expert.py | 17 ++++++++++------- .../tests/bots/experts/domaintools/__init__.py | 0 .../bots/experts/domaintools/test_expert.py | 7 ++++--- 4 files changed, 22 insertions(+), 10 deletions(-) create mode 100644 intelmq/tests/bots/experts/domaintools/__init__.py diff --git a/intelmq/bots/BOTS b/intelmq/bots/BOTS index b73fd7910..0b2f23246 100644 --- a/intelmq/bots/BOTS +++ b/intelmq/bots/BOTS @@ -476,6 +476,14 @@ "redis_cache_ttl": "86400" } }, + "Domaintools": { + "description": "Domaintools expert is a bot which queries domaintools.com for a scoring of a domain name", + "module": "intelmq.bots.experts.domaintools.expert", + "parameters": { + "user": "", + "password": "" + } + }, "Field Reducer": { "description": "The field reducer bot is capable of removing fields from events.", "module": "intelmq.bots.experts.field_reducer.expert", diff --git a/intelmq/bots/experts/domaintools/expert.py b/intelmq/bots/experts/domaintools/expert.py index 687e20aac..52741b542 100644 --- a/intelmq/bots/experts/domaintools/expert.py +++ b/intelmq/bots/experts/domaintools/expert.py @@ -26,27 +26,30 @@ def init(self): self.api = API(self.parameters.user, self.parameters.password) def domaintools_get_score(self, fqdn): - + score = None if fqdn: - resp = self.api.reputation(fqdn, include_reason=False) # don't include a reason in the JSON response + resp = self.api.reputation(fqdn, include_reasons=False) # don't include a reason in the JSON response + try: score = resp['risk_score'] except exceptions.NotFoundException: - score = None + score = None except exceptions.BadRequestException: - score = None + score = None return score def process(self): event = self.receive_message() + extra = {} for key in ["source.", "destination."]: key_fqdn = key + "fqdn" if key_fqdn not in event: continue # can't query if we don't have a domain name - score = self.domaintools_get_score(key_fqdn) - if score: - event.add("extra.domaintools_score", score, raise_failure=False) + score = self.domaintools_get_score(event.get(key_fqdn)) + if score is not None: + extra["domaintools_score"] = score + event.add("extra", extra) self.send_message(event) self.acknowledge_message() diff --git a/intelmq/tests/bots/experts/domaintools/__init__.py b/intelmq/tests/bots/experts/domaintools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/intelmq/tests/bots/experts/domaintools/test_expert.py b/intelmq/tests/bots/experts/domaintools/test_expert.py index 08a8f7c5c..9d214d142 100644 --- a/intelmq/tests/bots/experts/domaintools/test_expert.py +++ b/intelmq/tests/bots/experts/domaintools/test_expert.py @@ -6,15 +6,15 @@ import unittest import intelmq.lib.test as test -from intelmq.bots.experts.gethostbyname.expert import DomaintoolsExpertBot +from intelmq.bots.experts.domaintools.expert import DomaintoolsExpertBot EXAMPLE_INPUT = {"__type": "Event", "source.fqdn": "google.com", "time.observation": "2015-01-01T00:00:00+00:00" } EXAMPLE_OUTPUT = {"__type": "Event", - "source.fqdn": "example.com", - "extra.domaintools_score": 0, + "source.fqdn": "google.com", + "extra": '{"domaintools_score": 0}', "time.observation": "2015-01-01T00:00:00+00:00" } NONEXISTING_INPUT = {"__type": "Event", @@ -33,6 +33,7 @@ class TestDomaintoolsExpertBot(test.BotTestCase, unittest.TestCase): @classmethod def set_bot(self): self.bot_reference = DomaintoolsExpertBot + self.sysconfig = {'user': 'mkendrick_first2017', 'password': 'c0e4e-e2527-dc6af-824a4-229d5'} def test_existing(self): self.input_message = EXAMPLE_INPUT From 9e9fa966142e148d3009f20f5516d6a1db8c4776 Mon Sep 17 00:00:00 2001 From: SYNchroACK Date: Thu, 14 Dec 2017 09:16:51 +0000 Subject: [PATCH 4/4] ENH: required improvements on domaintools expert --- intelmq/bots/experts/domaintools/expert.py | 67 ++++++++++++------- .../bots/experts/domaintools/test_expert.py | 2 +- 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/intelmq/bots/experts/domaintools/expert.py b/intelmq/bots/experts/domaintools/expert.py index 52741b542..284dd169a 100644 --- a/intelmq/bots/experts/domaintools/expert.py +++ b/intelmq/bots/experts/domaintools/expert.py @@ -14,29 +14,41 @@ class DomaintoolsExpertBot(Bot): def init(self): self.logger.info("Loading Domaintools expert.") - if (not API): - self.logger.exception("need to have the domaintools API installed. See https://github.com/domaintools/python_api") - self.stop() - if (not self.parameters.user): - self.logger.exception("need to specify user for domaintools expert in runtime.conf. Exiting") - self.stop() - if (not self.parameters.password): - self.logger.exception("need to specify password for the user for domaintools expert in runtime.conf. Exiting") - self.stop() + + if not API: + raise ValueError("need to have the domaintools API installed. See https://github.com/domaintools/python_api.") + + if not self.parameters.user: + raise ValueError("need to specify user for domaintools expert in runtime.conf.") + + if not self.parameters.password: + raise ValueError("need to specify password for the user for domaintools expert in runtime.conf.") + self.api = API(self.parameters.user, self.parameters.password) - def domaintools_get_score(self, fqdn): - score = None - if fqdn: - resp = self.api.reputation(fqdn, include_reasons=False) # don't include a reason in the JSON response + if not self.valid_credentials(): + raise ValueError("invalid credentials found in runtime.conf.") - try: - score = resp['risk_score'] - except exceptions.NotFoundException: - score = None - except exceptions.BadRequestException: - score = None - return score + def valid_credentials(self): + resp = self.api.reputation(fqdn, include_reasons=False) + try: + resp['risk_score'] + return True + except exceptions.NotAuthorizedException: + return False + + def get_score(self, fqdn): + # don't include a reason in the JSON response + resp = self.api.reputation(fqdn, include_reasons=False) + + try: + score = resp['risk_score'] + except exceptions.NotFoundException: + score = None + except exceptions.BadRequestException: + raise + + return score def process(self): event = self.receive_message() @@ -44,12 +56,17 @@ def process(self): for key in ["source.", "destination."]: key_fqdn = key + "fqdn" + if key_fqdn not in event: - continue # can't query if we don't have a domain name - score = self.domaintools_get_score(event.get(key_fqdn)) - if score is not None: - extra["domaintools_score"] = score - event.add("extra", extra) + continue + + score = self.get_score(event.get(key_fqdn)) + + if score: + extra["domaintools_score_" + key_fqdn] = score + + extra.update(event.get("extra")) + event.update("extra", extra) self.send_message(event) self.acknowledge_message() diff --git a/intelmq/tests/bots/experts/domaintools/test_expert.py b/intelmq/tests/bots/experts/domaintools/test_expert.py index 9d214d142..626acf476 100644 --- a/intelmq/tests/bots/experts/domaintools/test_expert.py +++ b/intelmq/tests/bots/experts/domaintools/test_expert.py @@ -33,7 +33,7 @@ class TestDomaintoolsExpertBot(test.BotTestCase, unittest.TestCase): @classmethod def set_bot(self): self.bot_reference = DomaintoolsExpertBot - self.sysconfig = {'user': 'mkendrick_first2017', 'password': 'c0e4e-e2527-dc6af-824a4-229d5'} + self.sysconfig = {'user': 'example01', 'password': 'mysecret'} def test_existing(self): self.input_message = EXAMPLE_INPUT