Dependency Checking (#75)

* Adds req testing methodology, needs fixes

* Improves dependency exception handling

* Better meets_requirements implementation

Still need to adjust tests to fake installation

* Changes to exception boolean to enable tool check

tests and class variables modified for new tool check

* Adjust test_get_scans to use appropriate variable

* Adds Go requirement where relevant

* Adds missing scan dependencies

* Add clarification to error message

pipeline/recon-pipeline.py

@@ -298,10 +298,14 @@ def do_scan(self, args):
         # get_scans() returns mapping of {classname: [modulename, ...]} in the recon module
         # each classname corresponds to a potential recon-pipeline command, i.e. AmassScan, GobusterScan ...
         scans = get_scans()
-
         # command is a list that will end up looking something like what's below
         # luigi --module pipeline.recon.web.webanalyze WebanalyzeScan --target abc.com --top-ports 100 --interface eth0
-        command = ["luigi", "--module", scans.get(args.scantype)[0]]
+        try:
+            command = ["luigi", "--module", scans.get(args.scantype)[0]]
+        except TypeError:
+            return self.poutput(
+                style(f"[!] {args.scantype} or one of its dependencies is not installed", fg="bright_red")
+            )
 
         tgt_file_path = None
         if args.target:

pipeline/recon/amass.py

@@ -9,7 +9,7 @@
 import pipeline.models.db_manager
 from ..tools import tools
 from .targets import TargetList
-from .helpers import get_tool_state
+from .helpers import meets_requirements
 from ..models.target_model import Target
 
 
@@ -43,21 +43,14 @@ class AmassScan(luigi.Task):
     """
 
     exempt_list = luigi.Parameter(default="")
+    requirements = ["go", "amass"]
+    exception = True
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.db_mgr = pipeline.models.db_manager.DBManager(db_location=self.db_location)
         self.results_subfolder = (Path(self.results_dir) / "amass-results").expanduser().resolve()
 
-    @staticmethod
-    def meets_requirements():
-        """ Reports whether or not this scan's needed tool(s) are installed or not """
-        needs = ["amass"]
-        tools = get_tool_state()
-
-        if tools:
-            return all([tools.get(x).get("installed") is True for x in needs])
-
     def requires(self):
         """ AmassScan depends on TargetList to run.
 
@@ -66,6 +59,7 @@ def requires(self):
         Returns:
             luigi.ExternalTask - TargetList
         """
+        meets_requirements(self.requirements, self.exception)
         args = {"target_file": self.target_file, "results_dir": self.results_dir, "db_location": self.db_location}
         return TargetList(**args)
 
@@ -89,7 +83,6 @@ def run(self):
         Returns:
             list: list of options/arguments, beginning with the name of the executable to run
         """
-
         self.results_subfolder.mkdir(parents=True, exist_ok=True)
 
         hostnames = self.db_mgr.get_all_hostnames()

pipeline/recon/helpers.py

@@ -6,11 +6,29 @@
 import importlib
 import ipaddress
 from pathlib import Path
+from cmd2.ansi import style
+
 from collections import defaultdict
 
 from ..recon.config import defaults
 
 
+def meets_requirements(requirements, exception):
+    """ Determine if tools required to perform task are installed. """
+    tools = get_tool_state()
+
+    for tool in requirements:
+        if not tools.get(tool).get("installed"):
+            if exception:
+                raise RuntimeError(
+                    style(f"[!!] {tool} is not installed, and is required to run this scan", fg="bright_red")
+                )
+            else:
+                return False
+
+    return True
+
+
 def get_tool_state() -> typing.Union[dict, None]:
     """ Load current tool state from disk. """
     tools = Path(defaults.get("tools-dir")) / ".tool-dict.pkl"
@@ -60,7 +78,9 @@ def get_scans():
                     # final check, this ensures that the tools necessary to AT LEAST run this scan are present
                     # does not consider upstream dependencies
                     try:
-                        if not sub_obj.meets_requirements():
+                        requirements = sub_obj.requirements
+                        exception = False  # let meets_req know we want boolean result
+                        if not meets_requirements(requirements, exception):
                             continue
                     except AttributeError:
                         # some scan's haven't implemented meets_requirements yet, silently allow them through

pipeline/recon/masscan.py

@@ -14,7 +14,7 @@
 from ..models.port_model import Port
 from ..models.ip_address_model import IPAddress
 
-from .helpers import get_tool_state
+from .helpers import meets_requirements
 from .config import top_tcp_ports, top_udp_ports, defaults, web_ports
 
 
@@ -58,21 +58,14 @@ class MasscanScan(luigi.Task):
     interface = luigi.Parameter(default=defaults.get("masscan-iface"))
     top_ports = luigi.IntParameter(default=0)  # IntParameter -> top_ports expected as int
     ports = luigi.Parameter(default="")
+    requirements = ["masscan"]
+    exception = True
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.db_mgr = pipeline.models.db_manager.DBManager(db_location=self.db_location)
         self.results_subfolder = (Path(self.results_dir) / "masscan-results").expanduser().resolve()
 
-    @staticmethod
-    def meets_requirements():
-        """ Reports whether or not this scan's needed tool(s) are installed or not """
-        needs = ["masscan"]
-        tools = get_tool_state()
-
-        if tools:
-            return all([tools.get(x).get("installed") is True for x in needs])
-
     def output(self):
         """ Returns the target output for this task.
 
@@ -91,6 +84,7 @@ def run(self):
         Returns:
             list: list of options/arguments, beginning with the name of the executable to run
         """
+        meets_requirements(self.requirements, self.exception)
         if not self.ports and not self.top_ports:
             # need at least one, can't be put into argparse scanner because things like amass don't require ports option
             logging.error("Must specify either --top-ports or --ports.")

pipeline/recon/nmap.py

@@ -3,6 +3,8 @@
 import subprocess
 import concurrent.futures
 from pathlib import Path
+from shutil import which
+from cmd2.ansi import style
 
 import luigi
 import sqlalchemy
@@ -13,10 +15,9 @@
 import pipeline.models.db_manager
 from .masscan import ParseMasscanOutput
 from .config import defaults
-from .helpers import get_ip_address_version, is_ip_address
+from .helpers import get_ip_address_version, is_ip_address, meets_requirements
 
 from ..tools import tools
-from .helpers import get_tool_state
 from ..models.port_model import Port
 from ..models.nse_model import NSEResult
 from ..models.target_model import Target
@@ -58,6 +59,8 @@ class ThreadedNmapScan(luigi.Task):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        if not which("nmap"):
+            raise RuntimeError(style("[!] nmap is not installed", fg="bright_red"))
         self.db_mgr = pipeline.models.db_manager.DBManager(db_location=self.db_location)
         self.results_subfolder = (Path(self.results_dir) / "nmap-results").expanduser().resolve()
 
@@ -238,19 +241,13 @@ class SearchsploitScan(luigi.Task):
         results_dir: specifies the directory on disk to which all Task results are written *Required by upstream Task*
     """
 
+    requirements = ["searchsploit"]
+    exception = True
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.db_mgr = pipeline.models.db_manager.DBManager(db_location=self.db_location)
 
-    @staticmethod
-    def meets_requirements():
-        """ Reports whether or not this scan's needed tool(s) are installed or not """
-        needs = ["searchsploit"]
-        tools = get_tool_state()
-
-        if tools:
-            return all([tools.get(x).get("installed") is True for x in needs])
-
     def requires(self):
         """ Searchsploit depends on ThreadedNmap to run.
 
@@ -261,6 +258,7 @@ def requires(self):
         Returns:
             luigi.Task - ThreadedNmap
         """
+        meets_requirements(self.requirements, self.exception)
         args = {
             "rate": self.rate,
             "ports": self.ports,

pipeline/recon/web/aquatone.py

@@ -13,7 +13,7 @@
 from ...tools import tools
 
 import pipeline.models.db_manager
-from ..helpers import get_tool_state
+from ..helpers import meets_requirements
 from ...models.port_model import Port
 from ...models.header_model import Header
 from ...models.endpoint_model import Endpoint
@@ -58,21 +58,14 @@ class AquatoneScan(luigi.Task):
 
     threads = luigi.Parameter(default=defaults.get("threads", ""))
     scan_timeout = luigi.Parameter(default=defaults.get("aquatone-scan-timeout", ""))
+    requirements = ["aquatone", "masscan"]
+    exception = True
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.db_mgr = pipeline.models.db_manager.DBManager(db_location=self.db_location)
         self.results_subfolder = Path(self.results_dir) / "aquatone-results"
 
-    @staticmethod
-    def meets_requirements():
-        """ Reports whether or not this scan's needed tool(s) are installed or not """
-        needs = ["aquatone"]
-        tools = get_tool_state()
-
-        if tools:
-            return all([tools.get(x).get("installed") is True for x in needs])
-
     def requires(self):
         """ AquatoneScan depends on GatherWebTargets to run.
 
@@ -82,6 +75,7 @@ def requires(self):
         Returns:
             luigi.Task - GatherWebTargets
         """
+        meets_requirements(self.requirements, self.exception)
         args = {
             "results_dir": self.results_dir,
             "rate": self.rate,

pipeline/recon/web/gobuster.py

@@ -12,7 +12,7 @@
 import pipeline.models.db_manager
 from ...tools import tools
 from ..config import defaults
-from ..helpers import get_tool_state
+from ..helpers import meets_requirements
 from .targets import GatherWebTargets
 from ...models.endpoint_model import Endpoint
 from ..helpers import get_ip_address_version, is_ip_address
@@ -59,21 +59,14 @@ class GobusterScan(luigi.Task):
     threads = luigi.Parameter(default=defaults.get("threads"))
     wordlist = luigi.Parameter(default=defaults.get("gobuster-wordlist"))
     extensions = luigi.Parameter(default=defaults.get("gobuster-extensions"))
+    requirements = ["recursive-gobuster", "go", "gobuster", "masscan"]
+    exception = True
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.db_mgr = pipeline.models.db_manager.DBManager(db_location=self.db_location)
         self.results_subfolder = Path(self.results_dir) / "gobuster-results"
 
-    @staticmethod
-    def meets_requirements():
-        """ Reports whether or not this scan's needed tool(s) are installed or not """
-        needs = ["recursive-gobuster", "gobuster"]
-        tools = get_tool_state()
-
-        if tools:
-            return all([tools.get(x).get("installed") is True for x in needs])
-
     def requires(self):
         """ GobusterScan depends on GatherWebTargets to run.
 
@@ -83,6 +76,7 @@ def requires(self):
         Returns:
             luigi.Task - GatherWebTargets
         """
+        meets_requirements(self.requirements, self.exception)
         args = {
             "results_dir": self.results_dir,
             "rate": self.rate,

pipeline/recon/web/subdomain_takeover.py

@@ -10,7 +10,7 @@
 import pipeline.models.db_manager
 from ...tools import tools
 from ..config import defaults
-from ..helpers import get_tool_state
+from ..helpers import meets_requirements
 from .targets import GatherWebTargets
 
 
@@ -47,21 +47,15 @@ class TKOSubsScan(luigi.Task):
         results_dir: specifes the directory on disk to which all Task results are written *Required by upstream Task*
     """
 
+    requirements = ["go", "tko-subs", "masscan"]
+    exception = True
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.db_mgr = pipeline.models.db_manager.DBManager(db_location=self.db_location)
         self.results_subfolder = (Path(self.results_dir) / "tkosubs-results").expanduser().resolve()
         self.output_file = self.results_subfolder / "tkosubs.csv"
 
-    @staticmethod
-    def meets_requirements():
-        """ Reports whether or not this scan's needed tool(s) are installed or not """
-        needs = ["tko-subs"]
-        tools = get_tool_state()
-
-        if tools:
-            return all([tools.get(x).get("installed") is True for x in needs])
-
     def requires(self):
         """ TKOSubsScan depends on GatherWebTargets to run.
 
@@ -71,6 +65,7 @@ def requires(self):
         Returns:
             luigi.Task - GatherWebTargets
         """
+        meets_requirements(self.requirements, self.exception)
         args = {
             "results_dir": self.results_dir,
             "rate": self.rate,
@@ -177,22 +172,15 @@ class SubjackScan(luigi.Task):
     """
 
     threads = luigi.Parameter(default=defaults.get("threads"))
+    requirements = ["go", "subjack", "masscan"]
+    exception = True
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.db_mgr = pipeline.models.db_manager.DBManager(db_location=self.db_location)
         self.results_subfolder = (Path(self.results_dir) / "subjack-results").expanduser().resolve()
         self.output_file = self.results_subfolder / "subjack.txt"
 
-    @staticmethod
-    def meets_requirements():
-        """ Reports whether or not this scan's needed tool(s) are installed or not """
-        needs = ["subjack"]
-        tools = get_tool_state()
-
-        if tools:
-            return all([tools.get(x).get("installed") is True for x in needs])
-
     def requires(self):
         """ SubjackScan depends on GatherWebTargets to run.
 
@@ -202,6 +190,7 @@ def requires(self):
         Returns:
             luigi.Task - GatherWebTargets
         """
+        meets_requirements(self.requirements, self.exception)
         args = {
             "results_dir": self.results_dir,
             "rate": self.rate,