From 003658aa66ea4a9ec43cbe2b84c30bd02c5fc0c8 Mon Sep 17 00:00:00 2001 From: epi <43392618+epi052@users.noreply.github.com> Date: Thu, 27 Aug 2020 20:27:43 -0500 Subject: [PATCH] removed tool_dict dependency, tests incomplete (#93) * removed tool_dict dependency * updated tests * updated go version * added defaults for failing iteration during tool installation * Update pythonapp.yml * updated docs --- .github/workflows/pythonapp.yml | 10 +++--- .pre-commit-config.yaml | 5 +-- README.md | 3 +- docs/overview/installation.rst | 2 +- pipeline/recon-pipeline.py | 36 +++---------------- pipeline/recon/helpers.py | 15 ++------ pipeline/tools/amass.yaml | 1 - pipeline/tools/aquatone.yaml | 1 - pipeline/tools/exploitdb.yaml | 1 - pipeline/tools/go.yaml | 3 +- pipeline/tools/gobuster.yaml | 1 - pipeline/tools/loader.py | 4 +++ pipeline/tools/luigi-service.yaml | 4 +-- pipeline/tools/masscan.yaml | 1 - pipeline/tools/recursive-gobuster.yaml | 1 - pipeline/tools/searchsploit.yaml | 1 - pipeline/tools/seclists.yaml | 1 - pipeline/tools/subjack.yaml | 1 - pipeline/tools/tko-subs.yaml | 1 - pipeline/tools/waybackurls.yaml | 1 - pipeline/tools/webanalyze.yaml | 1 - tests/test_recon/test_helpers.py | 34 ++++++------------ tests/test_shell/test_recon_pipeline_shell.py | 30 ++++++++-------- .../test_tools_install_command.py | 2 +- tests/test_web/test_gobuster.py | 4 ++- 25 files changed, 55 insertions(+), 109 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index ec73d1d..c25855f 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -1,6 +1,6 @@ name: recon-pipeline build -on: [push, pull_request] +on: [push] jobs: lint: @@ -24,9 +24,9 @@ jobs: # stop the build if there are Python syntax errors or undefined names pipenv run flake8 . --count - name: Check code formatting with black - uses: lgeiger/black-action@master - with: - args: ". --check" + run: | + pipenv install "black==19.10b0" + pipenv run black -l 120 --check . test-shell: @@ -126,4 +126,4 @@ jobs: - name: Test with pytest run: | pipenv install pytest cmd2 luigi sqlalchemy python-libnmap - pipenv run python -m pytest -vv --show-capture=all tests/test_tools_install \ No newline at end of file + pipenv run python -m pytest -vv --show-capture=all tests/test_tools_install diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 19277c5..8e20c1c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,8 +16,9 @@ repos: - repo: local hooks: - id: tests + pass_filenames: false name: run tests - entry: pytest + entry: python language: system types: [python] - args: ['tests/test_web', 'tests/test_recon', 'tests/test_shell', 'tests/test_models'] + args: ['-m', 'pytest', 'tests/test_web', 'tests/test_recon', 'tests/test_shell', 'tests/test_models'] diff --git a/README.md b/README.md index 13e9133..2d4fc01 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,8 @@ After installing the python dependencies, the `recon-pipeline` shell provides it Individual tools may be installed by running `tools install TOOLNAME` where `TOOLNAME` is one of the known tools that make up the pipeline. -The installer maintains a (naive) list of installed tools at `~/.local/recon-pipeline/tools/.tool-dict.pkl`. The installer in no way attempts to be a package manager. It knows how to execute the steps necessary to install and remove its tools. Beyond that, it's like Jon Snow, **it knows nothing**. +The installer does not maintain state. In order to determine whether a tool is installed or not, it checks the `path` variable defined in the tool's .yaml file. The installer in no way attempts to be a package manager. It knows how to execute the steps necessary to install and remove its tools. Beyond that, it's +like Jon Snow, **it knows nothing**. [![asciicast](https://asciinema.org/a/343745.svg)](https://asciinema.org/a/343745) diff --git a/docs/overview/installation.rst b/docs/overview/installation.rst index b40ea1c..1e19505 100644 --- a/docs/overview/installation.rst +++ b/docs/overview/installation.rst @@ -59,7 +59,7 @@ A simple ``tools install all`` will handle all installation steps. Installation Individual tools may be installed by running ``tools install TOOLNAME`` where ``TOOLNAME`` is one of the known tools that make up the pipeline. -The installer maintains a (naive) list of installed tools at ``~/.local/recon-pipeline/tools/.tool-dict.pkl``. The installer in no way attempts to be a package manager. It knows how to execute the steps necessary to install and remove its tools. Beyond that, it's +The installer does not maintain state. In order to determine whether a tool is installed or not, it checks the `path` variable defined in the tool's .yaml file. The installer in no way attempts to be a package manager. It knows how to execute the steps necessary to install and remove its tools. Beyond that, it's like Jon Snow, **it knows nothing**. Current tool status can be viewed using ``tools list``. Tools can also be uninstalled using the ``tools uninstall all`` command. It is also possible to individually uninstall them in the same manner as shown above. diff --git a/pipeline/recon-pipeline.py b/pipeline/recon-pipeline.py index fe643b1..f1df7af 100755 --- a/pipeline/recon-pipeline.py +++ b/pipeline/recon-pipeline.py @@ -5,7 +5,6 @@ import time import shlex import shutil -import pickle import tempfile import textwrap import selectors @@ -109,7 +108,7 @@ def stop(self): # close any fds that were registered and still haven't been unregistered for key in selector.get_map(): - selector.get_key(key).fileobj.close() + selector.get_key(key).fileobj.close() # pragma: no cover def stopped(self): """ Helper to determine whether the SelectorThread's Event is set or not. """ @@ -117,7 +116,7 @@ def stopped(self): def run(self): """ Run thread that executes a select loop; handles async stdout/stderr processing of subprocesses. """ - while not self.stopped(): + while not self.stopped(): # pragma: no cover for k, mask in selector.select(): callback = k.data callback(k.fileobj) @@ -346,15 +345,6 @@ def do_scan(self, args): def _get_dict(self): """Retrieves tool dict if available""" - - # imported tools variable is in global scope, and we reassign over it later - global tools - - persistent_tool_dict = self.tools_dir / ".tool-dict.pkl" - - if persistent_tool_dict.exists(): - tools = pickle.loads(persistent_tool_dict.read_bytes()) - return tools def _finalize_tool_action(self, tool: str, tool_dict: dict, return_values: List[int], action: ToolActions): @@ -387,16 +377,9 @@ def _finalize_tool_action(self, tool: str, tool_dict: dict, return_values: List[ ) ) - # store any tool installs/failures (back) to disk - persistent_tool_dict = self.tools_dir / ".tool-dict.pkl" - - pickle.dump(tool_dict, persistent_tool_dict.open("wb")) - def tools_install(self, args): """ Install any/all of the libraries/tools necessary to make the recon-pipeline function. """ - tools = self._get_dict() - if args.tool == "all": # show all tools have been queued for installation [ @@ -424,16 +407,6 @@ def tools_install(self, args): # install the dependency before continuing with installation self.do_tools(f"install {dependency}") - # this prevents a stale copy of tools when dependency installs alter the state - # ex. - # amass (which depends on go) grabs copy of tools (go installed false) - # amass calls install with go as the arg - # go grabs a copy of tools - # go is installed and state is saved (go installed true) - # recursion goes back to amass call (go installed false due to stale tools data) - # amass installs and re-saves go's state as installed=false - tools = self._get_dict() - if tools.get(args.tool).get("installed"): return self.poutput(style(f"[!] {args.tool} is already installed.", fg="yellow")) else: @@ -448,7 +421,7 @@ def tools_install(self, args): if addl_env_vars is not None: addl_env_vars.update(dict(os.environ)) - for command in tools.get(args.tool).get("install_commands"): + for command in tools.get(args.tool, {}).get("install_commands", []): # run all commands required to install the tool # print each command being run @@ -478,7 +451,6 @@ def tools_install(self, args): def tools_uninstall(self, args): """ Uninstall any/all of the libraries/tools used by recon-pipeline""" - tools = self._get_dict() if args.tool == "all": # show all tools have been queued for installation @@ -525,7 +497,7 @@ def tools_reinstall(self, args): def tools_list(self, args): """ List status of pipeline tools """ - for key, value in self._get_dict().items(): + for key, value in tools.items(): status = [style(":Missing:", fg="bright_magenta"), style("Installed", fg="bright_green")] self.poutput(style(f"[{status[value.get('installed')]}] - {value.get('path') or key}")) diff --git a/pipeline/recon/helpers.py b/pipeline/recon/helpers.py index ed19c51..aed6ca9 100644 --- a/pipeline/recon/helpers.py +++ b/pipeline/recon/helpers.py @@ -1,6 +1,4 @@ import sys -import pickle -import typing import inspect import pkgutil import importlib @@ -10,13 +8,12 @@ from collections import defaultdict -from ..recon.config import defaults +from ..tools import tools def meets_requirements(requirements, exception): """ Determine if tools required to perform task are installed. """ - tools = get_tool_state() - + print(tools.items()) for tool in requirements: if not tools.get(tool).get("installed"): if exception: @@ -29,14 +26,6 @@ def meets_requirements(requirements, exception): return True -def get_tool_state() -> typing.Union[dict, None]: - """ Load current tool state from disk. """ - tools = Path(defaults.get("tools-dir")) / ".tool-dict.pkl" - - if tools.exists(): - return pickle.loads(tools.read_bytes()) - - def get_scans(): """ Iterates over the recon package and its modules to find all of the classes that end in [Ss]can. diff --git a/pipeline/tools/amass.yaml b/pipeline/tools/amass.yaml index 4036742..98988d2 100644 --- a/pipeline/tools/amass.yaml +++ b/pipeline/tools/amass.yaml @@ -1,4 +1,3 @@ -installed: false dependencies: [go] go: &gotool !get_tool_path "{go[path]}" path: &path !join_path [!get_default "{gopath}", "bin/amass"] diff --git a/pipeline/tools/aquatone.yaml b/pipeline/tools/aquatone.yaml index 38a251b..ca10059 100644 --- a/pipeline/tools/aquatone.yaml +++ b/pipeline/tools/aquatone.yaml @@ -1,4 +1,3 @@ -installed: false tools: &tools !get_default "{tools-dir}" path: &path !join_path [*tools, aquatone] diff --git a/pipeline/tools/exploitdb.yaml b/pipeline/tools/exploitdb.yaml index 712505a..5772e37 100644 --- a/pipeline/tools/exploitdb.yaml +++ b/pipeline/tools/exploitdb.yaml @@ -1,3 +1,2 @@ -installed: true tools: &tools !get_default "{tools-dir}" path: !join_path [*tools, exploitdb] \ No newline at end of file diff --git a/pipeline/tools/go.yaml b/pipeline/tools/go.yaml index de58172..ef86eef 100644 --- a/pipeline/tools/go.yaml +++ b/pipeline/tools/go.yaml @@ -1,7 +1,6 @@ -installed: false bashrc: &bashrc !join_path [!get_default "{home}", .bashrc] path: &gotool !join_path [!get_default "{goroot}", go/bin/go] -dlpath: &dlpath !join_empty ["https://dl.google.com/go/go1.14.6.linux-", !get_default "{arch}", ".tar.gz"] +dlpath: &dlpath !join_empty ["https://dl.google.com/go/go1.14.7.linux-", !get_default "{arch}", ".tar.gz"] install_commands: - !join ["wget -q", *dlpath, "-O /tmp/go.tar.gz"] diff --git a/pipeline/tools/gobuster.yaml b/pipeline/tools/gobuster.yaml index 35edbc4..c83f9c8 100644 --- a/pipeline/tools/gobuster.yaml +++ b/pipeline/tools/gobuster.yaml @@ -1,4 +1,3 @@ -installed: false dependencies: [go, seclists] go: &gotool !get_tool_path "{go[path]}" path: &path !join_path [!get_default "{gopath}", bin/gobuster] diff --git a/pipeline/tools/loader.py b/pipeline/tools/loader.py index 01f608d..273032d 100644 --- a/pipeline/tools/loader.py +++ b/pipeline/tools/loader.py @@ -1,3 +1,4 @@ +import uuid import yaml from pathlib import Path @@ -59,3 +60,6 @@ def load_yaml(file): for file in definitions.iterdir(): if file.name.endswith(".yaml") and file.name.replace(".yaml", "") not in tools: load_yaml(file) + +for tool_name, tool_definition in tools.items(): + tool_definition["installed"] = Path(tool_definition.get("path", f"/{uuid.uuid4()}")).exists() diff --git a/pipeline/tools/luigi-service.yaml b/pipeline/tools/luigi-service.yaml index ff974de..2df9d2d 100644 --- a/pipeline/tools/luigi-service.yaml +++ b/pipeline/tools/luigi-service.yaml @@ -1,9 +1,9 @@ -installed: false project-dir: &proj !get_default "{project-dir}" service-file: &svcfile !join_path [*proj, luigid.service] +path: &path /lib/systemd/system/luigid.service install_commands: -- !join [sudo cp, *svcfile, /lib/systemd/system/luigid.service] +- !join [sudo cp, *svcfile, *path] - !join [sudo cp, $(which luigid), /usr/local/bin] - sudo systemctl daemon-reload - sudo systemctl start luigid.service diff --git a/pipeline/tools/masscan.yaml b/pipeline/tools/masscan.yaml index 4078353..856d3e1 100644 --- a/pipeline/tools/masscan.yaml +++ b/pipeline/tools/masscan.yaml @@ -1,4 +1,3 @@ -installed: false tools: &tools !get_default "{tools-dir}" path: &path !join_path [*tools, masscan] diff --git a/pipeline/tools/recursive-gobuster.yaml b/pipeline/tools/recursive-gobuster.yaml index 8b267b2..cb87e04 100644 --- a/pipeline/tools/recursive-gobuster.yaml +++ b/pipeline/tools/recursive-gobuster.yaml @@ -1,4 +1,3 @@ -installed: false dependencies: [gobuster] tools: &tools !get_default "{tools-dir}" path: &path !join_path [*tools, recursive-gobuster/recursive-gobuster.pyz] diff --git a/pipeline/tools/searchsploit.yaml b/pipeline/tools/searchsploit.yaml index 0785c9b..cca0735 100644 --- a/pipeline/tools/searchsploit.yaml +++ b/pipeline/tools/searchsploit.yaml @@ -1,4 +1,3 @@ -installed: false dependencies: [exploitdb] home: &home !get_default "{home}" tools: &tools !get_default "{tools-dir}" diff --git a/pipeline/tools/seclists.yaml b/pipeline/tools/seclists.yaml index 995f916..036a2d2 100644 --- a/pipeline/tools/seclists.yaml +++ b/pipeline/tools/seclists.yaml @@ -1,4 +1,3 @@ -installed: false tools: &tools !get_default "{tools-dir}" path: &path !join_path [*tools, seclists] diff --git a/pipeline/tools/subjack.yaml b/pipeline/tools/subjack.yaml index 2177aab..19ef943 100644 --- a/pipeline/tools/subjack.yaml +++ b/pipeline/tools/subjack.yaml @@ -1,4 +1,3 @@ -installed: false dependencies: [go] go: &gotool !get_tool_path "{go[path]}" path: &path !join_path [!get_default "{gopath}", bin/subjack] diff --git a/pipeline/tools/tko-subs.yaml b/pipeline/tools/tko-subs.yaml index 35b40f4..1c69f02 100644 --- a/pipeline/tools/tko-subs.yaml +++ b/pipeline/tools/tko-subs.yaml @@ -1,4 +1,3 @@ -installed: false dependencies: [go] go: &gotool !get_tool_path "{go[path]}" path: &path !join_path [!get_default "{gopath}", bin/tko-subs] diff --git a/pipeline/tools/waybackurls.yaml b/pipeline/tools/waybackurls.yaml index 1008719..a2d80c0 100644 --- a/pipeline/tools/waybackurls.yaml +++ b/pipeline/tools/waybackurls.yaml @@ -1,4 +1,3 @@ -installed: false dependencies: [go] go: &gotool !get_tool_path "{go[path]}" path: &path !join_path [!get_default "{gopath}", bin/waybackurls] diff --git a/pipeline/tools/webanalyze.yaml b/pipeline/tools/webanalyze.yaml index 4c444ac..9026a6c 100644 --- a/pipeline/tools/webanalyze.yaml +++ b/pipeline/tools/webanalyze.yaml @@ -1,4 +1,3 @@ -installed: false dependencies: [go] go: &gotool !get_tool_path "{go[path]}" path: &path !join_path [!get_default "{gopath}", bin/webanalyze] diff --git a/tests/test_recon/test_helpers.py b/tests/test_recon/test_helpers.py index d3518c8..92a7b5b 100644 --- a/tests/test_recon/test_helpers.py +++ b/tests/test_recon/test_helpers.py @@ -32,30 +32,18 @@ def test_get_scans(): @pytest.mark.parametrize( - "requirements, exception", - [ - (["amass"], True), - (["masscan"], True), - ( - [ - "amass", - "aquatone", - "masscan", - "tko-subs", - "recursive-gobuster", - "searchsploit", - "subjack", - "gobuster", - "webanalyze", - "waybackurls", - ], - False, - ), - ], + "requirements, exception, expected", + [(["amass"], True, None), (["amass"], False, False), (["aquatone"], False, True)], ) -def test_meets_requirements(requirements, exception): - with patch("pipeline.recon.helpers.get_tool_state"): - assert meets_requirements(requirements, exception) +def test_meets_requirements(requirements, exception, expected): + mdict = {"amass": {"installed": False}, "aquatone": {"installed": True}} + with patch("pipeline.recon.helpers.tools", autospec=dict) as mtools: + mtools.get.return_value = mdict.get(requirements[0]) + if exception: + with pytest.raises(RuntimeError): + meets_requirements(requirements, exception) + else: + assert meets_requirements(requirements, exception) is expected @pytest.mark.parametrize( diff --git a/tests/test_shell/test_recon_pipeline_shell.py b/tests/test_shell/test_recon_pipeline_shell.py index f86e51e..4c7aa5b 100644 --- a/tests/test_shell/test_recon_pipeline_shell.py +++ b/tests/test_shell/test_recon_pipeline_shell.py @@ -481,13 +481,15 @@ def test_remove_old_recon_tools(self, test_input, tmp_path): subfile.touch() assert subfile.exists() + old_loop = recon_shell.ReconShell.cmdloop - with patch("cmd2.Cmd.cmdloop"), patch("sys.exit"), patch("cmd2.Cmd.select") as mocked_select: - mocked_select.return_value = test_input + recon_shell.ReconShell.cmdloop = MagicMock() + recon_shell.cmd2.Cmd.select = MagicMock(return_value=test_input) + with patch("sys.exit"): recon_shell.main( name="__main__", old_tools_dir=tooldir, old_tools_dict=tooldict, old_searchsploit_rc=searchsploit_rc ) - + recon_shell.ReconShell.cmdloop = old_loop for file in [subfile, tooldir, tooldict, searchsploit_rc]: if test_input == "Yes": assert not file.exists() @@ -503,16 +505,16 @@ def test_check_scan_directory(self, test_input, tmp_path): new_tmp = tmp_path / f"check_scan_directory_test-{user_input}-{answer}" new_tmp.mkdir() - with patch("cmd2.Cmd.select") as mocked_select: - mocked_select.return_value = answer - - self.shell.check_scan_directory(str(new_tmp)) + recon_shell.cmd2.Cmd.select = MagicMock(return_value=answer) - assert new_tmp.exists() == exists - assert len(list(tmp_path.iterdir())) == numdirs + print(list(tmp_path.iterdir()), new_tmp) + self.shell.check_scan_directory(str(new_tmp)) - if answer == "Save": - assert ( - re.search(r"check_scan_directory_test-3-Save-[0-9]{6,8}-[0-9]+", str(list(tmp_path.iterdir())[0])) - is not None - ) + assert new_tmp.exists() == exists + print(list(tmp_path.iterdir()), new_tmp) + assert len(list(tmp_path.iterdir())) == numdirs + if answer == "Save": + assert ( + re.search(r"check_scan_directory_test-3-Save-[0-9]{6,8}-[0-9]+", str(list(tmp_path.iterdir())[0])) + is not None + ) diff --git a/tests/test_tools_install/test_tools_install_command.py b/tests/test_tools_install/test_tools_install_command.py index 357191b..e629452 100644 --- a/tests/test_tools_install/test_tools_install_command.py +++ b/tests/test_tools_install/test_tools_install_command.py @@ -55,7 +55,7 @@ def setup_go_test(self, tool_name, tool_dict): tool_dict.get(dependency)["path"] = dependency_path tool_dict.get(dependency).get("install_commands")[ 0 - ] = f"wget -q https://dl.google.com/go/go1.14.6.linux-amd64.tar.gz -O {tmp_path}/go.tar.gz" + ] = f"wget -q https://dl.google.com/go/go1.14.7.linux-amd64.tar.gz -O {tmp_path}/go.tar.gz" tool_dict.get(dependency).get("install_commands")[ 1 ] = f"tar -C {self.shell.tools_dir} -xvf {tmp_path}/go.tar.gz" diff --git a/tests/test_web/test_gobuster.py b/tests/test_web/test_gobuster.py index 8071b4b..fc5b3a7 100644 --- a/tests/test_web/test_gobuster.py +++ b/tests/test_web/test_gobuster.py @@ -1,3 +1,4 @@ +import os import shutil import tempfile from pathlib import Path @@ -41,7 +42,8 @@ def test_scan_run(self): assert mocked_run.called assert self.scan.parse_results.called - def test_scan_recursive_run(self): + def test_scan_recursive_run(self, tmp_path): + os.chdir(tmp_path) with patch("concurrent.futures.ThreadPoolExecutor.map") as mocked_run: self.scan.parse_results = MagicMock() self.scan.db_mgr.get_all_web_targets = MagicMock()