added tests for new ci stage; restructured go tool installs; closes #63…

…; closes #46 (#64)

.github/workflows/pythonapp.yml

@@ -107,3 +107,23 @@ jobs:
         run: |
           pipenv install pytest cmd2 luigi sqlalchemy python-libnmap
           pipenv run python -m pytest tests/test_models
+
+  test-unmocked-tool-installs:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Set up pipenv
+      run: |
+        python -m pip install --upgrade pip
+        pip install pipenv
+        pipenv install -d
+    - name: Test with pytest
+      run: |
+        pipenv install pytest cmd2 luigi sqlalchemy python-libnmap
+        pipenv run python -m pytest tests/test_tools_install

.pre-commit-config.yaml

@@ -3,8 +3,7 @@ repos:
     rev: stable
     hooks:
     - id: black
-      language_version: python3
-      args: ['pipeline', 'tests/test_web', 'tests/test_recon', 'tests/test_shell', 'tests/test_models']
+      args: ['pipeline', 'tests/test_web', 'tests/test_recon', 'tests/test_shell', 'tests/test_models', 'tests/test_tools_install']
 -   repo: https://gitlab.com/pycqa/flake8
     rev: 3.7.9
     hooks:

Pipfile

@@ -22,4 +22,4 @@ python-libnmap = "==0.7.0"
 pyyaml = "==5.3.1"
 
 [requires]
-python_version = "3.7"
+python_version = "3"

pipeline/recon-pipeline.py

@@ -124,6 +124,8 @@ def __init__(self, *args, **kwargs):
 
         self.tools_dir.mkdir(parents=True, exist_ok=True)
         Path(defaults.get("database-dir")).mkdir(parents=True, exist_ok=True)
+        Path(defaults.get("gopath")).mkdir(parents=True, exist_ok=True)
+        Path(defaults.get("goroot")).mkdir(parents=True, exist_ok=True)
 
         # register hooks to handle selector loop start and cleanup
         self.register_preloop_hook(self._preloop_hook)

pipeline/recon/config.py

@@ -13,10 +13,13 @@
     "home": Path.home(),
 }
 
+defaults["project-dir"] = str(Path(__file__).parents[2])
 defaults["tools-dir"] = f"{defaults.get('home')}/.local/recon-pipeline/tools"
 defaults["database-dir"] = f"{defaults.get('home')}/.local/recon-pipeline/databases"
+
+defaults["goroot"] = f"{defaults.get('tools-dir')}/pipeline-go"
+defaults["gopath"] = f"{defaults.get('tools-dir')}/pipeline-go-workspace"
 defaults["gobuster-wordlist"] = f"{defaults.get('tools-dir')}/seclists/Discovery/Web-Content/common.txt"
-defaults["project-dir"] = str(Path(__file__).parents[2])
 
 web_ports = {
     "80",

pipeline/tools/amass.yaml

@@ -1,12 +1,10 @@
 installed: false
 dependencies: [go]
-tools: &tools !get_default "{tools-dir}"
 go: &gotool !get_tool_path "{go[path]}"
-path: &amass !join_path [*tools, "amass"]
+path: !join_path [!get_default "{gopath}", bin/amass]
+environ: {"GO111MODULE": "on", "GOPATH": !get_default "{gopath}"}
 
 commands:
 - !join [*gotool, get github.com/OWASP/Amass/v3/...]
-- !join [cp ~/go/bin/amass, *amass]
 
-shell: true
-environ: {"GO111MODULE": "on"}
+

pipeline/tools/go.yaml

@@ -1,9 +1,8 @@
 installed: false
-home: &home !get_default "{home}"
-path: &gotool /usr/local/go/bin/go
-bashrc: &bashrc !join_path [*home, "/.bashrc;"]
+bashrc: &bashrc !join_path [!get_default "{home}", .bashrc]
+path: &gotool !join_path [!get_default "{goroot}", go/bin/go]
 
 commands:
 - wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz -O /tmp/go.tar.gz
-- sudo tar -C /usr/local -xvf /tmp/go.tar.gz
-- !join ["bash -c 'if [ ! $(echo ${PATH} | grep $(dirname", *gotool, ")) ]; then echo PATH=${PATH}:/usr/local/go/bin >>", *bashrc, "fi'"]
+- !join [tar -C, !get_default "{goroot}", -xvf /tmp/go.tar.gz]
+- !join ["bash -c 'if [ ! $(grep $(dirname", *gotool, ")", *bashrc, ") ]; then echo PATH=${PATH}:$(dirname", *gotool, ") >>", *bashrc, "; fi'"]

pipeline/tools/gobuster.yaml

@@ -1,12 +1,8 @@
 installed: false
 dependencies: [go, seclists]
-home: &home !get_default "{home}"
-path: !join_path [*home, go/bin/gobuster]
 go: &gotool !get_tool_path "{go[path]}"
-go_home: &gohome !join_path [*home, "/go/src/github.com/OJ/gobuster &&"]
+path: !join_path [!get_default "{gopath}", bin/gobuster]
+environ: {"GOPATH": !get_default "{gopath}"}
 
 commands:
 - !join [*gotool, get github.com/OJ/gobuster]
-- !join [(cd, *gohome, *gotool, build &&, *gotool, install)]
-
-shell: true

pipeline/tools/recursive-gobuster.yaml

@@ -1,10 +1,10 @@
 installed: false
-dependencies: [go]
+dependencies: [gobuster]
 tools: &tools !get_default "{tools-dir}"
 path: !join_path [*tools, recursive-gobuster/recursive-gobuster.pyz]
-recursive-parent: &recpar !join_path [*tools, recursive-gobuster]
+recursive-parent: &parent !join_path [*tools, recursive-gobuster]
 
 commands:
-- !join ["bash -c 'if [ -d", *recpar, "]; then cd", *recpar,
+- !join ["bash -c 'if [ -d", *parent, "]; then cd", *parent,
         "&& git fetch --all && git pull; else git clone https://github.com/epi052/recursive-gobuster.git",
-        *recpar, ; fi']
+        *parent, " ; fi'"]

pipeline/tools/searchsploit.yaml

@@ -10,8 +10,8 @@ sed-command: &sedcom !join_empty ["'s#/opt#", *tools, "#g'"]
 
 commands:
 - !join ["bash -c 'if [ -d /usr/share/exploitdb ]; then ln -fs /usr/share/exploitdb",
-        *exploitdb, "&& sudo ln -fs $(which searchsploit)", *searchsploit,
+        *exploitdb, "&& ln -fs $(which searchsploit)", *searchsploit,
         "; elif [ -d", *exploitdb, "]; then cd", *exploitdb,
-        "&& git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git", *exploitdb, ; fi']
-- !join ["bash -c 'if [ -f", *ss_rc, "]; then cp -n", *ss_rc, *home, ; fi']
-- !join ["bash -c 'if [ -f", *homesploit, "]; then sed -i", *sedcom, *homesploit, ; fi']
+        "&& git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git", *exploitdb, "; fi'"]
+- !join ["bash -c 'if [ -f", *ss_rc, "]; then cp -n", *ss_rc, *home, "; fi'"]
+- !join ["bash -c 'if [ -f", *homesploit, "]; then sed -i", *sedcom, *homesploit, "; fi'"]

pipeline/tools/seclists.yaml

@@ -3,6 +3,6 @@ tools: &tools !get_default "{tools-dir}"
 path: &secfile !join_path [*tools, seclists]
 
 commands:
-  - !join ["bash -c 'if [[ -d /usr/share/seclists ]]; then ln -s /usr/share/seclists",
-          *secfile, "; elif [[ -d", *secfile, "]] ; then cd", *secfile, "&& git fetch --all && git pull;",
-          else git clone https://github.com/danielmiessler/SecLists.git, *secfile, ; fi']
+- !join ["bash -c 'if [[ -d /usr/share/seclists ]]; then ln -s /usr/share/seclists",
+        *secfile, "; elif [[ -d", *secfile, "]] ; then cd", *secfile, "&& git fetch --all && git pull;",
+        "else git clone https://github.com/danielmiessler/SecLists.git", *secfile, "; fi'"]

pipeline/tools/subjack.yaml

@@ -1,13 +1,10 @@
 installed: false
 dependencies: [go]
 go: &gotool !get_tool_path "{go[path]}"
-home: &home !get_default "{home}"
-path: !join_path [*home, go/bin/subjack]
-subjack_home: &subjhome !join_path [*home, "/go/src/github.com/haccer/subjack &&"]
-fingerprints: !join_path [*home, go/src/github.com/haccer/subjack/fingerprints.json]
+path: !join_path [!get_default "{gopath}", bin/subjack]
+environ: {"GOPATH": !get_default "{gopath}"}
+
+fingerprints: !join_path [!get_default "{gopath}", src/github.com/haccer/subjack/fingerprints.json]
 
 commands:
 - !join [*gotool, get github.com/haccer/subjack]
-- !join [(cd, *subjhome, *gotool, install)]
-
-shell: true

pipeline/tools/tko-subs.yaml

@@ -1,13 +1,8 @@
 installed: false
 dependencies: [go]
 go: &gotool !get_tool_path "{go[path]}"
-home: &home !get_default "{home}"
-path: !join_path [*home, go/bin/tko-subs]
-tko_home: &tkohome !join_path [*home, "go/src/github.com/anshumanbh/tko-subs &&"]
-git_dir: !join_path [*home, go/src/github.com/anshumanbh/tko-subs]
+path: !join_path [!get_default "{gopath}", bin/tko-subs]
+environ: {"GOPATH": !get_default "{gopath}"}
 
 commands:
 - !join [*gotool, get, github.com/anshumanbh/tko-subs]
-- !join [(cd, *tkohome, *gotool, "build &&", *gotool, "install)"]
-
-shell: true

pipeline/tools/waybackurls.yaml

@@ -1,7 +1,8 @@
 installed: false
 dependencies: [go]
 go: &gotool !get_tool_path "{go[path]}"
-path: !join_path [!get_default "{home}", go, bin, waybackurls]
+path: !join_path [!get_default "{gopath}", bin/waybackurls]
+environ: {"GOPATH": !get_default "{gopath}"}
 
 commands:
 - !join [*gotool, get, github.com/tomnomnom/waybackurls]

pipeline/tools/webanalyze.yaml

@@ -1,12 +1,8 @@
 installed: false
 dependencies: [go]
-home: &home !get_default "{home}"
 go: &gotool !get_tool_path "{go[path]}"
-path: !join_path [*home, go/bin/webanalyze]
-webanalyze_home: &webhome !join_path [*home, "/go/src/github.com/rverton/webanalyze &&"]
+path: !join_path [!get_default "{gopath}", bin/webanalyze]
+environ: {"GOPATH": !get_default "{gopath}"}
 
 commands:
 - !join [*gotool, get github.com/rverton/webanalyze/...]
-- !join [(cd, *webhome, *gotool, "build &&", *gotool, install)]
-
-shell: true