Merge branch 'white/migration'

.github/issue_template.md

@@ -12,7 +12,7 @@ Paste the output of the *./faraday.py --version* command
 
 ## Component Name
 
-If you know where the problem lays indicate it: 
+If you know where the problem lays indicate it:
 WebGui/GTKGui/Plugin/Console/Continuous Scanning/Etc.
 
 ## Steps to reproduce
@@ -42,7 +42,7 @@ If you don't find anything on the logs, please provide screenshots of the error.
 
 ### Configuration files
 
-Mention any settings you have changed/added/removed. 
+Mention any settings you have changed/added/removed.
 
 ### Reports/Extra data
 
@@ -51,11 +51,10 @@ If you are having issues with plugins, please attach relevant files if possible.
 
 ### OS
 
-Provide information on your operating system. Example: 
+Provide information on your operating system. Example:
 
-$ cat /etc/lsb-release 
+$ cat /etc/lsb-release
 DISTRIB_ID=Ubuntu
 DISTRIB_RELEASE=16.10
 DISTRIB_CODENAME=yakkety
 DISTRIB_DESCRIPTION="Ubuntu 16.10"
-

.gitignore

@@ -25,6 +25,8 @@ pip-log.txt
 .coverage
 .tox
 nosetests.xml
+test_cases/.cache/
+test_cases/htmlcov/
 
 # Translations
 *.mo
@@ -59,4 +61,8 @@ jsconfig.json
 .idea
 
 # vFeed DB
-data/vfeed.db
+data/vfeed.db
+
+# Images based on DB Schema
+entity_dbschema.png
+uml_schema.png

AUTHORS

@@ -1,13 +1,13 @@
 The PRIMARY AUTHORS are:
 
-    * Daniel Foguelman 
-    * Esteban Guillardoy 
+    * Daniel Foguelman
+    * Esteban Guillardoy
     * Ezequiel Tavella
-    * Facundo de Guzmán 
-    * Federico Kirschbaum 
+    * Facundo de Guzmán
+    * Federico Kirschbaum
     * Francisco Amato
     * Franco Linares
-    * German Riera 
+    * German Riera
     * Joaquín López Pereyra
     * Leonardo Lazzaro
     * Martín Rocha
@@ -40,3 +40,4 @@ Project contributors
     * Buanzo
     * Necrose99
     * csk
+    * Mike Zhong (go bears)

Jenkinsfile

@@ -0,0 +1,133 @@
+#!groovy
+node (label: "master"){
+    def ENV_PATH = "$HOME/venv/faraday"
+    echo "${ENV_PATH}"
+
+    stage("Clean virtualenv") {
+        sh "rm -rf ${ENV_PATH}"
+    }
+
+    stage("Install Python Virtual Enviroment") {
+        sh "/usr/local/bin/virtualenv --no-site-packages ${ENV_PATH}"
+    }
+
+    // Get the latest version of our application code.
+    stage ("Pull Code from SCM") {
+        checkout scm
+    }
+
+    stage ("Install Application Dependencies") {
+        sh """
+            source ${ENV_PATH}/bin/activate
+            pip install virtualenv responses
+            pip install -U -r $WORKSPACE/requirements.txt
+            pip install -U -r $WORKSPACE/requirements_server.txt
+            pip install -U -r $WORKSPACE/requirements_extras.txt
+            pip install -U -r $WORKSPACE/requirements_dev.txt
+            deactivate
+           """
+    }
+
+    stage ("Check code style") {
+        sh """
+            sloccount --duplicates --wide --details $WORKSPACE   | fgrep -v .git > $WORKSPACE/sloccount.sc || :
+            find $WORKSPACE -name \\"*.py\\" | egrep -v '^./tests/'  | xargs pyflakes  > $WORKSPACE/pyflakes.log || :
+            find $WORKSPACE -name \\"*.py\\" | egrep -v '^./tests/'  | xargs pylint --output-format=parseable --reports=y > $WORKSPACE/pylint.log || :
+            eslint -c /home/faraday/.eslintrc.js -f checkstyle $WORKSPACE/server/www/scripts/**/* > eslint.xml || true
+
+           """
+           warnings canComputeNew: false, canResolveRelativePaths: false, consoleParsers: [[parserName: 'PyFlakes']], defaultEncoding: '', excludePattern: '', healthy: '', includePattern: '', messagesPattern: '', parserConfigurations: [[parserName: 'AcuCobol Compiler', pattern: 'pyflakes.log']], unHealthy: ''
+
+    }
+
+    stage ("Run Unit/Integration Tests") {
+        def testsError = null
+        try {
+            sh """
+                source ${ENV_PATH}/bin/activate
+                cd $WORKSPACE && pytest -v --junitxml=$WORKSPACE/xunit.xml || :
+                deactivate
+               """
+               step([$class: 'CoberturaPublisher', autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: '**/coverage.xml', failNoReports: false, failUnhealthy: false, failUnstable: false, maxNumberOfBuilds: 0, onlyStable: false, sourceEncoding: 'ASCII', zoomCoverageChart: false])
+        }
+        catch(err) {
+            testsError = err
+            currentBuild.result = 'FAILURE'
+        }
+        finally {
+            junit "**/xunit.xml"
+            notifyBuild(currentBuild.result, "SQLite Build")
+            if (testsError) {
+                throw testsError
+            }
+
+        }
+    }
+
+    stage ("Run Unit/Integration Tests (with PostgreSQL)") {
+        def testsError = null
+        try {
+            withCredentials([string(credentialsId: 'postgresql_connection_string', variable: 'CONN_STRING')]) {
+                sh """
+                    source ${ENV_PATH}/bin/activate
+                    cd $WORKSPACE && pytest -v --junitxml=$WORKSPACE/xunit-postgres.xml --connection-string "$CONN_STRING" || :
+                    deactivate
+                """
+                step([$class: 'CoberturaPublisher', autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: '**/coverage.xml', failNoReports: false, failUnhealthy: false, failUnstable: false, maxNumberOfBuilds: 0, onlyStable: false, sourceEncoding: 'ASCII', zoomCoverageChart: false])
+            }
+        }
+        catch(err) {
+            testsError = err
+            currentBuild.result = 'FAILURE'
+        }
+        finally {
+            junit "**/xunit-postgres.xml"
+            notifyBuild(currentBuild.result, "PostgreSQL Build")
+            if (testsError) {
+                throw testsError
+            }
+
+        }
+    }
+
+    stage ("Run Closure Compiler") {
+        try {
+            sh """
+                java -jar /home/faraday/closure-compiler-v20180610.jar $WORKSPACE/server/www/scripts
+            """
+        }
+        catch (err) {
+            currentBuild.result = 'FAILURE'
+        }
+        finally {
+            notifyBuild(currentBuild.result, "Closure compiler")
+        }
+    }
+}
+
+def notifyBuild(String buildStatus = 'STARTED', String extraMessage = '') {
+  // build status of null means successful
+  buildStatus =  buildStatus ?: 'SUCCESSFUL'
+
+  // Default values
+  def colorName = 'RED'
+  def colorCode = '#FF0000'
+  def subject = "${buildStatus}: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'"
+  def summary = "${subject} (${env.BUILD_URL}) " + extraMessage
+
+  // Override default values based on build status
+  if (buildStatus == 'STARTED') {
+    color = 'YELLOW'
+    colorCode = '#FFFF00'
+  } else if (buildStatus == 'SUCCESSFUL') {
+    color = 'GREEN'
+    colorCode = '#00FF00'
+  } else {
+    color = 'RED'
+    colorCode = '#FF0000'
+    summary = summary + ' @channel'
+  }
+
+  // Send notifications
+  slackSend (color: colorCode, message: summary)
+}

README.md

@@ -1,55 +1,147 @@
-![Faraday Logo](https://raw.github.com/wiki/infobyte/faraday/images/Faraday-Logo.png)
+## About
 
-Faraday introduces a new concept - IPE (Integrated Penetration-Test Environment) a multiuser Penetration test IDE. Designed for distribution, indexation and analysis of the data generated during a security audit.
+Faraday introduces a new concept - IPE (Integrated Penetration-Test Environment) a multiuser Penetration test IDE. Designed for distributing, indexing, and analyzing the data generated during a security audit.
 
-The main purpose of Faraday is to re-use the available tools in the community to take advantage of them in a multiuser way.
+> Made for true pentesters!
 
-Designed for simplicity, users should notice no difference between their own terminal application and the one included in Faraday. Developed with a specialized set of functionalities that help users improve their own work. Do you remember yourself programming without an IDE? Well, Faraday does the same as an IDE does for you when programming, but from the perspective of a penetration test.
+Faraday was made to let you take advantage of the available tools in the community in a truly multiuser way.
 
-Please read the [RELEASE notes](https://github.com/infobyte/faraday/blob/master/RELEASE.md)!
+Designed for simplicity, users should notice no difference between their own terminal application and the one included in Faraday. Developed with a specialized set of functionalities, users improve their own work. Do you remember the last time you programmed without an IDE? What IDEs are to programming, Faraday is to pentesting.
 
-![GUI - Web](https://raw.github.com/wiki/infobyte/faraday/images/GUI_Dashboard_new.png)
+![GUI - GTK](https://raw.github.com/wiki/infobyte/faraday/images/client/gtk_main_window.png)
 
-Plugins
----
-Don't change the way you work today! Faraday plays well with others, right now it has more than [50 supported tools](https://github.com/infobyte/faraday/wiki/Plugin-List), among them you will find: 
+Faraday crunches the data you load into different visualizations that are useful to managers and pentesters alike.
+
+![GUI - Web](https://raw.github.com/wiki/infobyte/faraday/images/dashboard/dashboard.png)
+
+To read about the latest features check out the [release notes](https://github.com/infobyte/faraday/blob/master/RELEASE.md)!
+
+## Getting Started!
+
+Check out our documentacion for datailed information on how to install Faraday in all of our supported platforms:
+
+![Supported Os](https://raw.github.com/wiki/infobyte/faraday/images/platform/supported.png)
+
+To begin the instalation process check our out [First Step](https://raw.github.com/wiki/infobyte/faraday/First-steps) Wiki.
+
+## New Features!
+All of Faraday's latest features and updates are always available on our [blog](http://blog.infobytesec.com/search/label/english).
+There are new entries every few weeks, don't forget to check out our amaizing new improvements on it's last entry!
+
+
+## Plugins list
+
+You feed data to Faraday from your favorite tools through Plugins. Right now there are more than [60+ supported tools](https://github.com/infobyte/faraday/wiki/Plugin-List), among which you will find:
 
 ![](https://raw.github.com/wiki/infobyte/faraday/images/plugins/Plugins.png)
 
-There are 3 kind of plugins:
- * Plugins that intercept commands, fired directly when a command is detected in the console. These are transparent to you and no additional action on your part is needed.
- * Plugins that import file reports. You have to copy the report to **$HOME/.faraday/report/[workspacename]** (replacing **[workspacename]** with the actual name of your Workspace) and Faraday will automatically detect, process and add it to the HostTree.
- * Plugin connectors or online (BeEF, Metasploit, Burp), these connect to external APIs or databases, or talk directly to Faraday's RPC API.
+There are three Plugin types: **console** plugins which intercept and interpret the output of the tools you execute, **report** plugins which allows you to import previously generated XMLs, and **online** plugins which access Faraday's API or allow Faraday to connect to external APIs and databases.
+
+[Read more about Plugins](http://github.com/infobyte/faraday/wiki/Plugin-List).
+
+## Features
+
+### Workspaces
+Information is organized into various **Workspaces**. Each Workspace contains a pentest team's assignments and all the intel that is discovered.
+
+### Conflicts
+If two plugins produce clashing information for an individual element, a conflict that the user will have to resolve is generated.  An example is if **user1** incorporates host *127.0.0.1 OS:Linux* and **user2** incorporates *127.0.0.1 OS: Linux Ubuntu 13.10*.
+
+On our [GTK interface](https://github.com/infobyte/faraday/wiki/Usage#gtk-gui) there's a button on the bottom right corner of the main window displaying the number of conflicts in the current workspace. To resolve them, just click on the button and a window will open where you can edit the conflicting objects and select which one to keep.
+
+### Faraday plugin
+
+Using our plugin you can perform various actions using the command line, for example:
+
+    $ cd faraday-dev/bin/
+    $ ./fplugin create_host 192.154.33.222 Android
+    1a7b2981c7becbcb3d5318056eb29a58817f5e67
+    $ ./fplugin filter_services http ssh -p 21 -a
+    Filtering services for ports: 21, 22, 80, 443, 8080, 8443
+
+    192.168.20.1    ssh     [22]    tcp open    None
+    192.168.20.1    http    [443]   tcp open    None
+    192.168.20.7    ssh     [22]    tcp open    Linux
+    192.168.20.7    http    [443]   tcp open    Linux
+    192.168.20.11   ssh     [22]    tcp open    Linux
+
+
+Read more about the [Faraday Plugin](https://github.com/infobyte/faraday/wiki/faraday-plugin).
+
+### Notifications
+Updating objects on other Faraday instances result in notifications on your
+Faraday GTK Client.
+
+![](https://raw.github.com/wiki/infobyte/faraday/images/client/gtk_notifications_dialog.png)
+
+
+### CSV Exporting
+Faraday supports CSV Exporting from its WEB UI.
+[More information](Exporting-the-information)
+
+## Links
+
+* Homepage: https://www.faradaysec.com
+* User forum: https://forum.faradaysec.com
+* User's manual: https://github.com/infobyte/faraday/wiki
+* Download: [.tar.gz](https://github.com/infobyte/faraday/tarball/master)
+* Commits RSS feed: https://github.com/infobyte/faraday/commits/master.atom
+* Issue tracker: https://github.com/infobyte/faraday/issues
+* Frequently Asked Questions (FAQ): https://github.com/infobyte/faraday/wiki/FAQ
+* Mailing list subscription: https://groups.google.com/forum/#!forum/faradaysec
+* Twitter: [@faradaysec](https://twitter.com/faradaysec)
+* [Demos](https://github.com/infobyte/faraday/wiki/Demos)
+* IRC: [ircs://irc.freenode.net/faraday-dev](ircs://irc.freenode.net/faraday-dev) [WebClient](https://webchat.freenode.net/?nick=wikiuser&channels=faraday-dev&prompt=1&uio=d4)
+* Screenshots: https://github.com/infobyte/faraday/wiki/Screenshots
+* Send your ideas and suggestions here: [https://www.faradaysec.com/ideas](https://www.faradaysec.com/ideas)
+
+## Presentations
+
+* Ekoparty Security Conference - 2017:
+   * http://blog.infobytesec.com/2017/10/ekoparty-2017-review_23.html
+
+* Black Hat Arsenal Asia - 2017:
+   * https://www.blackhat.com/asia-17/arsenal.html#faraday
+
+* Zero Nights - 2016
+   * https://www.slideshare.net/AlexanderLeonov2/enterprise-vulnerability-management-zeronights16
+
+* AV Tokio - 2016:
+   * http://en.avtokyo.org/avtokyo2016/event
 
-Getting started
----
-The following platforms are supported:
+* Black Hat Arsenal USA - 2016:
 
-![platform](https://raw.github.com/wiki/infobyte/faraday/images/platform/supported.png) 
+   * https://www.blackhat.com/us-16/arsenal.html#faraday
 
-Read more about [supported platforms and installation specifics](https://github.com/infobyte/faraday/wiki/First-steps).
+* Black Hat Arsenal Europe - 2016
+   * https://www.blackhat.com/eu-16/arsenal.html#faraday
 
-#### Quick install
+* SecurityWeekly - 2016:
+   * http://securityweekly.com/2016/08/02/security-weekly-475-federico-kirschbaum/
 
-This applies only to Debian, Ubuntu, Kali and Backtrack. For the full installation guide [visit our wiki](https://github.com/infobyte/faraday/wiki/First-steps).
+* Bsides Latam - 2016:
+   * http://www.infobytesec.com/down/Faraday_BsideLatam_2016.pdf
 
-Download the [latest tarball](https://github.com/infobyte/faraday/tarball/master) or clone our repo:
+* Black Hat Arsenal Asia - 2016:
+   * https://www.blackhat.com/asia-16/arsenal.html#faraday
 
-```
-$ git clone https://github.com/infobyte/faraday.git faraday-dev
-$ cd faraday-dev
-$ ./install.sh
-$ ./faraday-server.py
-$ ./faraday.py
-```
+* Black Hat Arsenal Europe - 2015:
+   * https://www.blackhat.com/eu-15/arsenal.html#faraday
 
-More about Faraday
----
-Want to read more about the project? Try our [wiki](https://github.com/infobyte/faraday/wiki).
+* Black Hat Arsenal USA - 2015:
+   * https://www.blackhat.com/us-15/arsenal.html#faraday
+   * http://blog.infobytesec.com/2015/08/blackhat-2015_24.html
 
-Already a user and have a question or bug report? Check out our [FAQ](https://github.com/infobyte/faraday/wiki/FAQ) and [troubleshooting](https://github.com/infobyte/faraday/wiki/troubleshooting) pages. If you're still having troubles you can [open a ticket](https://github.com/infobyte/faraday/issues/new).
+* RSA - 2015:
+   * http://www.rsaconference.com/events/us15/expo-sponsors/exhibitor-list/1782/infobyte-llc
+   * http://blog.infobytesec.com/2015/05/infobyte-en-la-rsa-2015.html
 
-Join our community! Subscribe to our [mailing list](https://groups.google.com/forum/#!forum/faradaysec) or find us on Twitter [@faradaysec](https://twitter.com/faradaysec) 
+* Ekoparty Security Conference - 2014:
+   * https://www.youtube.com/watch?v=_j0T2S6Ppfo
 
-Do you have a question? Troubleshooting? Joing our IRC channel #faraday-dev in [freenode](ircs://irc.freenode.net/faraday-dev) or access directly from this link: [![Visit our IRC channel](https://kiwiirc.com/buttons/irc.freenode.org/faraday-dev.png)](https://kiwiirc.com/client/irc.freenode.org/?nick=faraday_gi|?#faraday-dev)
+* Black Hat Arsenal - 2011
+   * http://www.infobytesec.com/down/Faraday_BH2011_Arsenal.pdf
 
+* Ekoparty Security Conference - 2010:
+   * http://prezi.com/fw46zt6_zgi8/faraday/
+   * http://vimeo.com/16516987