From c07178d3077c9e74ae7468bf45026af9e600c1a0 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 2 Jun 2017 15:40:20 -0300 Subject: [PATCH 0001/1506] ADD initial websockets support --- gui/gtk/server.py | 39 ++++---- persistence/server/changes_stream.py | 141 ++++++++++++++++++++------- persistence/server/server.py | 17 +++- requirements.txt | 2 + server/app.py | 6 +- server/models.py | 17 +++- server/web.py | 26 +++-- 7 files changed, 184 insertions(+), 64 deletions(-) diff --git a/gui/gtk/server.py b/gui/gtk/server.py index a9a9314e6fc..8c6ed1e055e 100644 --- a/gui/gtk/server.py +++ b/gui/gtk/server.py @@ -7,11 +7,14 @@ ''' -import threading, time, requests +import time +import threading + from model.guiapi import notification_center from decorators import safe_io_with_server from persistence.server import models, server_io_exceptions + class ServerIO(object): def __init__(self, active_workspace): self.__active_workspace = active_workspace @@ -99,7 +102,6 @@ def continously_get_changes(self): coming from other instances of Faraday. Return the thread on any exception, of if self.stream is None. """ - # There is very arcane, dark magic involved in this method. # What you need to know: do not touch it. # If you touch it, do check out persitence/server/changes_stream.py @@ -160,22 +162,25 @@ def notification_dispatcher(obj_id, obj_type, obj_name, deleted, revision): def get_changes(): # dark maaaaaagic *sing with me!* dark maaaaaagic - if self.stream: - try: - for change, obj_type, obj_name in self.stream: - with self.changes_lock: - change = filter_changes(change, obj_type) - if change: - deleted = bool(change.get('deleted')) - obj_id = change.get('id') - revision = change.get("changes")[-1].get('rev') - notification_dispatcher(obj_id, obj_type, obj_name, - deleted, revision) - except server_io_exceptions.ChangesStreamStoppedAbruptly: - notification_center.WorkspaceProblem() + while True: + if self.stream: + try: + for change, obj_type, obj_name in self.stream: + print(change, obj_type, obj_name) + with self.changes_lock: + change = filter_changes(change, obj_type) + if change: + deleted = bool(change.get('deleted')) + obj_id = change.get('id') + revision = change.get("changes")[-1].get('rev') + notification_dispatcher(obj_id, obj_type, obj_name, + deleted, revision) + except server_io_exceptions.ChangesStreamStoppedAbruptly: + notification_center.WorkspaceProblem() + return False + else: return False - else: - return False + time.sleep(0.5) get_changes_thread = threading.Thread(target=get_changes) get_changes_thread.daemon = True diff --git a/persistence/server/changes_stream.py b/persistence/server/changes_stream.py index 62279bba73b..54a22aa55a4 100644 --- a/persistence/server/changes_stream.py +++ b/persistence/server/changes_stream.py @@ -6,19 +6,20 @@ See the file 'doc/LICENSE' for the license information ''' -import requests, json -from persistence.server.server_io_exceptions import ChangesStreamStoppedAbruptly +import time +import json +import threading +from Queue import Queue, Empty -class CouchChangesStream(object): - def __init__(self, workspace_name, server_url, since=0, heartbeat='1000', feed='continuous', **params): - self._base_url = server_url - self._change_url = "{0}/_changes".format(server_url) - self.since = since - self.heartbeat = heartbeat - self.feed = feed - self._params = params - self._response = None - self._stop = False +import requests +import websocket + +from persistence.server.server_io_exceptions import ( + ChangesStreamStoppedAbruptly +) + + +class ChangesStream(object): def __enter__(self): return self @@ -30,28 +31,7 @@ def __next__(self): return self def __iter__(self): - try: - params = {'since' : self.since, 'heartbeat': self.heartbeat, 'feed': self.feed} - self._response = requests.get(self._change_url, stream=True, params=params, **self._params) - if self._response: - for raw_line in self._response.iter_lines(): - line = self._sanitize(raw_line) - if not line: - if self._stop: break - else: continue - change = self._parse_change(line) - if not change: - continue - object_type, object_name = self._get_object_type_and_name_from_change(change) - yield change, object_type, object_name - if not self._stop: # why did we stop if no one asked me to stop? - raise ChangesStreamStoppedAbruptly - - except (requests.exceptions.RequestException, ChangesStreamStoppedAbruptly): - self.stop() - raise ChangesStreamStoppedAbruptly - except Exception as e: - self.stop() + raise NotImplementedError('Abstract class') def _get_object_type_and_name_from_change(self, change): try: @@ -86,3 +66,96 @@ def stop(self): self._response.close() self._response = None self._stop = True + + +class WebsocketsChangesStream(ChangesStream): + def __init__(self, workspace_name, server_url, **params): + self._base_url = server_url + self.changes_queue = Queue() + self.workspace_name = workspace_name + self._response = None + websocket.enableTrace(True) + self.ws = websocket.WebSocketApp( + "ws://{0}:9000".format(self._base_url), + on_message=self.on_message, + on_error=self.on_error, + on_close=self.on_close) + + self.ws.on_open = self.on_open + thread = threading.Thread(target=self.ws.run_forever, args=()) + thread.daemon = True + thread.start() + + def on_message(self, ws, message): + self.changes_queue.put(message) + + def on_error(self, ws, error): + print error + + def on_close(selg, ws): + print "### closed ###" + + def on_open(self, ws): + print 'open' + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + return False + + def __next__(self): + return self + + def __iter__(self): + try: + data = json.loads(self.changes_queue.get_nowait()) + except Empty: + raise StopIteration + yield data[0], data[1], data[2] + + def _get_object_type_and_name_from_change(self, change): + try: + id = change['id'] + response = requests.get("{0}/{1}".format(self._base_url, id), **self._params) + object_json = response.json() + except Exception: + return None, None + return object_json.get('type'), object_json.get('name') + + +class CouchChangesStream(ChangesStream): + + def __init__(self, workspace_name, server_url, since=0, heartbeat='1000', feed='continuous', **params): + self._base_url = server_url + self._change_url = "{0}/_changes".format(server_url) + self.since = since + self.heartbeat = heartbeat + self.feed = feed + self._params = params + self._response = None + self._stop = False + + def __iter__(self): + try: + params = {'since' : self.since, 'heartbeat': self.heartbeat, 'feed': self.feed} + self._response = requests.get(self._change_url, stream=True, params=params, **self._params) + if self._response: + for raw_line in self._response.iter_lines(): + line = self._sanitize(raw_line) + if not line: + if self._stop: break + else: continue + change = self._parse_change(line) + if not change: + continue + object_type, object_name = self._get_object_type_and_name_from_change(change) + yield change, object_type, object_name + if not self._stop: # why did we stop if no one asked me to stop? + raise ChangesStreamStoppedAbruptly + + except (requests.exceptions.RequestException, ChangesStreamStoppedAbruptly): + self.stop() + raise ChangesStreamStoppedAbruptly + except Exception as e: + self.stop() diff --git a/persistence/server/server.py b/persistence/server/server.py index c9f7e9b95b6..98783a2ff74 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -34,7 +34,10 @@ Unauthorized, MoreThanOneObjectFoundByID) -from persistence.server.changes_stream import CouchChangesStream +from persistence.server.changes_stream import ( + CouchChangesStream, + WebsocketsChangesStream +) # NOTE: Change is you want to use this module by itself. # If FARADAY_UP is False, SERVER_URL must be a valid faraday server url @@ -459,9 +462,17 @@ def get_objects(workspace_name, object_signature, **params): return appropiate_function(workspace_name, **params) + +def _websockets_changes(workspace_name, **extra_params): + return WebsocketsChangesStream(workspace_name, 'localhost', **extra_params) + + # cha cha cha chaaaanges! -def get_changes_stream(workspace_name, since=0, heartbeat='1000', **extra_params): - return _couch_changes(workspace_name, since=since, feed='continuous', +def get_changes_stream(workspace_name, since=0, heartbeat='1000', stream_provider=_websockets_changes, **extra_params): + """ + stream_provider: A function that returns an instance of a Stream provider + """ + return stream_provider(workspace_name, since=since, feed='continuous', heartbeat=heartbeat, **extra_params) def get_workspaces_names(): diff --git a/requirements.txt b/requirements.txt index b8d549c3033..8f2e66b1eaa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,5 @@ requests tornado flask colorama +websocket-client +autobahn diff --git a/server/app.py b/server/app.py index 9d5b7afed4f..7cb5d3bebdf 100644 --- a/server/app.py +++ b/server/app.py @@ -3,14 +3,16 @@ # See the file 'doc/LICENSE' for the license information import flask +from flask_socketio import SocketIO + import server.config import server.database from server.utils.logger import LOGGING_HANDLERS - app = flask.Flask(__name__) + def setup(): app.debug = server.config.is_debug_mode() minify_json_output(app) @@ -23,6 +25,7 @@ def remove_session_context(exception=None): for handler in LOGGING_HANDLERS: app.logger.addHandler(handler) + def minify_json_output(app): class MiniJSONEncoder(flask.json.JSONEncoder): item_separator = ',' @@ -34,4 +37,3 @@ class MiniJSONEncoder(flask.json.JSONEncoder): # Load APIs import server.api import server.modules.info - diff --git a/server/models.py b/server/models.py index 778d35dcd1c..1fe798996e5 100644 --- a/server/models.py +++ b/server/models.py @@ -4,7 +4,15 @@ import json -from sqlalchemy import Column, Integer, String, Boolean, ForeignKey, Float, Text, UniqueConstraint +from sqlalchemy import ( + Column, + Integer, + String, + Boolean, + ForeignKey, + Float, + Text, + UniqueConstraint) from sqlalchemy.orm import relationship from sqlalchemy.ext.declarative import declarative_base @@ -13,10 +21,12 @@ Base = declarative_base() + class EntityNotFound(Exception): def __init__(self, entity_id): super(EntityNotFound, self).__init__("Entity (%s) wasn't found" % entity_id) + class FaradayEntity(object): # Document Types: [u'Service', u'Communication', u'Vulnerability', u'CommandRunInformation', u'Reports', u'Host', u'Workspace', u'Interface'] @classmethod @@ -226,6 +236,7 @@ def add_relationships_from_db(self, session): query = session.query(Host).join(EntityMetadata).filter(EntityMetadata.couchdb_id == host_id) self.host = query.one() + class Service(FaradayEntity, Base): DOC_TYPE = 'Service' @@ -390,6 +401,7 @@ def add_relationships_from_db(self, session): query = session.query(Service).join(EntityMetadata).filter(EntityMetadata.couchdb_id == parent_id) self.service = query.one() + class Note(FaradayEntity, Base): DOC_TYPE = 'Note' @@ -410,6 +422,7 @@ def update_from_document(self, document): self.description=document.get('description', None) self.owned=document.get('owned', False) + class Credential(FaradayEntity, Base): DOC_TYPE = 'Cred' @@ -462,6 +475,7 @@ def add_relationships_from_db(self, session): query = session.query(Service).join(EntityMetadata).filter(EntityMetadata.couchdb_id == parent_id) self.service = query.one() + class Command(FaradayEntity, Base): DOC_TYPE = 'CommandRunInformation' @@ -490,4 +504,3 @@ def update_from_document(self, document): self.params = document.get('params', None) self.user = document.get('user', None) self.workspace = document.get('workspace', None) - diff --git a/server/web.py b/server/web.py index ca0e8c6f7c5..127d840cdd6 100644 --- a/server/web.py +++ b/server/web.py @@ -4,16 +4,24 @@ import os import functools -import twisted.web -import server.config +import twisted.web from twisted.web import proxy from twisted.internet import ssl, reactor, error from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.web.static import File from twisted.web.wsgi import WSGIResource +from autobahn.twisted.websocket import ( + WebSocketServerFactory, + listenWS +) +import server.config from server.utils import logger from server.app import app +from server.websocket_factories import ( + WorkspaceServerFactory, + BroadcastServerProtocol +) class HTTPProxyClient(proxy.ProxyClient): @@ -53,7 +61,7 @@ def proxyClientFactoryClass(self, *args, **kwargs): if enabled. """ client_factory = HTTPProxyClientFactory(*args, **kwargs) - + if self.__ssl_enabled: with open(server.config.ssl.certificate) as cert_file: cert = ssl.Certificate.loadPEM(cert_file.read()) @@ -65,7 +73,7 @@ def proxyClientFactoryClass(self, *args, **kwargs): isClient=True, wrappedFactory=client_factory) else: return client_factory - + def getChild(self, path, request): """ Keeps the implementation of this class throughout the path @@ -87,7 +95,7 @@ def __init__(self, enable_ssl=False): self.__config_server() self.__config_couchdb_conn() self.__build_server_tree() - + def __config_server(self): self.__bind_address = server.config.faraday_server.bind_address self.__listen_port = int(server.config.faraday_server.port) @@ -135,10 +143,16 @@ def __build_web_resource(self): def __build_api_resource(self): return WSGIResource(reactor, reactor.getThreadPool(), app) + def __build_websockets_resource(self): + print(u"wss://{0}:9000".format(self.__bind_address)) + factory = WorkspaceServerFactory(u"ws://{0}:9000".format(self.__bind_address)) + factory.protocol = BroadcastServerProtocol + return factory + def run(self): site = twisted.web.server.Site(self.__root_resource) self.__listen_func( self.__listen_port, site, interface=self.__bind_address) + listenWS(self.__build_websockets_resource()) reactor.run() - From b430377dca11d3f3b92b60aad8cf740950acf0e1 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 5 Jun 2017 11:35:42 -0300 Subject: [PATCH 0002/1506] REFACTOR remove arcane code for couchdb changes When we used the changes api of couchdb we needed to filter and check the data sent by the changes API. Now we have full control of our websockets msgs format which allow us to simplify the code. --- gui/gtk/server.py | 99 +++++++--------------------- persistence/server/changes_stream.py | 2 +- server/app.py | 3 + 3 files changed, 29 insertions(+), 75 deletions(-) diff --git a/gui/gtk/server.py b/gui/gtk/server.py index 8c6ed1e055e..da11738f1ee 100644 --- a/gui/gtk/server.py +++ b/gui/gtk/server.py @@ -102,83 +102,34 @@ def continously_get_changes(self): coming from other instances of Faraday. Return the thread on any exception, of if self.stream is None. """ - # There is very arcane, dark magic involved in this method. - # What you need to know: do not touch it. - # If you touch it, do check out persitence/server/changes_stream.py - # there lies _most_ of the darkest magic - - def filter_changes(change, obj_type): - local_changes = models.local_changes() - cool_types = ("Host", "Interface", "Service", "Vulnerability", - "VulnerabilityWeb", "CommandRunInfomation", "Cred", - "Note") - - if not change.get('changes') or not change['changes'][0].get('rev'): - # not a change really right? - return None - - is_deleted = bool(change.get('deleted')) - if obj_type is None and not is_deleted: - # if obj_type is None it's a deleted change. retrieve its type later - return None - - if obj_type is not None and obj_type not in cool_types: - return None - - if change['changes'][0]['rev'] == local_changes.get(change['id']): - del local_changes[change['id']] - return None - - if not change or change.get('last_seq'): - return None - - if change['id'].startswith('_design'): # XXX: is this still neccesary? - return None - - return change - - def notification_dispatcher(obj_id, obj_type, obj_name, deleted, revision): - if deleted: - obj_name, obj_type = self.get_deleted_object_name_and_type(obj_id) - notification_center.deleteObject(obj_id) - update = False - else: - is_new_object = revision.split("-")[0] == "1" - obj = self.get_object(obj_type, obj_id) - if obj: - if is_new_object: - notification_center.addObject(obj) - update = False - else: - notification_center.editObject(obj) - update = True - else: - update = False - for var in [var for var in [obj_id, obj_type, obj_name] if var is None]: - var = "" - notification_center.changeFromInstance(obj_id, obj_type, - obj_name, deleted=deleted, - update=update) def get_changes(): - # dark maaaaaagic *sing with me!* dark maaaaaagic + if not self.stream: + return False while True: - if self.stream: - try: - for change, obj_type, obj_name in self.stream: - print(change, obj_type, obj_name) - with self.changes_lock: - change = filter_changes(change, obj_type) - if change: - deleted = bool(change.get('deleted')) - obj_id = change.get('id') - revision = change.get("changes")[-1].get('rev') - notification_dispatcher(obj_id, obj_type, obj_name, - deleted, revision) - except server_io_exceptions.ChangesStreamStoppedAbruptly: - notification_center.WorkspaceProblem() - return False - else: + try: + for obj_information in self.stream: + action = obj_information.get('action') + obj_id = obj_information.get('id') + obj_type = obj_information.get('type') + obj_name = obj_information.get('name') + obj = self.get_object(obj_type, obj_id) + if action == 'CREATE': + notification_center.addObject(obj) + elif action == 'UPDATE': + notification_center.editObject(obj) + elif action == 'DELETE': + notification_center.deleteObject(obj_id) + else: + raise Exception('Invalid action') + notification_center.changeFromInstance( + obj_id, + obj_type, + obj_name, + deleted=action == 'DELETE', + update=action == 'UPDATE') + except server_io_exceptions.ChangesStreamStoppedAbruptly: + notification_center.WorkspaceProblem() return False time.sleep(0.5) diff --git a/persistence/server/changes_stream.py b/persistence/server/changes_stream.py index 54a22aa55a4..3748bd9b939 100644 --- a/persistence/server/changes_stream.py +++ b/persistence/server/changes_stream.py @@ -112,7 +112,7 @@ def __iter__(self): data = json.loads(self.changes_queue.get_nowait()) except Empty: raise StopIteration - yield data[0], data[1], data[2] + yield data def _get_object_type_and_name_from_change(self, change): try: diff --git a/server/app.py b/server/app.py index 7cb5d3bebdf..d0f67dcd763 100644 --- a/server/app.py +++ b/server/app.py @@ -37,3 +37,6 @@ class MiniJSONEncoder(flask.json.JSONEncoder): # Load APIs import server.api import server.modules.info + +# Load SQLAlchemy Events +import server.events From 851cc3a5c6ca6b75505a1cb75f1fb5377ddac16f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 5 Jun 2017 11:37:43 -0300 Subject: [PATCH 0003/1506] ADD twisted websockets factories and sqlalchemy events --- server/events.py | 15 +++++++ server/websocket_factories.py | 81 +++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 server/events.py create mode 100644 server/websocket_factories.py diff --git a/server/events.py b/server/events.py new file mode 100644 index 00000000000..d88f14926bb --- /dev/null +++ b/server/events.py @@ -0,0 +1,15 @@ +from models import Host +from sqlalchemy import event + +from server.websocket_factories import changes_queue + + +@event.listens_for(Host, 'after_insert') +def receive_after_insert_host(mapper, connection, target): + msg = { + 'id': target.id, + 'action': 'CREATE', + 'type': 'Host', + 'name': target.name + } + changes_queue.put(msg) diff --git a/server/websocket_factories.py b/server/websocket_factories.py new file mode 100644 index 00000000000..7a6f6f40f2b --- /dev/null +++ b/server/websocket_factories.py @@ -0,0 +1,81 @@ +import json +import Cookie +from Queue import Queue, Empty + +from twisted.internet import reactor +# from twisted.python import log + +from autobahn.twisted.websocket import ( + WebSocketServerFactory, + WebSocketServerProtocol +) + +changes_queue = Queue() + + +class BroadcastServerProtocol(WebSocketServerProtocol): + + def onConnect(self, request): + protocol, headers = None, {} + # see if there already is a cookie set .. + print request + if 'cookie' in request.headers: + print 'cookie' + print (str(request.headers['cookie'])) + print 'cookie' + try: + cookie = Cookie.SimpleCookie() + cookie.load(str(request.headers['cookie'])) + except Cookie.CookieError: + pass + return (protocol, headers) + + def onOpen(self): + self.factory.register(self) + + def onMessage(self, payload, isBinary): + if not isBinary: + msg = "{} from {}".format(payload.decode('utf8'), self.peer) + self.factory.broadcast(msg) + + def connectionLost(self, reason): + WebSocketServerProtocol.connectionLost(self, reason) + self.factory.unregister(self) + + +class WorkspaceServerFactory(WebSocketServerFactory): + + def __init__(self, url): + WebSocketServerFactory.__init__(self, url) + self.workspace_clients = [] + self.tickcount = 0 + self.tick() + + def tick(self): + try: + msg = changes_queue.get_nowait() + self.broadcast(json.dumps(msg)) + except Empty: + pass + reactor.callLater(0.5, self.tick) + + def register(self, client): + print 'register' + print client + print 'register' + + if client not in self.workspace_clients: + print("registered client {}".format(client.peer)) + self.workspace_clients.append(client) + + def unregister(self, client): + if client in self.workspace_clients: + print("unregistered client {}".format(client.peer)) + self.workspace_clients.remove(client) + + def broadcast(self, msg): + print("broadcasting prepared message '{}' ..".format(msg)) + preparedMsg = self.prepareMessage(msg) + for client in self.workspace_clients: + reactor.callFromThread(client.sendPreparedMessage, preparedMsg) + print("prepared message sent to {}".format(client.peer)) From 5d7fd55d2c73ed654e4a923bebf5cf710e7072d9 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 13 Jun 2017 14:54:21 -0300 Subject: [PATCH 0004/1506] ADD improve gtk code to handle events. add orm events for Service object Some logic from gtk events removed since we have object type information. Add more ORM events for Service model. Also added update and delete events. --- gui/customevents.py | 20 ++++++------ gui/gtk/application.py | 2 +- gui/gtk/mainwidgets.py | 9 ++---- gui/gtk/server.py | 9 +++--- gui/notifier.py | 14 ++++----- persistence/server/changes_stream.py | 2 +- server/events.py | 46 ++++++++++++++++++++++++---- 7 files changed, 65 insertions(+), 37 deletions(-) diff --git a/gui/customevents.py b/gui/customevents.py index 51a67a61e42..b72eacd57ec 100644 --- a/gui/customevents.py +++ b/gui/customevents.py @@ -144,23 +144,22 @@ def __init__(self, host_id): class ChangeFromInstanceCustomEvent(CustomEvent): - def __init__(self, object_id, object_type, object_name, - deleted=False, update=False): + def __init__(self, action, object_id, object_type, object_name): CustomEvent.__init__(self, CHANGEFROMINSTANCE) self.object_id = object_id self.object_type = object_type self.object_name = object_name - self.deleted = deleted - self.updated_or_created = "updated" if update else "created" + self.action = action def __str__(self): - if not self.object_type or not self.object_name and self.deleted: - return "An object was deleted" - if self.deleted: - return "The {0} {1} was deleted".format(self.object_type, self.object_name) + action_msg = { + 'UPDATE': 'updated', + 'CREATE': 'created', + 'DELETE': 'deleted', + } return "The {0} {1} was {2}".format(self.object_type, self.object_name, - self.updated_or_created) + action_msg[self.action]) class AddObjectCustomEvent(CustomEvent): def __init__(self, new_obj): @@ -168,9 +167,10 @@ def __init__(self, new_obj): self.new_obj = new_obj class DeleteObjectCustomEvent(CustomEvent): - def __init__(self, obj_id): + def __init__(self, obj_id, obj_type): CustomEvent.__init__(self, DELETEOBJECT) self.obj_id = obj_id + self.obj_type = obj_type class UpdateObjectCustomEvent(CustomEvent): def __init__(self, obj): diff --git a/gui/gtk/application.py b/gui/gtk/application.py index f6c30a9d82c..2be79472547 100644 --- a/gui/gtk/application.py +++ b/gui/gtk/application.py @@ -566,7 +566,7 @@ def add_object(): def delete_object(): if event.obj_id: - GObject.idle_add(self.hosts_sidebar.remove_object, event.obj_id) + GObject.idle_add(self.hosts_sidebar.remove_object, event.obj_id, event.obj_type) host_count, service_count, vuln_count = self.update_counts() GObject.idle_add(self.statusbar.update_ws_info, host_count, service_count, vuln_count) diff --git a/gui/gtk/mainwidgets.py b/gui/gtk/mainwidgets.py index 9ca430bd819..e42a915ddca 100644 --- a/gui/gtk/mainwidgets.py +++ b/gui/gtk/mainwidgets.py @@ -9,7 +9,6 @@ import gi import os import math -import operator import webbrowser gi.require_version('Gtk', '3.0') @@ -397,11 +396,9 @@ def add_object(self, obj): if object_type == "Vulnerability" or object_type == "VulnerabilityWeb": self.add_vuln(vuln=obj) - def remove_object(self, obj_id): + def remove_object(self, obj_id, obj_type): """Remove an obj of id obj_id from the model, if found there""" - potential_host_id = obj_id.split('.')[0] - is_host = len(obj_id.split('.')) == 1 - if is_host: + if obj_type == 'Host': self.remove_host(host_id=obj_id) # elif not is_host and self._is_vuln_of_host(vuln_id=obj_id, host_id=potential_host_id): # self.remove_vuln(vuln_id=obj_id) @@ -409,7 +406,7 @@ def remove_object(self, obj_id): # Since we don't know the type of the delete object, # we have to assume it's a vulnerability so the host's # name is updated with the ammount of vulns - host = self.get_single_host_function(potential_host_id) + host = self.get_single_host_function(obj_id) if host: self._modify_vuln_amount_of_single_host_in_model(host.getID(), host.getVulnAmount()) diff --git a/gui/gtk/server.py b/gui/gtk/server.py index da11738f1ee..3e6985ac56a 100644 --- a/gui/gtk/server.py +++ b/gui/gtk/server.py @@ -77,7 +77,7 @@ def get_object(self, object_signature, object_id): def get_host(self, host_id): return models.get_host(self.active_workspace, host_id) - @safe_io_with_server((0,0,0,0)) + @safe_io_with_server((0, 0, 0, 0)) def get_workspace_numbers(self): return models.get_workspace_numbers(self.active_workspace) @@ -119,15 +119,14 @@ def get_changes(): elif action == 'UPDATE': notification_center.editObject(obj) elif action == 'DELETE': - notification_center.deleteObject(obj_id) + notification_center.deleteObject(obj_id, obj_type) else: raise Exception('Invalid action') notification_center.changeFromInstance( + action, obj_id, obj_type, - obj_name, - deleted=action == 'DELETE', - update=action == 'UPDATE') + obj_name) except server_io_exceptions.ChangesStreamStoppedAbruptly: notification_center.WorkspaceProblem() return False diff --git a/gui/notifier.py b/gui/notifier.py index 16a28bc881c..0563938e488 100644 --- a/gui/notifier.py +++ b/gui/notifier.py @@ -75,13 +75,11 @@ def conflictUpdate(self, vulns_changed): def conflictResolution(self, conflicts): self._notifyWidgets(events.ResolveConflictsCustomEvent(conflicts)) - def changeFromInstance(self, obj_id, obj_type, obj_name, - deleted=False, update=False): - self._notifyWidgets(events.ChangeFromInstanceCustomEvent(obj_id, + def changeFromInstance(self, action, obj_id, obj_type, obj_name): + self._notifyWidgets(events.ChangeFromInstanceCustomEvent(action, + obj_id, obj_type, - obj_name, - deleted=deleted, - update=update)) + obj_name)) def addHostFromChanges(self, obj): self._notifyWidgets(events.AddHostChangesEvent(obj)) @@ -89,8 +87,8 @@ def addHostFromChanges(self, obj): def editObject(self, obj): self._notifyWidgets(events.UpdateObjectCustomEvent(obj)) - def deleteObject(self, obj_id): - self._notifyWidgets(events.DeleteObjectCustomEvent(obj_id)) + def deleteObject(self, obj_id, obj_type): + self._notifyWidgets(events.DeleteObjectCustomEvent(obj_id, obj_type)) def addObject(self, new_object): self._notifyWidgets(events.AddObjectCustomEvent(new_object)) diff --git a/persistence/server/changes_stream.py b/persistence/server/changes_stream.py index 3748bd9b939..18d3ac98aad 100644 --- a/persistence/server/changes_stream.py +++ b/persistence/server/changes_stream.py @@ -6,7 +6,6 @@ See the file 'doc/LICENSE' for the license information ''' -import time import json import threading from Queue import Queue, Empty @@ -69,6 +68,7 @@ def stop(self): class WebsocketsChangesStream(ChangesStream): + def __init__(self, workspace_name, server_url, **params): self._base_url = server_url self.changes_queue = Queue() diff --git a/server/events.py b/server/events.py index d88f14926bb..992d510c596 100644 --- a/server/events.py +++ b/server/events.py @@ -1,15 +1,49 @@ -from models import Host +from models import ( + Host, + Service, +) from sqlalchemy import event from server.websocket_factories import changes_queue -@event.listens_for(Host, 'after_insert') -def receive_after_insert_host(mapper, connection, target): +def new_object_event(mapper, connection, instance): msg = { - 'id': target.id, + 'id': instance.id, 'action': 'CREATE', - 'type': 'Host', - 'name': target.name + 'type': instance.__class__.__name__, + 'name': instance.name } changes_queue.put(msg) + + +def delete_object_event(mapper, connection, instance): + msg = { + 'id': instance.id, + 'action': 'DELETE', + 'type': instance.__class__.__name__, + 'name': instance.name + } + changes_queue.put(msg) + + +def update_object_event(mapper, connection, instance): + msg = { + 'id': instance.id, + 'action': 'UPDATE', + 'type': instance.__class__.__name__, + 'name': instance.name + } + changes_queue.put(msg) + +# New object bindings +event.listen(Host, 'after_insert', new_object_event) +event.listen(Service, 'after_insert', new_object_event) + +# Delete object bindings +event.listen(Host, 'after_delete', delete_object_event) +event.listen(Service, 'after_delete', delete_object_event) + +# Update object bindings +event.listen(Host, 'after_update', update_object_event) +event.listen(Service, 'after_update', update_object_event) From cbbe739322a41ce98761ce73e88dddbc30661253 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 14 Jun 2017 19:22:04 -0300 Subject: [PATCH 0005/1506] ADD join workspace feature without authentication --- persistence/server/changes_stream.py | 17 ++++-- server/events.py | 15 ++++-- server/websocket_factories.py | 80 +++++++++++++++++++--------- 3 files changed, 82 insertions(+), 30 deletions(-) diff --git a/persistence/server/changes_stream.py b/persistence/server/changes_stream.py index 18d3ac98aad..55a3bb15582 100644 --- a/persistence/server/changes_stream.py +++ b/persistence/server/changes_stream.py @@ -79,13 +79,25 @@ def __init__(self, workspace_name, server_url, **params): "ws://{0}:9000".format(self._base_url), on_message=self.on_message, on_error=self.on_error, + on_open=self.on_open, on_close=self.on_close) - - self.ws.on_open = self.on_open + # ws.run_forever will call on_message, on_error, on_close and on_open + # see websocket client python docs on: + # https://github.com/websocket-client/websocket-client thread = threading.Thread(target=self.ws.run_forever, args=()) thread.daemon = True thread.start() + def stop(self): + self.ws.close() + super(WebsocketsChangesStream, self).stop() + + def on_open(self, ws): + self.ws.send(json.dumps({ + 'action': 'JOIN_WORKSPACE', + 'workspace': self.workspace_name, + })) + def on_message(self, ws, message): self.changes_queue.put(message) @@ -95,7 +107,6 @@ def on_error(self, ws, error): def on_close(selg, ws): print "### closed ###" - def on_open(self, ws): print 'open' def __enter__(self): diff --git a/server/events.py b/server/events.py index 992d510c596..fcb485a15d4 100644 --- a/server/events.py +++ b/server/events.py @@ -8,34 +8,43 @@ def new_object_event(mapper, connection, instance): + # Since we don't have jet a model for workspace we + # retrieve the name from the connection string + workspace_name = repr(connection.engine).split('.')[0].split('/')[-1] msg = { 'id': instance.id, 'action': 'CREATE', 'type': instance.__class__.__name__, - 'name': instance.name + 'name': instance.name, + 'workspace': workspace_name } changes_queue.put(msg) def delete_object_event(mapper, connection, instance): + workspace_name = repr(connection.engine).split('.')[0].split('/')[-1] msg = { 'id': instance.id, 'action': 'DELETE', 'type': instance.__class__.__name__, - 'name': instance.name + 'name': instance.name, + 'workspace': workspace_name } changes_queue.put(msg) def update_object_event(mapper, connection, instance): + workspace_name = repr(connection.engine).split('.')[0].split('/')[-1] msg = { 'id': instance.id, 'action': 'UPDATE', 'type': instance.__class__.__name__, - 'name': instance.name + 'name': instance.name, + 'workspace': workspace_name } changes_queue.put(msg) + # New object bindings event.listen(Host, 'after_insert', new_object_event) event.listen(Service, 'after_insert', new_object_event) diff --git a/server/websocket_factories.py b/server/websocket_factories.py index 7a6f6f40f2b..d955207afe6 100644 --- a/server/websocket_factories.py +++ b/server/websocket_factories.py @@ -1,5 +1,6 @@ import json import Cookie +from collections import defaultdict from Queue import Queue, Empty from twisted.internet import reactor @@ -10,6 +11,7 @@ WebSocketServerProtocol ) + changes_queue = Queue() @@ -30,13 +32,20 @@ def onConnect(self, request): pass return (protocol, headers) - def onOpen(self): - self.factory.register(self) - - def onMessage(self, payload, isBinary): - if not isBinary: - msg = "{} from {}".format(payload.decode('utf8'), self.peer) - self.factory.broadcast(msg) + def onMessage(self, payload, is_binary): + """ + We only support JOIN and LEAVE workspace messages. + When authentication is implemented we need to verify + that the user can join the selected workspace. + When authentication is implemented we need to reply + the client if the join failed. + """ + if not is_binary: + message = json.loads(payload) + if message['action'] == 'JOIN_WORKSPACE': + self.factory.join_workspace(self, message['workspace']) + if message['action'] == 'LEAVE_WORKSPACE': + self.factory.leave_workspace(self, message['workspace']) def connectionLost(self, reason): WebSocketServerProtocol.connectionLost(self, reason) @@ -44,14 +53,30 @@ def connectionLost(self, reason): class WorkspaceServerFactory(WebSocketServerFactory): + """ + This factory uses the changes_queue to broadcast + message via websockets. + + Any message put on that queue will be sent to clients. + Clients subscribe to workspace channels. + This factory will broadcast message to clients subscribed + on workspace. + + The message in the queue must contain the workspace. + """ def __init__(self, url): WebSocketServerFactory.__init__(self, url) - self.workspace_clients = [] - self.tickcount = 0 + # this dict has a key for each channel + # values are list of clients. + self.workspace_clients = defaultdict(list) self.tick() def tick(self): + """ + Uses changes_queue to broadcast messages to clients. + broadcast method knowns each client workspace. + """ try: msg = changes_queue.get_nowait() self.broadcast(json.dumps(msg)) @@ -59,23 +84,30 @@ def tick(self): pass reactor.callLater(0.5, self.tick) - def register(self, client): - print 'register' - print client - print 'register' - - if client not in self.workspace_clients: + def join_workspace(self, client, workspace): + print('Join workspace {0}'.format(workspace)) + if client not in self.workspace_clients[workspace]: print("registered client {}".format(client.peer)) - self.workspace_clients.append(client) - - def unregister(self, client): - if client in self.workspace_clients: - print("unregistered client {}".format(client.peer)) - self.workspace_clients.remove(client) + self.workspace_clients[workspace].append(client) + + def leave_workspace(self, client, workspace_name): + print('Leave workspace {0}'.format(workspace_name)) + self.workspace_clients[workspace_name].remove(client) + + def unregister(self, client_to_unregister): + """ + Search for the client_to_unregister in all workspaces + """ + for workspace_name, clients in self.workspace_clients.items(): + for client in clients: + if client == client_to_unregister: + print("unregistered client from workspace {0}".format(workspace_name)) + self.leave_workspace(client, workspace_name) + return def broadcast(self, msg): print("broadcasting prepared message '{}' ..".format(msg)) - preparedMsg = self.prepareMessage(msg) - for client in self.workspace_clients: - reactor.callFromThread(client.sendPreparedMessage, preparedMsg) + prepared_msg = json.loads(self.prepareMessage(msg).payload) + for client in self.workspace_clients[prepared_msg['workspace']]: + reactor.callFromThread(client.sendPreparedMessage, self.prepareMessage(msg)) print("prepared message sent to {}".format(client.peer)) From fa510a2ad4c6078fb591541bdeb64dabbd743268 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 6 Jul 2017 13:37:28 -0300 Subject: [PATCH 0006/1506] Add initial jenkins file. The jenkins file current tasks are: * Download requirements * Generates some reports * Execute unit tests --- Jenkinsfile | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000000..8d0176de1f4 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,63 @@ +#!groovy +node { + def ENV_PATH = "$HOME/venv/faraday" + echo "${ENV_PATH}" + + stage("Clean virtualenv") { + sh "rm -rf ${ENV_PATH}" + } + + stage("Install Python Virtual Enviroment") { + sh "virtualenv --no-site-packages ${ENV_PATH}" + } + + // Get the latest version of our application code. + stage ("Pull Code from SCM") { + checkout scm + } + + stage ("Install Application Dependencies") { + sh """ + source ${ENV_PATH}/bin/activate + pip install nose nosexcover virtualenv responses + pip install -r $WORKSPACE/requirements.txt + pip install -r $WORKSPACE/requirements_server.txt + deactivate + """ + } + + stage ("Check code style") { + sh """ + sloccount --duplicates --wide --details $WORKSPACE | fgrep -v .git > $WORKSPACE/sloccount.sc || : + find $WORKSPACE -name \\"*.py\\" | egrep -v '^./tests/' | xargs pyflakes > $WORKSPACE/pyflakes.log || : + find $WORKSPACE -name \\"*.py\\" | egrep -v '^./tests/' | xargs pylint --output-format=parseable --reports=y > $WORKSPACE/pylint.log || : + eslint -c /home/faraday/.eslintrc.js -f checkstyle $WORKSPACE/server/www/scripts/**/* > eslint.xml || true + + """ + warnings canComputeNew: false, canResolveRelativePaths: false, consoleParsers: [[parserName: 'PyFlakes']], defaultEncoding: '', excludePattern: '', healthy: '', includePattern: '', messagesPattern: '', parserConfigurations: [[parserName: 'AcuCobol Compiler', pattern: 'pyflakes.log']], unHealthy: '' + + } + + stage ("Run Unit/Integration Tests") { + def testsError = null + try { + sh """ + source ${ENV_PATH}/bin/activate + cd $WORKSPACE && nosetests --verbose --with-xunit --xunit-file=$WORKSPACE/xunit.xml --with-xcoverage --xcoverage-file=$WORKSPACE/coverage.xml -ignore-files='.*dont_run_rest_controller_apis.*' --no-byte-compile -v `find test_cases -name '*.py'| grep -v dont_run` || : + deactivate + """ + step([$class: 'CoberturaPublisher', autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: '**/coverage.xml', failNoReports: false, failUnhealthy: false, failUnstable: false, maxNumberOfBuilds: 0, onlyStable: false, sourceEncoding: 'ASCII', zoomCoverageChart: false]) + } + catch(err) { + testsError = err + currentBuild.result = 'FAILURE' + } + finally { + junit "**/xunit.xml" + + if (testsError) { + throw testsError + } + } + } +} From 42b06106940a3472d7e34a384cb0ea2ef6c3d6e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 13 Jul 2017 18:22:43 -0300 Subject: [PATCH 0007/1506] Add flask_security test code It is the code in https://github.com/mattupstate/flask-security/blob/develop/docs/quickstart.rst#sqlalchemy-application-1 with a few modifications. --- requirements_server.txt | 2 ++ server/app.py | 27 ++++++++++++++++++++++++++- server/database.py | 15 +++++++++++++++ server/models.py | 35 +++++++++++++++++++++++++++++++++-- 4 files changed, 76 insertions(+), 3 deletions(-) diff --git a/requirements_server.txt b/requirements_server.txt index 7fe736a1fcb..0f680874e7f 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -7,3 +7,5 @@ sqlalchemy>=1.0.12 pyopenssl>=16.0.0 service_identity>=16.0.0 pyasn1-modules +flask-security +bcrypt diff --git a/server/app.py b/server/app.py index 9d5b7afed4f..6932f998471 100644 --- a/server/app.py +++ b/server/app.py @@ -3,13 +3,38 @@ # See the file 'doc/LICENSE' for the license information import flask +from flask_security import Security, login_required, \ + SQLAlchemySessionUserDatastore + import server.config import server.database - +import server.models from server.utils.logger import LOGGING_HANDLERS app = flask.Flask(__name__) +app.config['SECRET_KEY'] = 'supersecret' +app.config['SECURITY_PASSWORD_SINGLE_HASH'] = True + +# Setup Flask-Security +user_datastore = SQLAlchemySessionUserDatastore(server.database.common_session, + server.models.User, + server.models.Role) +security = Security(app, user_datastore) + +# Create a user to test with +@app.before_first_request +def create_user(): + server.database.init_common_db() + user_datastore.create_user(email='matt@nobien.net', password='password') + server.database.common_session.commit() + +# Views +@app.route('/test') +@login_required +def home(): + return 'Logged in!!' + def setup(): app.debug = server.config.is_debug_mode() diff --git a/server/database.py b/server/database.py index c4a12d1fbb4..79c2dcc60e8 100644 --- a/server/database.py +++ b/server/database.py @@ -441,3 +441,18 @@ class WorkspaceNotFound(Exception): def __init__(self, workspace_name): super(WorkspaceNotFound, self).__init__('Workspace "%s" not found' % workspace_name) + +# +# Connection to a database common to al workspaces +# + +common_engine = create_engine('sqlite://') +common_session = scoped_session(sessionmaker(autocommit=False, + autoflush=False, + bind=common_engine)) +server.models.CommonBase.metadata.bind = common_engine + +def init_common_db(): + from server.models import CommonBase + CommonBase.metadata.create_all(bind=common_engine) + CommonBase.query = common_session.query_property() diff --git a/server/models.py b/server/models.py index 145e142d5fc..bf0fb878766 100644 --- a/server/models.py +++ b/server/models.py @@ -4,14 +4,17 @@ import json -from sqlalchemy import Column, Integer, String, Boolean, ForeignKey, Float, Text, UniqueConstraint -from sqlalchemy.orm import relationship +from sqlalchemy import Column, Integer, String, Boolean, ForeignKey, Float, \ + Text, UniqueConstraint, DateTime +from sqlalchemy.orm import relationship, backref from sqlalchemy.ext.declarative import declarative_base +from flask_security import UserMixin, RoleMixin SCHEMA_VERSION = 'W.2.6.0' Base = declarative_base() +CommonBase = declarative_base() class EntityNotFound(Exception): def __init__(self, entity_id): @@ -493,3 +496,31 @@ def update_from_document(self, document): self.user = document.get('user', None) self.workspace = document.get('workspace', None) + +class RolesUsers(CommonBase): + __tablename__ = 'roles_users' + id = Column(Integer(), primary_key=True) + user_id = Column('user_id', Integer(), ForeignKey('user.id')) + role_id = Column('role_id', Integer(), ForeignKey('role.id')) + +class Role(CommonBase, RoleMixin): + __tablename__ = 'role' + id = Column(Integer(), primary_key=True) + name = Column(String(80), unique=True) + description = Column(String(255)) + +class User(CommonBase, UserMixin): + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + email = Column(String(255), unique=True) + username = Column(String(255)) + password = Column(String(255)) + last_login_at = Column(DateTime()) + current_login_at = Column(DateTime()) + last_login_ip = Column(String(100)) + current_login_ip = Column(String(100)) + login_count = Column(Integer) + active = Column(Boolean()) + confirmed_at = Column(DateTime()) + roles = relationship('Role', secondary='roles_users', + backref=backref('users', lazy='dynamic')) From 8eb47b124196c475a8c00cdc3044e40ee0a73a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 14 Jul 2017 17:01:09 -0300 Subject: [PATCH 0008/1506] Replace login redirects for 403 errors --- server/app.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/app.py b/server/app.py index 6932f998471..5c4f2384164 100644 --- a/server/app.py +++ b/server/app.py @@ -22,6 +22,12 @@ server.models.Role) security = Security(app, user_datastore) +# We are exposing a RESTful API, so don't redirect a user to a login page in +# case of being unauthorized, raise a 403 error instead +@app.login_manager.unauthorized_handler +def unauthorized(): + flask.abort(403) + # Create a user to test with @app.before_first_request def create_user(): From e9a0cf42137fffac859694e9d2a48cee3e540269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 14 Jul 2017 18:36:55 -0300 Subject: [PATCH 0009/1506] Require logged user to access API endpoints --- server/app.py | 28 ++++++++++++++++------------ server/database.py | 2 +- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/server/app.py b/server/app.py index 5c4f2384164..6a8c4279dec 100644 --- a/server/app.py +++ b/server/app.py @@ -29,18 +29,22 @@ def unauthorized(): flask.abort(403) # Create a user to test with -@app.before_first_request -def create_user(): - server.database.init_common_db() - user_datastore.create_user(email='matt@nobien.net', password='password') - server.database.common_session.commit() - -# Views -@app.route('/test') -@login_required -def home(): - return 'Logged in!!' - +# @app.before_first_request +# def create_user(): +server.database.init_common_db() +user_datastore.create_user(email='matt@nobien.net', password='password') +server.database.common_session.commit() + +# Make API endpoints require a login user by default. Based on +# https://stackoverflow.com/questions/13428708/best-way-to-make-flask-logins-login-required-the-default +app.view_functions['security.login'].is_public = True +app.view_functions['security.logout'].is_public = True +@app.before_request +def default_login_required(): + view = app.view_functions.get(flask.request.endpoint) + logged_in = 'user_id' in flask.session + if (not logged_in and not getattr(view, 'is_public', False)): + flask.abort(403) def setup(): app.debug = server.config.is_debug_mode() diff --git a/server/database.py b/server/database.py index 79c2dcc60e8..05e72c082a1 100644 --- a/server/database.py +++ b/server/database.py @@ -446,7 +446,7 @@ def __init__(self, workspace_name): # Connection to a database common to al workspaces # -common_engine = create_engine('sqlite://') +common_engine = create_engine('sqlite:////tmp/test.db') common_session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=common_engine)) From 272ccd6666148588629b67ff1fc8af0320173353 Mon Sep 17 00:00:00 2001 From: Andres Date: Sat, 15 Jul 2017 15:04:33 -0300 Subject: [PATCH 0010/1506] #4148 fix hostnames, buttons, status and severity row alignment --- server/dao/vuln.py | 2 +- .../statusReport/partials/ui-grid/columns/severitycolumn.html | 2 +- .../statusReport/partials/ui-grid/columns/statuscolumn.html | 2 +- .../scripts/statusReport/partials/ui-grid/confirmbutton.html | 2 +- .../www/scripts/statusReport/partials/ui-grid/deletebutton.html | 2 +- .../www/scripts/statusReport/partials/ui-grid/editbutton.html | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/dao/vuln.py b/server/dao/vuln.py index a2376ba2c29..27df6b07864 100644 --- a/server/dao/vuln.py +++ b/server/dao/vuln.py @@ -190,7 +190,7 @@ def get_parent_id(couchdb_id): 'tags': [], 'type': vuln.document_type, 'target': host.name, - 'hostnames': hostnames.split(',') if hostnames else '', + 'hostnames': filter(None, hostnames.split(',')) if hostnames else '', 'service': "(%s/%s) %s" % (service.ports, service.protocol, service.s_name) if service.ports else '' }} diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html index 38d4231cf26..742e68662dd 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html @@ -1 +1 @@ -
{{COL_FIELD | uppercase}}
\ No newline at end of file +
{{COL_FIELD | uppercase}}
\ No newline at end of file diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html index f3017594dbf..654f2f50783 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html @@ -1 +1 @@ -
{{COL_FIELD | uppercase}}
\ No newline at end of file +
{{COL_FIELD | uppercase}}
\ No newline at end of file diff --git a/server/www/scripts/statusReport/partials/ui-grid/confirmbutton.html b/server/www/scripts/statusReport/partials/ui-grid/confirmbutton.html index 23cc629fdf0..f6f79e1fc24 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/confirmbutton.html +++ b/server/www/scripts/statusReport/partials/ui-grid/confirmbutton.html @@ -1,3 +1,3 @@ -
+
\ No newline at end of file diff --git a/server/www/scripts/statusReport/partials/ui-grid/deletebutton.html b/server/www/scripts/statusReport/partials/ui-grid/deletebutton.html index cfd0eb53185..bc52917722e 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/deletebutton.html +++ b/server/www/scripts/statusReport/partials/ui-grid/deletebutton.html @@ -1,3 +1,3 @@ -
+
\ No newline at end of file diff --git a/server/www/scripts/statusReport/partials/ui-grid/editbutton.html b/server/www/scripts/statusReport/partials/ui-grid/editbutton.html index ebb1c761d47..1824ba2945f 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/editbutton.html +++ b/server/www/scripts/statusReport/partials/ui-grid/editbutton.html @@ -1,3 +1,3 @@ -
+
\ No newline at end of file From 68cdd981343dcbeeab31f4f6962f3ac7c478775c Mon Sep 17 00:00:00 2001 From: Andres Date: Sat, 15 Jul 2017 16:38:35 -0300 Subject: [PATCH 0011/1506] Fix #4148 Limit severity word length --- .../statusReport/partials/ui-grid/columns/severitycolumn.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html index 742e68662dd..214fbe0a547 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html @@ -1 +1 @@ -
{{COL_FIELD | uppercase}}
\ No newline at end of file +
{{COL_FIELD | uppercase}}
\ No newline at end of file From 68108197f9bf28ba3a96f682e6fb4cad93e936f2 Mon Sep 17 00:00:00 2001 From: Andres Date: Sat, 15 Jul 2017 16:58:16 -0300 Subject: [PATCH 0012/1506] Fix #4148 Creator column aligment --- .../statusReport/partials/ui-grid/columns/creatorcolumn.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/creatorcolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/creatorcolumn.html index 980e44f0ad9..6eb751c4c5d 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/creatorcolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/creatorcolumn.html @@ -1,6 +1,4 @@
{{COL_FIELD.split("(")[0] !== " " ? COL_FIELD : "EMPTY" + COL_FIELD}}
\ No newline at end of file From aa3ee5298cb06b87543463ebeb44d0620c422a8e Mon Sep 17 00:00:00 2001 From: Andres Date: Sat, 15 Jul 2017 18:50:32 -0300 Subject: [PATCH 0013/1506] Fix #4150 Full width and height grid. --- server/www/estilos.css | 15 ++++++++++--- .../statusReport/controllers/statusReport.js | 22 ++++++++++++++++++- .../statusReport/partials/statusReport.html | 4 ++-- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/server/www/estilos.css b/server/www/estilos.css index cde25056779..795357a3228 100644 --- a/server/www/estilos.css +++ b/server/www/estilos.css @@ -1037,8 +1037,8 @@ div.btn-group .vulns-filter{left: -2px;} div.btn-group .dropdown-menu.margin{margin-left: 38px} /* UI GRID */ .ui-grid-cell.ui-grid-disable-selection.ui-grid-row-header-cell{pointer-events: auto!important;} -.grid { - height: 98vh; +.grid { + height: 100%; } .ui-grid-cell-contents.left-rows.text-center.ng-scope{white-space: normal!important; } .ui-grid .ui-grid-row:nth-child(odd) .ui-grid-cell .ui-grid-cell-contents .crop-text, .ui-grid .ui-grid-row:nth-child(even) .ui-grid-cell .ui-grid-cell-contents .crop-text, .ui-grid-cell-contents{ @@ -1050,7 +1050,7 @@ div.btn-group .dropdown-menu.margin{margin-left: 38px} .white-space{white-space: pre!important; word-wrap: break-word;} .ui-grid-cell-contents.center > .pos-middle.crop-text{top: 2%!important;white-space: inherit!important;} .overflow-cell{ - height: 95px!important; + height: 30px!important; overflow-y: auto!important; } div.ui-grid-header-cell .ui-grid-cell-contents{white-space: normal;} @@ -1059,6 +1059,15 @@ div.alert.alert-danger.alert-dismissible .ws-list a:hover{text-decoration: none} a.button-disable{cursor: not-allowed;pointer-events: none;opacity: 0.5} .almost-expired{background-color: #dca7a7 !important} +.statusReport-Search:before,.statusReport-Search:after { + display: table; + content: " "; +} + +.statusReport-Search:after { + clear: both; +} + #custom-search-input{ border: solid 1px #E4E4E4; border-radius: 6px; diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index a2cb6c1f083..2508fdc6013 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -29,6 +29,8 @@ angular.module('faradayApp') $scope.vulnWebSelected; $scope.confirmed = false; + + $scope.gridHeight; var allVulns; var searchFilter = {}; @@ -64,7 +66,7 @@ angular.module('faradayApp') enableHorizontalScrollbar: 0, treeRowHeaderAlwaysVisible: false, enableGroupHeaderSelection: true, - rowHeight: 95 + rowHeight: 30 }; $scope.gridOptions.columnDefs = []; @@ -202,6 +204,11 @@ angular.module('faradayApp') loadVulns(); + resizeGrid(); + + angular.element($window).bind('resize', function () { + resizeGrid(); + }); }; var defineColumns = function() { @@ -441,6 +448,19 @@ angular.module('faradayApp') return res; }; + var resizeGrid = function() { + $scope.gridHeight = getGridHeight('grid', 'right-main', 15); + } + + var getGridHeight = function(gridClass, contentClass, bottomOffset) { + var contentOffset = angular.element(document.getElementsByClassName(contentClass)[0]).offset(); + var contentHeight = angular.element(document.getElementsByClassName(contentClass)[0]).height(); + var gridOffset = angular.element(document.getElementsByClassName(gridClass)).offset(); + if (gridOffset !== undefined) { + var gridHeight = contentHeight - (gridOffset.top) - bottomOffset; + return gridHeight + 'px'; + } + } $scope.saveAsModel = function() { var self = this; diff --git a/server/www/scripts/statusReport/partials/statusReport.html b/server/www/scripts/statusReport/partials/statusReport.html index cbeaff33672..791edb8e381 100644 --- a/server/www/scripts/statusReport/partials/statusReport.html +++ b/server/www/scripts/statusReport/partials/statusReport.html @@ -98,7 +98,7 @@

Status report for {{ workspace

-
+
@@ -126,7 +126,7 @@

Add columns

-
+
Total From e32bf5d747d9b8b60a160481b6a08e9b1c6c76db Mon Sep 17 00:00:00 2001 From: Andres Date: Sun, 16 Jul 2017 14:55:56 -0300 Subject: [PATCH 0014/1506] FIx #4150 One line row --- server/www/estilos.css | 4 +++ .../statusReport/controllers/statusReport.js | 28 ++++++++++++++----- .../partials/ui-grid/columns/desccolumn.html | 2 +- .../ui-grid/columns/evidencecolumn.html | 4 +-- .../ui-grid/columns/hostnamescolumn.html | 4 +-- .../ui-grid/columns/impactcolumn.html | 4 +-- .../columns/policyviolationscolumn.html | 4 +-- .../partials/ui-grid/columns/refscolumn.html | 4 +-- .../ui-grid/columns/resolutioncolumn.html | 2 +- 9 files changed, 37 insertions(+), 19 deletions(-) diff --git a/server/www/estilos.css b/server/www/estilos.css index 795357a3228..10ff27cf113 100644 --- a/server/www/estilos.css +++ b/server/www/estilos.css @@ -1156,4 +1156,8 @@ a.button-disable{cursor: not-allowed;pointer-events: none;opacity: 0.5} .accordion-expand-button-disabled { display: none !important; +} + +.tooltip-inner { + white-space: pre-wrap; } \ No newline at end of file diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 2508fdc6013..ab407bfbde3 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -3,6 +3,10 @@ // See the file 'doc/LICENSE' for the license information angular.module('faradayApp') + .filter('removeLinesbreak',function(){ + return function(text){ + return text?text.replace(/\n/g, ' '):''; + }}) .controller('statusReportCtrl', ['$scope', '$filter', '$routeParams', '$location', '$uibModal', '$cookies', '$q', '$window', 'BASEURL', @@ -245,7 +249,7 @@ angular.module('faradayApp') displayName : "date", cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/datecolumn.html', headerCellTemplate: header, - width: '90', + width: '75', sort: getColumnSort('date'), visible: $scope.columns["date"] }); @@ -432,12 +436,6 @@ angular.module('faradayApp') $cookies.put('SRsortDirection', sortDirection || ''); } - $scope.ifTooltip = function(text) { - if(text !== undefined && text.length > 450) { - return text; - } - }; - $scope.confirmedTooltip = function(isConfirmed) { var res = ""; if(isConfirmed === true) { @@ -1040,5 +1038,21 @@ angular.module('faradayApp') return $scope.encodeUrl(srvName); } + $scope.concatForTooltip = function (items, isArray, useDoubleLinebreak) { + var elements = []; + for (var property in items) { + if (items.hasOwnProperty(property)) { + if (isArray) { + elements.push(items[property]) + } + else { + elements.push(property) + } + } + } + + return elements.join("\n" + (useDoubleLinebreak ? "\n" : "")); + } + init(); }]); diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/desccolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/desccolumn.html index b922f68a884..6d8c812a2ad 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/desccolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/desccolumn.html @@ -1 +1 @@ -
{{COL_FIELD CUSTOM_FILTERS}}
\ No newline at end of file +
{{COL_FIELD CUSTOM_FILTERS | removeLinesbreak}}
\ No newline at end of file diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/evidencecolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/evidencecolumn.html index 7d6259c502f..94a5951ec6c 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/evidencecolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/evidencecolumn.html @@ -1,4 +1,4 @@ -
-

{{key | decodeURIComponent}}

+
+ {{key | decodeURIComponent}} 
{{COL_FIELD CUSTOM_FILTERS}}
\ No newline at end of file diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/hostnamescolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/hostnamescolumn.html index 3bf0f68cb17..4331717f18a 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/hostnamescolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/hostnamescolumn.html @@ -1,6 +1,6 @@
-
+

{{hostname}}

-
{{COL_FIELD.split('(')[0] !== ' ' ? COL_FIELD : 'EMPTY' + COL_FIELD}}
+
{{COL_FIELD.split('(')[0] !== ' ' ? COL_FIELD : 'EMPTY' + COL_FIELD}}
\ No newline at end of file diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/impactcolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/impactcolumn.html index 405596b1aa3..53d30452071 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/impactcolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/impactcolumn.html @@ -1,6 +1,6 @@
-
-

{{key}}

+
+ {{key}} 
{{COL_FIELD.split('(')[0] !== ' ' ? COL_FIELD : 'EMPTY' + COL_FIELD}}
\ No newline at end of file diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/policyviolationscolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/policyviolationscolumn.html index a1460537bcf..236206ea032 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/policyviolationscolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/policyviolationscolumn.html @@ -1,6 +1,6 @@
-
-

{{policyviolation}}

+
+ {{policyviolation}} 
{{COL_FIELD.split('(')[0] !== ' ' ? COL_FIELD : 'EMPTY' + COL_FIELD}}
\ No newline at end of file diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/refscolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/refscolumn.html index 3e03f2568de..1abc4979da4 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/refscolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/refscolumn.html @@ -1,6 +1,6 @@
-
-

{{ref}}

+
+ {{ref}} 
{{COL_FIELD.split('(')[0] !== ' ' ? COL_FIELD : 'EMPTY' + COL_FIELD}}
\ No newline at end of file diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/resolutioncolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/resolutioncolumn.html index 10649e11bce..cb39e5f8b76 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/resolutioncolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/resolutioncolumn.html @@ -1 +1 @@ -
{{COL_FIELD.split("(")[0] !== " " ? COL_FIELD : "EMPTY" + COL_FIELD}}
\ No newline at end of file +
{{COL_FIELD.split("(")[0] !== " " ? COL_FIELD : "EMPTY" + COL_FIELD | removeLinesbreak}}
\ No newline at end of file From 880e6038041a83a57b8e939a1ec83d5c501156a9 Mon Sep 17 00:00:00 2001 From: Andres Date: Sun, 16 Jul 2017 15:41:07 -0300 Subject: [PATCH 0015/1506] #4151 #4152 Remove buttons --- server/www/estilos.css | 2 +- .../statusReport/controllers/statusReport.js | 8 +---- .../statusReport/partials/statusReport.html | 30 ++++++++++--------- .../partials/ui-grid/deletebutton.html | 3 -- .../partials/ui-grid/editbutton.html | 3 -- 5 files changed, 18 insertions(+), 28 deletions(-) delete mode 100644 server/www/scripts/statusReport/partials/ui-grid/deletebutton.html delete mode 100644 server/www/scripts/statusReport/partials/ui-grid/editbutton.html diff --git a/server/www/estilos.css b/server/www/estilos.css index 10ff27cf113..eb3801f6ea8 100644 --- a/server/www/estilos.css +++ b/server/www/estilos.css @@ -725,7 +725,7 @@ text.host-name{font-size: 13px;font-weight:bold;} justify-content: center; align-items: center; } - #merge, #delete, #new, #tags, #group-by, left{margin:5px;float:right} + #merge, #delete, #new, #tags, #group-by, #add-columns, left{margin:5px;float:right} .left_auths{margin:5px;float:right;margin-top: -52px;} .left_auth{margin:5px;float:right;margin-top: -40px;} diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index ab407bfbde3..11a0badae3a 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -217,9 +217,7 @@ angular.module('faradayApp') var defineColumns = function() { $scope.gridOptions.columnDefs.push({ name: 'selectAll', width: '20', headerCellTemplate: "", pinnedLeft:true }); - $scope.gridOptions.columnDefs.push({ name: 'confirmVuln', width: '40', headerCellTemplate: "
", cellTemplate: 'scripts/statusReport/partials/ui-grid/confirmbutton.html' }); - $scope.gridOptions.columnDefs.push({ name: 'deleteVuln', width: '40', headerCellTemplate: "
", cellTemplate: 'scripts/statusReport/partials/ui-grid/deletebutton.html' }); - $scope.gridOptions.columnDefs.push({ name: 'editVuln', width: '30', headerCellTemplate: "
", cellTemplate: 'scripts/statusReport/partials/ui-grid/editbutton.html' }); + $scope.gridOptions.columnDefs.push({ name: 'confirmVuln', width: '22', headerCellTemplate: "
", cellTemplate: 'scripts/statusReport/partials/ui-grid/confirmbutton.html' }); function getColumnSort(columnName){ if($cookies.get('SRsortColumn') === columnName){ @@ -668,10 +666,6 @@ angular.module('faradayApp') _edit($scope.getCurrentSelection()); }; - $scope.editVuln = function(vuln) { - _edit([vuln]); - }; - var _edit = function(vulns) { if (vulns.length == 1) { var modal = $uibModal.open({ diff --git a/server/www/scripts/statusReport/partials/statusReport.html b/server/www/scripts/statusReport/partials/statusReport.html index 791edb8e381..8318cedd8d4 100644 --- a/server/www/scripts/statusReport/partials/statusReport.html +++ b/server/www/scripts/statusReport/partials/statusReport.html @@ -75,11 +75,24 @@

Status report for {{ workspace
  • Create vulnerability templates

    • +
    + + + +
    -
    +
    -
    -
    -

    Add columns

    - -
    +
    diff --git a/server/www/scripts/statusReport/partials/ui-grid/deletebutton.html b/server/www/scripts/statusReport/partials/ui-grid/deletebutton.html deleted file mode 100644 index bc52917722e..00000000000 --- a/server/www/scripts/statusReport/partials/ui-grid/deletebutton.html +++ /dev/null @@ -1,3 +0,0 @@ -
    - -
    \ No newline at end of file diff --git a/server/www/scripts/statusReport/partials/ui-grid/editbutton.html b/server/www/scripts/statusReport/partials/ui-grid/editbutton.html deleted file mode 100644 index 1824ba2945f..00000000000 --- a/server/www/scripts/statusReport/partials/ui-grid/editbutton.html +++ /dev/null @@ -1,3 +0,0 @@ -
    - -
    \ No newline at end of file From 9555a6a889fd7eac64412b7a108989900c784b3b Mon Sep 17 00:00:00 2001 From: Andres Date: Sun, 16 Jul 2017 16:44:33 -0300 Subject: [PATCH 0016/1506] #4150 Grid height resize on first load --- server/www/estilos.css | 3 +++ .../www/scripts/statusReport/controllers/statusReport.js | 8 +++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/server/www/estilos.css b/server/www/estilos.css index eb3801f6ea8..286825cea0f 100644 --- a/server/www/estilos.css +++ b/server/www/estilos.css @@ -1056,6 +1056,9 @@ div.btn-group .dropdown-menu.margin{margin-left: 38px} div.ui-grid-header-cell .ui-grid-cell-contents{white-space: normal;} div.alert.alert-danger.alert-dismissible .ws-list{text-align: center; text-transform: uppercase; padding: 10px} div.alert.alert-danger.alert-dismissible .ws-list a:hover{text-decoration: none} +.grid .ui-grid-header { + padding: 3px 5px; +} a.button-disable{cursor: not-allowed;pointer-events: none;opacity: 0.5} .almost-expired{background-color: #dca7a7 !important} diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 11a0badae3a..17bc9fabc9b 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -126,6 +126,10 @@ angular.module('faradayApp') } loadVulns(); }); + + $scope.gridApi.core.on.rowsRendered($scope, function() { + resizeGrid(); + }); }; // load all workspaces @@ -208,8 +212,6 @@ angular.module('faradayApp') loadVulns(); - resizeGrid(); - angular.element($window).bind('resize', function () { resizeGrid(); }); @@ -405,7 +407,7 @@ angular.module('faradayApp') visible: $scope.columns["creator"] }); $scope.gridOptions.columnDefs.push({ name : 'policyviolations', - displayName : "policy violations", + displayName : "policyviolations", cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/policyviolationscolumn.html', headerCellTemplate: header, width: '100', From 911ce2690439cecdff8423fadb23bf7e4bde7fcc Mon Sep 17 00:00:00 2001 From: Andres Date: Sun, 16 Jul 2017 18:05:40 -0300 Subject: [PATCH 0017/1506] #4150 click reference on vuln modal edit --- server/www/index.html | 1 + .../statusReport/controllers/modalEdit.js | 9 ++++- .../statusReport/controllers/statusReport.js | 40 ++++--------------- .../statusReport/partials/modalEdit.html | 2 +- .../statusReport/providers/reference.js | 39 ++++++++++++++++++ 5 files changed, 55 insertions(+), 36 deletions(-) create mode 100644 server/www/scripts/statusReport/providers/reference.js diff --git a/server/www/index.html b/server/www/index.html index 070f187447c..e65e87925bd 100644 --- a/server/www/index.html +++ b/server/www/index.html @@ -133,6 +133,7 @@ + diff --git a/server/www/scripts/statusReport/controllers/modalEdit.js b/server/www/scripts/statusReport/controllers/modalEdit.js index 111e004a6f2..e3bba688b82 100644 --- a/server/www/scripts/statusReport/controllers/modalEdit.js +++ b/server/www/scripts/statusReport/controllers/modalEdit.js @@ -3,8 +3,8 @@ // See the file 'doc/LICENSE' for the license information angular.module('faradayApp') - .controller('modalEditCtrl', ['$modalInstance', 'EASEOFRESOLUTION', 'STATUSES', 'commonsFact', 'severities', 'vuln', 'cweFact', - function($modalInstance, EASEOFRESOLUTION, STATUSES, commons, severities, vuln, cweFact) { + .controller('modalEditCtrl', ['$modalInstance', 'EASEOFRESOLUTION', 'STATUSES', 'commonsFact', 'severities', 'vuln', 'cweFact', 'referenceService', + function($modalInstance, EASEOFRESOLUTION, STATUSES, commons, severities, vuln, cweFact, referenceService) { var vm = this; @@ -140,6 +140,11 @@ angular.module('faradayApp') } } + vm.openReference = function(text) + { + window.open(referenceService.processReference(text), '_blank'); + } + vm.newPolicyViolation = function() { if (vm.new_policyviolation != "") { // we need to check if the policy violation already exists diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 17bc9fabc9b..c299dd6050e 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -9,13 +9,14 @@ angular.module('faradayApp') }}) .controller('statusReportCtrl', ['$scope', '$filter', '$routeParams', - '$location', '$uibModal', '$cookies', '$q', '$window', 'BASEURL', - 'SEVERITIES', 'EASEOFRESOLUTION', 'STATUSES', 'hostsManager', 'commonsFact', + '$location', '$uibModal', '$cookies', '$q', '$window', 'BASEURL', + 'SEVERITIES', 'EASEOFRESOLUTION', 'STATUSES', 'hostsManager', 'commonsFact', 'vulnsManager', 'workspacesFact', 'csvService', 'uiGridConstants', 'vulnModelsManager', + 'referenceService', function($scope, $filter, $routeParams, $location, $uibModal, $cookies, $q, $window, BASEURL, SEVERITIES, EASEOFRESOLUTION, STATUSES, hostsManager, commonsFact, - vulnsManager, workspacesFact, csvService, uiGridConstants, vulnModelsManager) { + vulnsManager, workspacesFact, csvService, uiGridConstants, vulnModelsManager, referenceService) { $scope.baseurl; $scope.columns; $scope.easeofresolution; @@ -29,7 +30,7 @@ angular.module('faradayApp') $scope.workspaces; $scope.currentPage; $scope.gridOptions; - $scope.vulnModelsManager; + $scope.vulnModelsManager; $scope.vulnWebSelected; $scope.confirmed = false; @@ -55,7 +56,7 @@ angular.module('faradayApp') $scope.reverse = true; $scope.vulns = []; $scope.selected = false; - $scope.vulnModelsManager = vulnModelsManager; + $scope.vulnModelsManager = vulnModelsManager; $scope.gridOptions = { multiSelect: true, @@ -500,34 +501,7 @@ angular.module('faradayApp') }; $scope.processReference = function(text) { - var url = 'http://google.com/', - url_pattern = new RegExp('^(http|https):\\/\\/?'); - - var cve_pattern = new RegExp(/^CVE-\d{4}-\d{4,7}$/), - cwe_pattern = new RegExp(/^CWE(-|:)\d{1,7}$/), - edb_pattern = new RegExp(/^EDB-ID:\s?\d{1,}$/), - osvdb_pattern = new RegExp(/^OSVDB:\s?\d{1,}$/); - - var cve = text.search(cve_pattern), - cwe = text.search(cwe_pattern), - edb = text.search(edb_pattern), - osvdb = text.search(osvdb_pattern); - - if(url_pattern.test(text)) { - url = text; - } else if(cve > -1) { - url = "https://cve.mitre.org/cgi-bin/cvename.cgi?name=" + text.substring(cve + 4); - } else if(cwe > -1) { - url = "https://cwe.mitre.org/data/definitions/" + text.substring(cwe + 4) + ".html"; - } else if(osvdb > -1) { - url = "http://osvdb.org/show/osvdb/" + text.substring(osvdb + 6); - } else if(edb > -1) { - url = "https://www.exploit-db.com/exploits/" + text.substring(edb + 7); - } else { - url += 'search?q=' + text; - } - - return url; + return referenceService.processReference(text); }; $scope.groupBy = function(property) { diff --git a/server/www/scripts/statusReport/partials/modalEdit.html b/server/www/scripts/statusReport/partials/modalEdit.html index acab6c1fc77..4306e1f56e0 100644 --- a/server/www/scripts/statusReport/partials/modalEdit.html +++ b/server/www/scripts/statusReport/partials/modalEdit.html @@ -75,7 +75,7 @@
    Confirmed
    - +
    diff --git a/server/www/scripts/statusReport/providers/reference.js b/server/www/scripts/statusReport/providers/reference.js new file mode 100644 index 00000000000..75312cce946 --- /dev/null +++ b/server/www/scripts/statusReport/providers/reference.js @@ -0,0 +1,39 @@ +// Faraday Penetration Test IDE +// Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +// See the file 'doc/LICENSE' for the license information + +angular.module('faradayApp') + .factory('referenceService', function () { + return { + processReference: function (text) { + var url = 'http://google.com/', + url_pattern = new RegExp('^(http|https):\\/\\/?'); + + var cve_pattern = new RegExp(/^CVE-\d{4}-\d{4,7}$/), + cwe_pattern = new RegExp(/^CWE(-|:)\d{1,7}$/), + edb_pattern = new RegExp(/^EDB-ID:\s?\d{1,}$/), + osvdb_pattern = new RegExp(/^OSVDB:\s?\d{1,}$/); + + var cve = text.search(cve_pattern), + cwe = text.search(cwe_pattern), + edb = text.search(edb_pattern), + osvdb = text.search(osvdb_pattern); + + if (url_pattern.test(text)) { + url = text; + } else if (cve > -1) { + url = "https://cve.mitre.org/cgi-bin/cvename.cgi?name=" + text.substring(cve + 4); + } else if (cwe > -1) { + url = "https://cwe.mitre.org/data/definitions/" + text.substring(cwe + 4) + ".html"; + } else if (osvdb > -1) { + url = "http://osvdb.org/show/osvdb/" + text.substring(osvdb + 6); + } else if (edb > -1) { + url = "https://www.exploit-db.com/exploits/" + text.substring(edb + 7); + } else { + url += 'search?q=' + text; + } + + return url; + } + } + }); \ No newline at end of file From 36d5192a3040b5452bc1c573826a03af8185d124 Mon Sep 17 00:00:00 2001 From: Andres Date: Sun, 16 Jul 2017 18:37:44 -0300 Subject: [PATCH 0018/1506] #4149 Test changing columns description --- server/www/scripts/statusReport/controllers/statusReport.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index c299dd6050e..965fc18b685 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -235,7 +235,7 @@ angular.module('faradayApp') var header = '
    '+ '
    {{ col.displayName CUSTOM_FILTERS }}'+ - ' '+ + ' '+ ' '+ ' '+ '  '+ @@ -264,8 +264,9 @@ angular.module('faradayApp') $scope.gridOptions.columnDefs.push({ name : 'severity', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/severitycolumn.html', headerCellTemplate: header, + displayName : "sever", type: 'string', - width: '70', + width: '80', visible: $scope.columns["severity"], sort: getColumnSort('severity'), sortingAlgorithm: compareSeverities From 8c48334baf5661512d9eebae685afac004b7c7f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 17 Jul 2017 12:30:40 -0300 Subject: [PATCH 0019/1506] Add tests for user authentication --- server/app.py | 10 ++++---- server/database.py | 3 ++- test_cases/test_server.py | 52 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 test_cases/test_server.py diff --git a/server/app.py b/server/app.py index 6a8c4279dec..ce2d772373e 100644 --- a/server/app.py +++ b/server/app.py @@ -29,11 +29,11 @@ def unauthorized(): flask.abort(403) # Create a user to test with -# @app.before_first_request -# def create_user(): -server.database.init_common_db() -user_datastore.create_user(email='matt@nobien.net', password='password') -server.database.common_session.commit() +@app.before_first_request +def create_user(): + server.database.init_common_db() + # user_datastore.create_user(email='matt@nobien.net', password='password') + # server.database.common_session.commit() # Make API endpoints require a login user by default. Based on # https://stackoverflow.com/questions/13428708/best-way-to-make-flask-logins-login-required-the-default diff --git a/server/database.py b/server/database.py index 05e72c082a1..48961793f00 100644 --- a/server/database.py +++ b/server/database.py @@ -446,7 +446,8 @@ def __init__(self, workspace_name): # Connection to a database common to al workspaces # -common_engine = create_engine('sqlite:////tmp/test.db') +common_engine = create_engine(os.environ.get('COMMON_DB_PATH', + 'sqlite:////tmp/test.db')) common_session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=common_engine)) diff --git a/test_cases/test_server.py b/test_cases/test_server.py new file mode 100644 index 00000000000..f6034fd7f52 --- /dev/null +++ b/test_cases/test_server.py @@ -0,0 +1,52 @@ +import os +import sys +sys.path.append(os.path.abspath(os.getcwd())) +import unittest +import tempfile +from server.database import init_common_db + +def endpoint(): + return 'OK' + +class AuthTestCase(unittest.TestCase): + + ENDPOINT_ROUTE = '/' + + def setUp(self): + self.db_fd, self.db_name = tempfile.mkstemp() + os.environ['COMMON_DB_PATH'] = 'sqlite:///' + self.db_name + from server.app import app + app.testing = True + self.app = app.test_client() + app.route(self.ENDPOINT_ROUTE)(endpoint) + + def tearDown(self): + os.close(self.db_fd) + os.unlink(self.db_name) + + def test_403_when_getting_an_existent_view_and_not_logged(self): + res = self.app.get('/') + self.assertEqual(res.status_code, 403) + + def test_403_when_posting_an_existent_view_and_not_logged(self): + res = self.app.post('/', 'data') + self.assertEqual(res.status_code, 403) + + def test_403_when_accessing_a_non_existent_view_and_not_logged(self): + res = self.app.post('/dfsdfsdd', 'data') + self.assertEqual(res.status_code, 403) + + def test_200_when_logged_in(self): + with self.app.session_transaction() as sess: + sess['user_id'] = 1 + res = self.app.get('/') + self.assertEqual(res.status_code, 200) + + def test_200_when_not_logged_but_endpoint_is_public(self): + endpoint.is_public = True + res = self.app.get('/') + self.assertEqual(res.status_code, 200) + endpoint.is_public = False + +if __name__ == '__main__': + unittest.main() From 125df868021eb7bf89b075b2fca9a90b54a80caa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 17 Jul 2017 15:58:43 -0300 Subject: [PATCH 0020/1506] Refactor common database setup to improve unit testing and code quality --- server/app.py | 12 ++++++++---- server/database.py | 17 ++++++++--------- test_cases/test_server.py | 28 ++++++++++++++++++++++------ 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/server/app.py b/server/app.py index ce2d772373e..b52e9920b5c 100644 --- a/server/app.py +++ b/server/app.py @@ -2,6 +2,7 @@ # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information +import os import flask from flask_security import Security, login_required, \ SQLAlchemySessionUserDatastore @@ -17,7 +18,8 @@ app.config['SECURITY_PASSWORD_SINGLE_HASH'] = True # Setup Flask-Security -user_datastore = SQLAlchemySessionUserDatastore(server.database.common_session, +common_session = server.database.setup_common() +user_datastore = SQLAlchemySessionUserDatastore(common_session, server.models.User, server.models.Role) security = Security(app, user_datastore) @@ -31,9 +33,11 @@ def unauthorized(): # Create a user to test with @app.before_first_request def create_user(): - server.database.init_common_db() - # user_datastore.create_user(email='matt@nobien.net', password='password') - # server.database.common_session.commit() + if app.testing: + return + # server.database.init_common_db() + user_datastore.create_user(email='matt@nobien.net', password='password') + common_session.commit() # Make API endpoints require a login user by default. Based on # https://stackoverflow.com/questions/13428708/best-way-to-make-flask-logins-login-required-the-default diff --git a/server/database.py b/server/database.py index 48961793f00..b51fabfc4a5 100644 --- a/server/database.py +++ b/server/database.py @@ -443,17 +443,16 @@ def __init__(self, workspace_name): # -# Connection to a database common to al workspaces +# Connection to a database common to all workspaces # -common_engine = create_engine(os.environ.get('COMMON_DB_PATH', - 'sqlite:////tmp/test.db')) -common_session = scoped_session(sessionmaker(autocommit=False, - autoflush=False, - bind=common_engine)) -server.models.CommonBase.metadata.bind = common_engine - -def init_common_db(): +def setup_common(db_path='sqlite:////tmp/test.db'): + common_engine = create_engine(db_path) + common_session = scoped_session(sessionmaker(autocommit=False, + autoflush=False, + bind=common_engine)) + server.models.CommonBase.metadata.bind = common_engine from server.models import CommonBase CommonBase.metadata.create_all(bind=common_engine) CommonBase.query = common_session.query_property() + return common_session diff --git a/test_cases/test_server.py b/test_cases/test_server.py index f6034fd7f52..f7f898fa4cb 100644 --- a/test_cases/test_server.py +++ b/test_cases/test_server.py @@ -3,7 +3,10 @@ sys.path.append(os.path.abspath(os.getcwd())) import unittest import tempfile -from server.database import init_common_db +import server.app as server +from flask_security import Security, SQLAlchemySessionUserDatastore +from server.models import User, Role +from server.database import setup_common def endpoint(): return 'OK' @@ -14,11 +17,19 @@ class AuthTestCase(unittest.TestCase): def setUp(self): self.db_fd, self.db_name = tempfile.mkstemp() - os.environ['COMMON_DB_PATH'] = 'sqlite:///' + self.db_name - from server.app import app - app.testing = True - self.app = app.test_client() - app.route(self.ENDPOINT_ROUTE)(endpoint) + db_path = 'sqlite:///' + self.db_name + server.app.testing = True + + server.common_session = setup_common(db_path) + server.user_datastore = SQLAlchemySessionUserDatastore( + server.common_session, User, Role) + server.security.datastore = server.user_datastore + + self.user = server.user_datastore.create_user( + email='user@test.net', password='password') + server.common_session.commit() + self.app = server.app.test_client() + server.app.route(self.ENDPOINT_ROUTE)(endpoint) def tearDown(self): os.close(self.db_fd) @@ -48,5 +59,10 @@ def test_200_when_not_logged_but_endpoint_is_public(self): self.assertEqual(res.status_code, 200) endpoint.is_public = False + def test_403_when_logged_user_is_inactive(self): + self.assertTrue(self.user_datastore.deactivate_user(self.user.id)) + res = self.app.get('/') + self.assertEqual(res.status_code, 403) + if __name__ == '__main__': unittest.main() From ab3290117546561b6a5c9b11b25501c9fe24d689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 17 Jul 2017 16:10:22 -0300 Subject: [PATCH 0021/1506] Fix and add inactive and deleted user tests --- test_cases/test_server.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test_cases/test_server.py b/test_cases/test_server.py index f7f898fa4cb..d3011e919e5 100644 --- a/test_cases/test_server.py +++ b/test_cases/test_server.py @@ -60,9 +60,15 @@ def test_200_when_not_logged_but_endpoint_is_public(self): endpoint.is_public = False def test_403_when_logged_user_is_inactive(self): - self.assertTrue(self.user_datastore.deactivate_user(self.user.id)) + self.assertTrue(server.user_datastore.deactivate_user(self.user)) res = self.app.get('/') self.assertEqual(res.status_code, 403) + def test_403_when_logged_user_is_deleted(self): + server.user_datastore.delete_user(self.user) + res = self.app.get('/') + self.assertEqual(res.status_code, 403) + + if __name__ == '__main__': unittest.main() From 421e11359009e7b08d7e9eaf03f8438eb8446d6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 17 Jul 2017 18:58:39 -0300 Subject: [PATCH 0022/1506] Refactor auth tests structure To make easier to extend them on commercial versions --- server/app.py | 3 ++- test_cases/test_server.py | 24 +++++++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/server/app.py b/server/app.py index b52e9920b5c..b6ffc106b32 100644 --- a/server/app.py +++ b/server/app.py @@ -36,7 +36,8 @@ def create_user(): if app.testing: return # server.database.init_common_db() - user_datastore.create_user(email='matt@nobien.net', password='password') + user_datastore.create_user(email='matt@nobien.net', + password='password') common_session.commit() # Make API endpoints require a login user by default. Based on diff --git a/test_cases/test_server.py b/test_cases/test_server.py index d3011e919e5..2d721c2fe4c 100644 --- a/test_cases/test_server.py +++ b/test_cases/test_server.py @@ -11,8 +11,8 @@ def endpoint(): return 'OK' -class AuthTestCase(unittest.TestCase): +class BaseAPITestCase(unittest.TestCase): ENDPOINT_ROUTE = '/' def setUp(self): @@ -25,9 +25,6 @@ def setUp(self): server.common_session, User, Role) server.security.datastore = server.user_datastore - self.user = server.user_datastore.create_user( - email='user@test.net', password='password') - server.common_session.commit() self.app = server.app.test_client() server.app.route(self.ENDPOINT_ROUTE)(endpoint) @@ -35,6 +32,20 @@ def tearDown(self): os.close(self.db_fd) os.unlink(self.db_name) + def login_as(self, user): + with self.app.session_transaction() as sess: + sess['user_id'] = user.id + +class TestAuthentication(BaseAPITestCase): + """Tests related to allow/dissallow access depending of whether + the user is logged in or not""" + + def setUp(self): + super(TestAuthentication, self).setUp() + self.user = server.user_datastore.create_user( + email='user@test.net', password='password') + server.common_session.commit() + def test_403_when_getting_an_existent_view_and_not_logged(self): res = self.app.get('/') self.assertEqual(res.status_code, 403) @@ -48,8 +59,7 @@ def test_403_when_accessing_a_non_existent_view_and_not_logged(self): self.assertEqual(res.status_code, 403) def test_200_when_logged_in(self): - with self.app.session_transaction() as sess: - sess['user_id'] = 1 + self.login_as(self.user) res = self.app.get('/') self.assertEqual(res.status_code, 200) @@ -57,7 +67,7 @@ def test_200_when_not_logged_but_endpoint_is_public(self): endpoint.is_public = True res = self.app.get('/') self.assertEqual(res.status_code, 200) - endpoint.is_public = False + del endpoint.is_public def test_403_when_logged_user_is_inactive(self): self.assertTrue(server.user_datastore.deactivate_user(self.user)) From 24ea279771f0815ee4cef4a81d528c829de556fb Mon Sep 17 00:00:00 2001 From: Andres Date: Mon, 17 Jul 2017 19:01:04 -0300 Subject: [PATCH 0023/1506] #4129 Cambios sobre codereview de @micabot --- server/www/estilos.css | 198 +++++++++--------- .../statusReport/controllers/modalEdit.js | 24 +-- .../statusReport/controllers/statusReport.js | 28 +-- .../statusReport/partials/statusReport.html | 6 +- .../statusReport/providers/reference.js | 12 +- 5 files changed, 135 insertions(+), 133 deletions(-) diff --git a/server/www/estilos.css b/server/www/estilos.css index 286825cea0f..dd22e0b6627 100644 --- a/server/www/estilos.css +++ b/server/www/estilos.css @@ -118,7 +118,7 @@ aside { } aside nav a:hover {background-color: #d13434;} aside nav a.active {background-color: #e1372f;} - + .left-nav { position:fixed; top:50px; @@ -369,7 +369,7 @@ article.untercio{ } .dato2 .txt { font-size: 11px; - text-transform: uppercase; + text-transform: uppercase; } .dato2 span{ cursor: pointer; @@ -456,7 +456,7 @@ div.modal-button .btn.btn-success{float: right} div.modal-footer button.btn{bottom: 1%;right: 1%;} div.modal-footer button.btn.btn-success{right: 10%} div.modal-footer button.btn-footer{position: absolute;} -.modal-content table.csv-export.status-report{ +.modal-content table.csv-export.status-report{ width: 60%; } @@ -473,8 +473,8 @@ table.status-report { table.status-report thead th a { color: #3a3a3a; text-shadow: 0 1px 0 #fff; - } -table.status-report thead tr th, table.status-report tfoot tr th { + } +table.status-report thead tr th, table.status-report tfoot tr th { background-image: linear-gradient(to bottom,#E8EFF0 0,#DDDDDD 100%); background-image: -webkit-linear-gradient(top,#E8EFF0 0,#DDDDDD 100%); filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#E8EFF0', endColorstr='#DDDDDD', GradientType=0); @@ -623,7 +623,7 @@ select.ng-valid{ #bar svg{margin-left: -10px;} #contenido{text-align: left;} -#sequence { +#sequence { height: 0px; } #sequence text, #legend text { @@ -718,7 +718,7 @@ text.host-name{font-size: 13px;font-weight:bold;} border-radius: 10px; color: #FFF; } - .box{ + .box{ display: flex; flex-direction: row; flex-wrap: wrap; @@ -767,7 +767,7 @@ text.host-name{font-size: 13px;font-weight:bold;} } a.ws-link span.impact-ok{top: 2px;left: -4px;color: #5cb85c;} a.ws-link span.impact-del{top: 3px;left: 2px;color: #d9534f;} -a.ws-link:hover { +a.ws-link:hover { color: #ffffff; text-decoration: none; } @@ -833,95 +833,95 @@ aside .alert.alert-danger.alert-dismissible{ z-index: 1000; } -.label-unclassified { - color: #FFFFFF; - background-color: #444147; - border-color: #444147; -} - - .label-unclassified:hover, - .label-unclassified:focus, - .label-unclassified:active, - .label-unclassified.active, - .open .dropdown-toggle.label-unclassified { - color: #FFFFFF; - background-color: #111012; - border-color: #444147; - } - - .label-unclassified:active, - .label-unclassified.active, - .open .dropdown-toggle.label-unclassified { - background-image: none; - } - - .label-unclassified.disabled, - .label-unclassified[disabled], - fieldset[disabled] .label-unclassified, - .label-unclassified.disabled:hover, - .label-unclassified[disabled]:hover, - fieldset[disabled] .label-unclassified:hover, - .label-unclassified.disabled:focus, - .label-unclassified[disabled]:focus, - fieldset[disabled] .label-unclassified:focus, - .label-unclassified.disabled:active, - .label-unclassified[disabled]:active, - fieldset[disabled] .label-unclassified:active, - .label-unclassified.disabled.active, - .label-unclassified[disabled].active, - fieldset[disabled] .label-unclassified.active { - background-color: #444147; - border-color: #444147; - } - - .label-unclassified .badge { - color: #444147; - background-color: #FFFFFF; +.label-unclassified { + color: #FFFFFF; + background-color: #444147; + border-color: #444147; +} + + .label-unclassified:hover, + .label-unclassified:focus, + .label-unclassified:active, + .label-unclassified.active, + .open .dropdown-toggle.label-unclassified { + color: #FFFFFF; + background-color: #111012; + border-color: #444147; + } + + .label-unclassified:active, + .label-unclassified.active, + .open .dropdown-toggle.label-unclassified { + background-image: none; + } + + .label-unclassified.disabled, + .label-unclassified[disabled], + fieldset[disabled] .label-unclassified, + .label-unclassified.disabled:hover, + .label-unclassified[disabled]:hover, + fieldset[disabled] .label-unclassified:hover, + .label-unclassified.disabled:focus, + .label-unclassified[disabled]:focus, + fieldset[disabled] .label-unclassified:focus, + .label-unclassified.disabled:active, + .label-unclassified[disabled]:active, + fieldset[disabled] .label-unclassified:active, + .label-unclassified.disabled.active, + .label-unclassified[disabled].active, + fieldset[disabled] .label-unclassified.active { + background-color: #444147; + border-color: #444147; + } + + .label-unclassified .badge { + color: #444147; + background-color: #FFFFFF; } -.label-high { - color: #ffffff; - background-color: #DF3936; - border-color: #DF3936; -} - - .label-high:hover, - .label-high:focus, - .label-high:active, - .label-high.active, - .open .dropdown-toggle.label-high { - color: #ffffff; - background-color: #992826; - border-color: #DF3936; - } - - .label-high:active, - .label-high.active, - .open .dropdown-toggle.label-high { - background-image: none; - } - - .label-high.disabled, - .label-high[disabled], - fieldset[disabled] .label-high, - .label-high.disabled:hover, - .label-high[disabled]:hover, - fieldset[disabled] .label-high:hover, - .label-high.disabled:focus, - .label-high[disabled]:focus, - fieldset[disabled] .label-high:focus, - .label-high.disabled:active, - .label-high[disabled]:active, - fieldset[disabled] .label-high:active, - .label-high.disabled.active, - .label-high[disabled].active, - fieldset[disabled] .label-high.active { - background-color: #DF3936; - border-color: #DF3936; - } - - .label-high .badge { - color: #DF3936; - background-color: #ffffff; +.label-high { + color: #ffffff; + background-color: #DF3936; + border-color: #DF3936; +} + + .label-high:hover, + .label-high:focus, + .label-high:active, + .label-high.active, + .open .dropdown-toggle.label-high { + color: #ffffff; + background-color: #992826; + border-color: #DF3936; + } + + .label-high:active, + .label-high.active, + .open .dropdown-toggle.label-high { + background-image: none; + } + + .label-high.disabled, + .label-high[disabled], + fieldset[disabled] .label-high, + .label-high.disabled:hover, + .label-high[disabled]:hover, + fieldset[disabled] .label-high:hover, + .label-high.disabled:focus, + .label-high[disabled]:focus, + fieldset[disabled] .label-high:focus, + .label-high.disabled:active, + .label-high[disabled]:active, + fieldset[disabled] .label-high:active, + .label-high.disabled.active, + .label-high[disabled].active, + fieldset[disabled] .label-high.active { + background-color: #DF3936; + border-color: #DF3936; + } + + .label-high .badge { + color: #DF3936; + background-color: #ffffff; } .btn-small-margin { margin-bottom: 15px !important; @@ -1037,7 +1037,7 @@ div.btn-group .vulns-filter{left: -2px;} div.btn-group .dropdown-menu.margin{margin-left: 38px} /* UI GRID */ .ui-grid-cell.ui-grid-disable-selection.ui-grid-row-header-cell{pointer-events: auto!important;} -.grid { +.grid { height: 100%; } .ui-grid-cell-contents.left-rows.text-center.ng-scope{white-space: normal!important; } @@ -1062,12 +1062,12 @@ div.alert.alert-danger.alert-dismissible .ws-list a:hover{text-decoration: none} a.button-disable{cursor: not-allowed;pointer-events: none;opacity: 0.5} .almost-expired{background-color: #dca7a7 !important} -.statusReport-Search:before,.statusReport-Search:after { +.status-report-search:before,.status-report-search:after { display: table; content: " "; } -.statusReport-Search:after { +.status-report-search:after { clear: both; } diff --git a/server/www/scripts/statusReport/controllers/modalEdit.js b/server/www/scripts/statusReport/controllers/modalEdit.js index e3bba688b82..f01cf8fc27c 100644 --- a/server/www/scripts/statusReport/controllers/modalEdit.js +++ b/server/www/scripts/statusReport/controllers/modalEdit.js @@ -3,9 +3,9 @@ // See the file 'doc/LICENSE' for the license information angular.module('faradayApp') - .controller('modalEditCtrl', ['$modalInstance', 'EASEOFRESOLUTION', 'STATUSES', 'commonsFact', 'severities', 'vuln', 'cweFact', 'referenceService', - function($modalInstance, EASEOFRESOLUTION, STATUSES, commons, severities, vuln, cweFact, referenceService) { - + .controller('modalEditCtrl', ['$modalInstance', 'EASEOFRESOLUTION', 'STATUSES', 'commonsFact', 'severities', 'vuln', 'cweFact', 'referenceFact', + function($modalInstance, EASEOFRESOLUTION, STATUSES, commons, severities, vuln, cweFact, referenceFact) { + var vm = this; vm.saveAsModelDisabled = false; @@ -39,7 +39,7 @@ angular.module('faradayApp') vm.cwe_filter = ""; vm.file_name_error = false; - + vm.data = { _attachments: {}, confirmed: false, @@ -56,11 +56,11 @@ angular.module('faradayApp') refs: {}, resolution: "", severity: undefined, - method: "", - path: "", - pname: "", + method: "", + path: "", + pname: "", params: "", - query: "", + query: "", request: "", response: "", website: "", @@ -72,10 +72,10 @@ angular.module('faradayApp') vm.populate(vm.vuln); - // TODO: EVIDENCE SHOUD BE LOADED ALREADY? + // TODO: EVIDENCE SHOUD BE LOADED ALREADY? if(vm.vuln._attachments !== undefined) { vm.data._attachments = vm.vuln._attachments; - vm.icons = commons.loadIcons(vm.data._attachments); + vm.icons = commons.loadIcons(vm.data._attachments); } }; @@ -84,7 +84,7 @@ angular.module('faradayApp') vm.vulnModelsManager.create(vm.data); vm.saveAsModelDisabled = true; } - + vm.selectedFiles = function(files, e) { files.forEach(function(file) { if(file.name.charAt(0) != "_") { @@ -142,7 +142,7 @@ angular.module('faradayApp') vm.openReference = function(text) { - window.open(referenceService.processReference(text), '_blank'); + window.open(referenceFact.processReference(text), '_blank'); } vm.newPolicyViolation = function() { diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 965fc18b685..b213086793a 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -4,7 +4,7 @@ angular.module('faradayApp') .filter('removeLinesbreak',function(){ - return function(text){ + return function(text){ return text?text.replace(/\n/g, ' '):''; }}) .controller('statusReportCtrl', @@ -12,11 +12,11 @@ angular.module('faradayApp') '$location', '$uibModal', '$cookies', '$q', '$window', 'BASEURL', 'SEVERITIES', 'EASEOFRESOLUTION', 'STATUSES', 'hostsManager', 'commonsFact', 'vulnsManager', 'workspacesFact', 'csvService', 'uiGridConstants', 'vulnModelsManager', - 'referenceService', + 'referenceFact', function($scope, $filter, $routeParams, $location, $uibModal, $cookies, $q, $window, BASEURL, SEVERITIES, EASEOFRESOLUTION, STATUSES, hostsManager, commonsFact, - vulnsManager, workspacesFact, csvService, uiGridConstants, vulnModelsManager, referenceService) { + vulnsManager, workspacesFact, csvService, uiGridConstants, vulnModelsManager, referenceFact) { $scope.baseurl; $scope.columns; $scope.easeofresolution; @@ -30,7 +30,7 @@ angular.module('faradayApp') $scope.workspaces; $scope.currentPage; $scope.gridOptions; - $scope.vulnModelsManager; + $scope.vulnModelsManager; $scope.vulnWebSelected; $scope.confirmed = false; @@ -56,7 +56,7 @@ angular.module('faradayApp') $scope.reverse = true; $scope.vulns = []; $scope.selected = false; - $scope.vulnModelsManager = vulnModelsManager; + $scope.vulnModelsManager = vulnModelsManager; $scope.gridOptions = { multiSelect: true, @@ -215,7 +215,7 @@ angular.module('faradayApp') angular.element($window).bind('resize', function () { resizeGrid(); - }); + }); }; var defineColumns = function() { @@ -450,17 +450,17 @@ angular.module('faradayApp') var resizeGrid = function() { $scope.gridHeight = getGridHeight('grid', 'right-main', 15); - } + }; var getGridHeight = function(gridClass, contentClass, bottomOffset) { var contentOffset = angular.element(document.getElementsByClassName(contentClass)[0]).offset(); var contentHeight = angular.element(document.getElementsByClassName(contentClass)[0]).height(); - var gridOffset = angular.element(document.getElementsByClassName(gridClass)).offset(); + var gridOffset = angular.element(document.getElementsByClassName(gridClass)).offset(); if (gridOffset !== undefined) { var gridHeight = contentHeight - (gridOffset.top) - bottomOffset; return gridHeight + 'px'; - } - } + } + }; $scope.saveAsModel = function() { var self = this; @@ -502,7 +502,7 @@ angular.module('faradayApp') }; $scope.processReference = function(text) { - return referenceService.processReference(text); + return referenceFact.processReference(text); }; $scope.groupBy = function(property) { @@ -919,13 +919,13 @@ angular.module('faradayApp') // Add the total amount of vulnerabilities as an option for pagination // if it is larger than our biggest page size if ($scope.gridOptions.totalItems > paginationOptions.defaultPageSizes[paginationOptions.defaultPageSizes.length - 1]) { - + $scope.gridOptions.paginationPageSizes = paginationOptions.defaultPageSizes.concat([$scope.gridOptions.totalItems]); - + // sadly, this will load the vuln list again because it fires a paginationChanged event if ($scope.gridOptions.paginationPageSize > $scope.gridOptions.totalItems) $scope.gridOptions.paginationPageSize = $scope.gridOptions.totalItems; - + // New vuln and MAX items per page setted => reload page size. if ($scope.gridOptions.paginationPageSize === $scope.gridOptions.totalItems - 1) $scope.gridOptions.paginationPageSize = $scope.gridOptions.totalItems; diff --git a/server/www/scripts/statusReport/partials/statusReport.html b/server/www/scripts/statusReport/partials/statusReport.html index 8318cedd8d4..07e83b160b5 100644 --- a/server/www/scripts/statusReport/partials/statusReport.html +++ b/server/www/scripts/statusReport/partials/statusReport.html @@ -103,7 +103,7 @@

    Status report for {{ workspace {{column}} -

    +
    -
    +
    @@ -125,7 +125,7 @@

    Status report for {{ workspace

    -
    +
    diff --git a/server/www/scripts/statusReport/providers/reference.js b/server/www/scripts/statusReport/providers/reference.js index 75312cce946..a9d168c1412 100644 --- a/server/www/scripts/statusReport/providers/reference.js +++ b/server/www/scripts/statusReport/providers/reference.js @@ -3,9 +3,10 @@ // See the file 'doc/LICENSE' for the license information angular.module('faradayApp') - .factory('referenceService', function () { - return { - processReference: function (text) { + .factory('referenceFact', [ + function () { + referenceFact = {}; + referenceFact.processReference = function (text) { var url = 'http://google.com/', url_pattern = new RegExp('^(http|https):\\/\\/?'); @@ -35,5 +36,6 @@ angular.module('faradayApp') return url; } - } - }); \ No newline at end of file + + return referenceFact; + }]); \ No newline at end of file From 883adec288ac0487504bd3e0a71609f6e99caaad Mon Sep 17 00:00:00 2001 From: Andres Date: Mon, 17 Jul 2017 19:59:33 -0300 Subject: [PATCH 0024/1506] #4129 Cambios sobre codereview de @micabot --- server/www/index.html | 1 + .../www/scripts/commons/filters/removeLinebreaks.js | 11 +++++++++++ .../scripts/statusReport/controllers/statusReport.js | 8 ++------ .../partials/ui-grid/columns/desccolumn.html | 2 +- .../partials/ui-grid/columns/resolutioncolumn.html | 2 +- .../www/scripts/statusReport/providers/reference.js | 3 ++- 6 files changed, 18 insertions(+), 9 deletions(-) create mode 100644 server/www/scripts/commons/filters/removeLinebreaks.js diff --git a/server/www/index.html b/server/www/index.html index e65e87925bd..bc7c4e516dc 100644 --- a/server/www/index.html +++ b/server/www/index.html @@ -101,6 +101,7 @@ + diff --git a/server/www/scripts/commons/filters/removeLinebreaks.js b/server/www/scripts/commons/filters/removeLinebreaks.js new file mode 100644 index 00000000000..0ce67b6b663 --- /dev/null +++ b/server/www/scripts/commons/filters/removeLinebreaks.js @@ -0,0 +1,11 @@ +// Faraday Penetration Test IDE +// Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +// See the file 'doc/LICENSE' for the license information + +// removes line breaks \n from text + +angular.module('faradayApp') + .filter('removeLinebreaks',function(){ + return function(text){ + return text?text.replace(/\n/g, ' '):''; + }}); \ No newline at end of file diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index b213086793a..4ec60b1f05e 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -3,10 +3,6 @@ // See the file 'doc/LICENSE' for the license information angular.module('faradayApp') - .filter('removeLinesbreak',function(){ - return function(text){ - return text?text.replace(/\n/g, ' '):''; - }}) .controller('statusReportCtrl', ['$scope', '$filter', '$routeParams', '$location', '$uibModal', '$cookies', '$q', '$window', 'BASEURL', @@ -1007,7 +1003,7 @@ angular.module('faradayApp') //TODO: this is horrible var srvName = srvStr.split(') ')[1]; return $scope.encodeUrl(srvName); - } + }; $scope.concatForTooltip = function (items, isArray, useDoubleLinebreak) { var elements = []; @@ -1023,7 +1019,7 @@ angular.module('faradayApp') } return elements.join("\n" + (useDoubleLinebreak ? "\n" : "")); - } + }; init(); }]); diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/desccolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/desccolumn.html index 6d8c812a2ad..9bc251418ed 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/desccolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/desccolumn.html @@ -1 +1 @@ -
    {{COL_FIELD CUSTOM_FILTERS | removeLinesbreak}}
    \ No newline at end of file +
    {{COL_FIELD CUSTOM_FILTERS | removeLinebreaks}}
    \ No newline at end of file diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/resolutioncolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/resolutioncolumn.html index cb39e5f8b76..e281edb1886 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/resolutioncolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/resolutioncolumn.html @@ -1 +1 @@ -
    {{COL_FIELD.split("(")[0] !== " " ? COL_FIELD : "EMPTY" + COL_FIELD | removeLinesbreak}}
    \ No newline at end of file +
    {{COL_FIELD.split("(")[0] !== " " ? COL_FIELD : "EMPTY" + COL_FIELD | removeLinebreaks}}
    \ No newline at end of file diff --git a/server/www/scripts/statusReport/providers/reference.js b/server/www/scripts/statusReport/providers/reference.js index a9d168c1412..a09f44bece2 100644 --- a/server/www/scripts/statusReport/providers/reference.js +++ b/server/www/scripts/statusReport/providers/reference.js @@ -5,7 +5,8 @@ angular.module('faradayApp') .factory('referenceFact', [ function () { - referenceFact = {}; + var referenceFact = {}; + referenceFact.processReference = function (text) { var url = 'http://google.com/', url_pattern = new RegExp('^(http|https):\\/\\/?'); From 5cf9e57b4855da2709c8648dc0f3fd7f054c4205 Mon Sep 17 00:00:00 2001 From: Andres Date: Mon, 17 Jul 2017 23:56:33 -0300 Subject: [PATCH 0025/1506] #4129 Grid row height change --- server/www/estilos.css | 7 +++++ .../statusReport/controllers/statusReport.js | 26 +++++++++---------- .../ui-grid/columns/severitycolumn.html | 2 +- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/server/www/estilos.css b/server/www/estilos.css index dd22e0b6627..bea6a48a281 100644 --- a/server/www/estilos.css +++ b/server/www/estilos.css @@ -1040,10 +1040,14 @@ div.btn-group .dropdown-menu.margin{margin-left: 38px} .grid { height: 100%; } +.ui-grid-cell-contents p{ + margin-bottom: 0; +} .ui-grid-cell-contents.left-rows.text-center.ng-scope{white-space: normal!important; } .ui-grid .ui-grid-row:nth-child(odd) .ui-grid-cell .ui-grid-cell-contents .crop-text, .ui-grid .ui-grid-row:nth-child(even) .ui-grid-cell .ui-grid-cell-contents .crop-text, .ui-grid-cell-contents{ white-space: pre; word-wrap: break-word!important; + font-size: 13px !important; } .grid.ui-grid{font-family: 12px!important;} .ui-grid-cell-contents.row-tooltip.text-center{white-space: normal!important; } @@ -1059,6 +1063,9 @@ div.alert.alert-danger.alert-dismissible .ws-list a:hover{text-decoration: none} .grid .ui-grid-header { padding: 3px 5px; } +.ui-grid-text-center { + text-align: center; +} a.button-disable{cursor: not-allowed;pointer-events: none;opacity: 0.5} .almost-expired{background-color: #dca7a7 !important} diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 4ec60b1f05e..02700a074a0 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -67,7 +67,7 @@ angular.module('faradayApp') enableHorizontalScrollbar: 0, treeRowHeaderAlwaysVisible: false, enableGroupHeaderSelection: true, - rowHeight: 30 + rowHeight: 47 }; $scope.gridOptions.columnDefs = []; @@ -216,7 +216,7 @@ angular.module('faradayApp') var defineColumns = function() { $scope.gridOptions.columnDefs.push({ name: 'selectAll', width: '20', headerCellTemplate: "", pinnedLeft:true }); - $scope.gridOptions.columnDefs.push({ name: 'confirmVuln', width: '22', headerCellTemplate: "
    ", cellTemplate: 'scripts/statusReport/partials/ui-grid/confirmbutton.html' }); + $scope.gridOptions.columnDefs.push({ name: 'confirmVuln', width: '23', headerCellTemplate: "
    ", cellTemplate: 'scripts/statusReport/partials/ui-grid/confirmbutton.html' }); function getColumnSort(columnName){ if($cookies.get('SRsortColumn') === columnName){ @@ -242,11 +242,21 @@ angular.module('faradayApp') '
    '+ '
    '; + $scope.gridOptions.columnDefs.push({ name : 'severity', + cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/severitycolumn.html', + headerCellTemplate: header, + displayName : "sev", + type: 'string', + width: '70', + visible: $scope.columns["severity"], + sort: getColumnSort('severity'), + sortingAlgorithm: compareSeverities + }); $scope.gridOptions.columnDefs.push({ name : 'date', displayName : "date", cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/datecolumn.html', headerCellTemplate: header, - width: '75', + width: '80', sort: getColumnSort('date'), visible: $scope.columns["date"] }); @@ -257,16 +267,6 @@ angular.module('faradayApp') sort: getColumnSort('name'), visible: $scope.columns["name"] }); - $scope.gridOptions.columnDefs.push({ name : 'severity', - cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/severitycolumn.html', - headerCellTemplate: header, - displayName : "sever", - type: 'string', - width: '80', - visible: $scope.columns["severity"], - sort: getColumnSort('severity'), - sortingAlgorithm: compareSeverities - }); $scope.gridOptions.columnDefs.push({ name : 'service', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/servicecolumn.html', headerCellTemplate: header, diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html index 214fbe0a547..193d29028a5 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html @@ -1 +1 @@ -
    {{COL_FIELD | uppercase}}
    \ No newline at end of file +
    {{COL_FIELD | uppercase}}
    \ No newline at end of file From 5387b9cb8b9d86dbc0d95db60aa69c2e3b6cdd69 Mon Sep 17 00:00:00 2001 From: Andres Date: Tue, 18 Jul 2017 00:02:53 -0300 Subject: [PATCH 0026/1506] #4150 open evidence on edit --- .../statusReport/controllers/modalEdit.js | 38 +++++++++++++------ .../statusReport/partials/modalEdit.html | 2 +- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/server/www/scripts/statusReport/controllers/modalEdit.js b/server/www/scripts/statusReport/controllers/modalEdit.js index f01cf8fc27c..8400ede1da2 100644 --- a/server/www/scripts/statusReport/controllers/modalEdit.js +++ b/server/www/scripts/statusReport/controllers/modalEdit.js @@ -3,11 +3,17 @@ // See the file 'doc/LICENSE' for the license information angular.module('faradayApp') - .controller('modalEditCtrl', ['$modalInstance', 'EASEOFRESOLUTION', 'STATUSES', 'commonsFact', 'severities', 'vuln', 'cweFact', 'referenceFact', - function($modalInstance, EASEOFRESOLUTION, STATUSES, commons, severities, vuln, cweFact, referenceFact) { + .controller('modalEditCtrl', + ['$modalInstance', '$routeParams','EASEOFRESOLUTION', 'STATUSES', 'commonsFact', + 'BASEURL', 'severities', 'vuln', 'cweFact', 'referenceFact', + 'encodeURIComponentFilter', + function($modalInstance, $routeParams,EASEOFRESOLUTION, STATUSES, commons, + BASEURL, severities, vuln, cweFact, referenceFact, + encodeURIComponent) { var vm = this; + vm.baseurl; vm.saveAsModelDisabled = false; vm.easeofresolution; vm.new_ref; @@ -30,6 +36,7 @@ angular.module('faradayApp') vm.new_ref = ""; vm.new_policyviolation = ""; vm.icons = {}; + vm.baseurl = BASEURL; vm.cweList = []; cweFact.get().then(function(data) { @@ -41,6 +48,7 @@ angular.module('faradayApp') vm.file_name_error = false; vm.data = { + _id: undefined, _attachments: {}, confirmed: false, data: "", @@ -83,10 +91,11 @@ angular.module('faradayApp') vm.modelMessage = "Done." vm.vulnModelsManager.create(vm.data); vm.saveAsModelDisabled = true; - } + }; vm.selectedFiles = function(files, e) { files.forEach(function(file) { + file.newfile = true; if(file.name.charAt(0) != "_") { if(!vm.data._attachments.hasOwnProperty(file)) vm.data._attachments[file.name] = file; } else { @@ -94,12 +103,12 @@ angular.module('faradayApp') } }); vm.icons = commons.loadIcons(vm._attachments); - } + }; vm.removeEvidence = function(name) { delete vm.data._attachments[name]; delete vm.icons[name]; - } + }; vm.toggleImpact = function(key) { vm.data.impact[key] = !vm.data.impact[key]; @@ -138,12 +147,17 @@ angular.module('faradayApp') vm.new_ref = ""; } } - } + }; - vm.openReference = function(text) - { - window.open(referenceFact.processReference(text), '_blank'); - } + vm.openReference = function(text) { + window.open(referenceFact.processReference(text), '_blank'); + }; + + vm.openEvidence = function(name) { + var currentEvidence = vm.data._attachments[name]; + if (!currentEvidence.newfile) + window.open(vm.baseurl + $routeParams.wsId + '/' + vm.data._id + '/' + encodeURIComponent(name), '_blank'); + }; vm.newPolicyViolation = function() { if (vm.new_policyviolation != "") { @@ -153,7 +167,7 @@ angular.module('faradayApp') vm.new_policyviolation = ""; } } - } + }; vm.populate = function(item) { for (var key in vm.data) { @@ -174,7 +188,7 @@ angular.module('faradayApp') policyviolations.push({value: policyviolation}); }); vm.data.policyviolations = policyviolations; - } + }; init(); }]); diff --git a/server/www/scripts/statusReport/partials/modalEdit.html b/server/www/scripts/statusReport/partials/modalEdit.html index 4306e1f56e0..e4d7c0f7a98 100644 --- a/server/www/scripts/statusReport/partials/modalEdit.html +++ b/server/www/scripts/statusReport/partials/modalEdit.html @@ -166,7 +166,7 @@

    Evidence

    • -
    • - + Hosts
    • @@ -51,7 +51,7 @@
    • - + Vulnerability Templates
    • diff --git a/server/www/scripts/statusReport/partials/statusReport.html b/server/www/scripts/statusReport/partials/statusReport.html index 07e83b160b5..ed075d96837 100644 --- a/server/www/scripts/statusReport/partials/statusReport.html +++ b/server/www/scripts/statusReport/partials/statusReport.html @@ -31,6 +31,35 @@

      Status report for {{ workspace
      +
      + + + +
      +
      + + + + +

    -
    - - - -
    -
    - - - - -
    - -
    +
    +
    +
    + + +
    +
    +
    -
    -
    @@ -140,6 +145,9 @@ + + + diff --git a/server/www/scripts/app.js b/server/www/scripts/app.js index dd6e84d5e92..3435bd800b2 100644 --- a/server/www/scripts/app.js +++ b/server/www/scripts/app.js @@ -222,6 +222,11 @@ faradayApp.config(['$routeProvider', 'ngClipProvider', '$uibTooltipProvider', controller: 'commercialCtrl', title: 'Executive Report | ' }). + when('/login', { + templateUrl: 'scripts/auth/partials/login.html', + controller: 'loginCtrl', + title: 'Login | ' + }). when('/users', { templateUrl: 'scripts/commons/partials/commercial.html', controller: 'commercialCtrl', @@ -271,10 +276,18 @@ faradayApp.config(['$routeProvider', 'ngClipProvider', '$uibTooltipProvider', }); }]); -faradayApp.run(['$location', '$rootScope', function($location, $rootScope) { +faradayApp.run(['$location', '$rootScope', 'loginSrv', function($location, $rootScope, loginSrv) { $rootScope.$on('$routeChangeSuccess', function(event, current, previous) { if(current.hasOwnProperty('$$route')) { $rootScope.title = current.$$route.title; } }); + $rootScope.$on('$routeChangeStart', function(event){ + // Require in all routes (except the login one) + // Taken from http://stackoverflow.com/questions/26145871/redirect-on-all-routes-to-login-if-not-authenticated + // I don't know why this doesn't cause an infinite loop + loginSrv.isAuthenticated().then(function(auth){ + if(!auth) $location.path('/login'); + }); + }); }]); diff --git a/server/www/scripts/auth/controllers/login.js b/server/www/scripts/auth/controllers/login.js new file mode 100644 index 00000000000..3c70bdd409d --- /dev/null +++ b/server/www/scripts/auth/controllers/login.js @@ -0,0 +1,92 @@ +// Faraday Penetration Test IDE +// Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +// See the file 'doc/LICENSE' for the license information + +angular.module('faradayApp') + .controller('loginCtrl', ['$scope', '$location', '$cookies', 'loginSrv', function($scope, $location, $cookies, loginSrv) { + + $scope.data = { + "user": null, + "pass": null + }; + + $scope.errorLoginFlag = false; + + $scope.checkResetError = function(){ + if($scope.errorLoginFlag == true) + $scope.errorMessage = ""; + }; + + $scope.login = function(){ + if ($scope.data.user && $scope.data.pass){ + loginSrv.login($scope.data.user, $scope.data.pass).then(function(user){ + var currentUrl = ""; + if($cookies.currentUrl != undefined) { + currentUrl = $cookies.currentUrl; + } + $location.path(currentUrl); + }, function(){ + $scope.errorMessage = "Invalid user or password"; + $scope.errorLoginFlag = true; + }); + } else { + $scope.errorMessage = "Every field is required"; + $scope.errorLoginFlag = true; + } + }; + + $scope.$on('$routeChangeSuccess', function(){ + loginSrv.isAuthenticated().then(function(auth){ + if(auth) $location.path('/'); + }); + }); + + }]); + +angular.module('faradayApp') + .controller('loginBarCtrl', ['$scope', '$location', '$cookies','loginSrv', function($scope, $location, $cookies,loginSrv) { + $scope.user = null; + $scope.auth = loginSrv.isAuth(); + + $scope.$watch(loginSrv.isAuth, function(newValue) { + loginSrv.getUser().then(function(user){ + $scope.user = user; + $scope.auth = newValue; + }); + }); + + $scope.getUser = function(){ + return $scope.user; + }; + + $scope.loginPage = function(){ + $location.path('/login'); + }; + + $scope.logout = function(){ + loginSrv.logout().then(function(){ + $location.path('/login'); + $cookies.currentUrl = ""; + }); + }; + }]); + +angular.module('faradayApp') + .controller('loginBackgroundCtrl', ['$scope', '$location', '$cookies', function($scope, $location, $cookies) { + $scope.component = ""; + $scope.isLogin = true; + + $scope.updateComponent = function() { + if($location.path() == "") { + $scope.component = "home"; + } else { + $scope.component = $location.path().split("/")[1]; + } + $cookies.currentComponent = $scope.component; + $scope.isLogin = $scope.component == "login"; + }; + + $scope.$on('$locationChangeSuccess', function() { + $scope.updateComponent(); + }); + }]); diff --git a/server/www/scripts/auth/partials/login.html b/server/www/scripts/auth/partials/login.html new file mode 100644 index 00000000000..afbfb76a732 --- /dev/null +++ b/server/www/scripts/auth/partials/login.html @@ -0,0 +1,35 @@ + + + +
    +
    +
    + +
    + +
    + +
    +
    +

    Faraday Professional - faradaysec.com

    +
    + +
    + +
    + diff --git a/server/www/scripts/auth/services/interceptor.js b/server/www/scripts/auth/services/interceptor.js new file mode 100644 index 00000000000..fc643a679af --- /dev/null +++ b/server/www/scripts/auth/services/interceptor.js @@ -0,0 +1,28 @@ +// Faraday Penetration Test IDE +// Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +// See the file 'doc/LICENSE' for the license information + +angular.module('faradayApp'). + factory('AuthInterceptor', ['$q', '$location', '$cookies', 'loginSrv', function($q, $location, $cookies, loginSrv){ + return { + response: function(response){ + return response; + }, + + responseError: function(response) { + if(response.status === 401 || response.status === 403){ + var deferred = $q.defer(); + loginSrv.isAuthenticated().then(function(auth){ + if(!auth) { + $location.path('/login'); + $cookies.currentComponent; + } + return deferred.reject(response); + }); + return deferred.promise; + }else{ + return $q.reject(response); + } + } + } + }]); diff --git a/server/www/scripts/auth/services/login.js b/server/www/scripts/auth/services/login.js new file mode 100644 index 00000000000..7727aba2bc6 --- /dev/null +++ b/server/www/scripts/auth/services/login.js @@ -0,0 +1,102 @@ +// Faraday Penetration Test IDE +// Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +// See the file 'doc/LICENSE' for the license information + +angular.module('faradayApp') + .service('loginSrv', ['BASEURL', '$q', function(BASEURL, $q) { + + loginSrv = { + is_authenticated: false, + user_obj: null, + last_time_checked: new Date(0), + + login: function(user, pass){ + var deferred = $q.defer(); + $.ajax({ + type: 'POST', + url: BASEURL + '_api/login', + data: JSON.stringify({"email": user, "password": pass}), + dataType: 'json', + contentType: 'application/json' + }) + .done(function(){ + $.getJSON(BASEURL + '_api/session', function(data) { + loginSrv.user_obj = data; + loginSrv.last_time_checked = new Date(); + loginSrv.is_authenticated = true; + deferred.resolve(loginSrv.user_obj); + }); + }) + .fail(function(){ + loginSrv.is_authenticated = false; + loginSrv.user_obj = null; + deferred.reject(); + }); + return deferred.promise; + }, + + _check: function(deferred) { + $.getJSON(BASEURL + '_api/session', function(data) { + loginSrv.user_obj = data; + loginSrv.is_authenticated = true; + loginSrv.last_time_checked = new Date(); + deferred.resolve(loginSrv.is_authenticated); + }).fail(function(){ + loginSrv.user_obj = null; + loginSrv.is_authenticated = false; + loginSrv.last_time_checked = new Date(); + deferred.resolve(loginSrv.is_authenticated); + }); + }, + + isAuthenticated: function(){ + var deferred = $q.defer(); + var seconds = (new Date() - loginSrv.last_time_checked) / 1000; + if (seconds > 1){ + //more than one second since checked for last time + loginSrv._check(deferred); + } else { + deferred.resolve(loginSrv.is_authenticated); + } + return deferred.promise; + }, + + isAuth: function(){ + return loginSrv.is_authenticated; + }, + + getUser: function(){ + var deferred = $q.defer(); + $.getJSON(BASEURL + '_api/session', function(data) { + loginSrv.user_obj = data; + deferred.resolve(loginSrv.user_obj); + }) + .fail(function(){ + deferred.reject(); + }); + return deferred.promise; + }, + + logout: function(){ + var deferred = $q.defer(); + var callback = function(){ + loginSrv.is_authenticated = false; + loginSrv.user_obj = null; + deferred.resolve(); + } + $.ajax({ + url: BASEURL + '_api/logout', + type: 'GET', + success: callback + }) + .fail(function(){ + deferred.reject(); + }); + return deferred.promise; + } + } + + loginSrv.isAuthenticated(); + + return loginSrv; + }]); diff --git a/server/www/scripts/navigation/controllers/navigationCtrl.js b/server/www/scripts/navigation/controllers/navigationCtrl.js index c981c7f7082..bd20b0f62a8 100644 --- a/server/www/scripts/navigation/controllers/navigationCtrl.js +++ b/server/www/scripts/navigation/controllers/navigationCtrl.js @@ -29,7 +29,7 @@ angular.module('faradayApp') catch(error){ console.log("Can't connect to faradaysec.com"); } - + }, function() { console.log("Can't connect to faradaysec.com"); }); @@ -90,7 +90,7 @@ angular.module('faradayApp') }; $scope.showNavigation = function() { - var noNav = ["home", "index", ""]; + var noNav = ["", "home", "login", "index"]; return noNav.indexOf($scope.component) < 0; }; @@ -113,4 +113,4 @@ angular.module('faradayApp') // if(navigator.userAgent.toLowerCase().indexOf('iceweasel') > -1) { // $scope.isIceweasel = "Your browser is not supported, please use Firefox or Chrome"; // } - }]); \ No newline at end of file + }]); From cf3fe9a5932fe9e8acbe3665457286e86bfab243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 8 Aug 2017 18:36:31 -0300 Subject: [PATCH 0060/1506] Login with username instead of with email Also change redirect URLs of login and logout views (to make it easier to test) --- server/app.py | 3 +++ server/models.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/server/app.py b/server/app.py index 2c6ce11b314..c97d153f81d 100644 --- a/server/app.py +++ b/server/app.py @@ -22,6 +22,9 @@ def create_app(db_connection_string=None, testing=None): app.config['SECRET_KEY'] = 'supersecret' app.config['SECURITY_PASSWORD_SINGLE_HASH'] = True app.config['WTF_CSRF_ENABLED'] = False + app.config['SECURITY_USER_IDENTITY_ATTRIBUTES'] = ['username'] + app.config['SECURITY_POST_LOGIN_VIEW'] = '/_api/session' + app.config['SECURITY_POST_LOGOUT_VIEW'] = '/_api/login' from server.models import db db.init_app(app) diff --git a/server/models.py b/server/models.py index dfa875c41b4..1d16c096718 100644 --- a/server/models.py +++ b/server/models.py @@ -297,7 +297,7 @@ def role(self): def get_security_payload(self): return { - "username": self.email, + "username": self.username, "role": self.role, "roles": [role.name for role in self.roles], # Deprectated "name": self.email From 0040baa5015c841609bc6770329990339cca2597 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 9 Aug 2017 18:29:28 -0300 Subject: [PATCH 0061/1506] Add unit test for vuln dao --- test_cases/dao/__init__.py | 0 test_cases/dao/vuln.py | 42 ++++++++++++++++++++++++++++++++++++++ test_cases/factories.py | 19 +++++++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 test_cases/dao/__init__.py create mode 100644 test_cases/dao/vuln.py create mode 100644 test_cases/factories.py diff --git a/test_cases/dao/__init__.py b/test_cases/dao/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test_cases/dao/vuln.py b/test_cases/dao/vuln.py new file mode 100644 index 00000000000..3bec60f253f --- /dev/null +++ b/test_cases/dao/vuln.py @@ -0,0 +1,42 @@ +import string +import random +import unittest +from server.models import Vulnerability +from server.database import Workspace + +from server.database import initialize, get_manager +from server.dao.vuln import VulnerabilityDAO + +initialize() + + +class VulnerabilityDAOTestCases(unittest.TestCase): + + def setUp(self): + self.manager = get_manager() + self.workspace_name = ''.join(random.choice(string.ascii_lowercase) for _ in range(15)) + + self.manager.create_workspace({'name': self.workspace_name}) + + def tearDown(self): + self.manager.delete_workspace({'name': self.workspace_name}) + + def _new_vuln(self, vuln_type): + new_vuln = Vulnerability() + new_vuln.name = 'Test vuln' + new_vuln.description = 'Test description' + new_vuln.vuln_type = vuln_type + self.manager.get_workspace(self.workspace_name).session.add(new_vuln) + self.manager.get_workspace(self.workspace_name).session.commit() + return new_vuln + + def test_count(self): + vuln_dao = VulnerabilityDAO(self.workspace_name) + res = vuln_dao.count() + expected = {'total_count': 0, 'web_vuln_count': 0, 'vuln_count': 0} + self.assertEquals(expected, res) + self._new_vuln('Vulnerability') + self._new_vuln('VulnerabilityWeb') + res = vuln_dao.count() + expected = {'total_count': 2, 'web_vuln_count': 1, 'vuln_count': 1} + self.assertEquals(expected, res) diff --git a/test_cases/factories.py b/test_cases/factories.py new file mode 100644 index 00000000000..f38b81ab963 --- /dev/null +++ b/test_cases/factories.py @@ -0,0 +1,19 @@ +import factory +from pytest_factoryboy import register +from server.models import Vulnerability, Workspace + + +class VulnerabilityFactory(factory.Factory): + + class Meta: + model = Vulnerability + + +class WorkspaceFactory(factory.Factory): + + class Meta: + model = Workspace + + +register(VulnerabilityFactory) +register(WorkspaceFactory) From ea38b9de824949ed56f35573ec6c7a0e1c602962 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 9 Aug 2017 18:30:59 -0300 Subject: [PATCH 0062/1506] Update unit test code --- test_cases/dao/vuln.py | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/test_cases/dao/vuln.py b/test_cases/dao/vuln.py index 3bec60f253f..13cdd735b61 100644 --- a/test_cases/dao/vuln.py +++ b/test_cases/dao/vuln.py @@ -1,42 +1,37 @@ +import os +import sys +sys.path.append(os.path.abspath(os.getcwd())) import string import random import unittest -from server.models import Vulnerability -from server.database import Workspace +from server.web import app +from server.models import Vulnerability, Workspace -from server.database import initialize, get_manager from server.dao.vuln import VulnerabilityDAO - -initialize() +from test_cases.factories import WorkspaceFactory, VulnerabilityFactory class VulnerabilityDAOTestCases(unittest.TestCase): def setUp(self): - self.manager = get_manager() self.workspace_name = ''.join(random.choice(string.ascii_lowercase) for _ in range(15)) - - self.manager.create_workspace({'name': self.workspace_name}) - - def tearDown(self): - self.manager.delete_workspace({'name': self.workspace_name}) + self.workspace = WorkspaceFactory.build() def _new_vuln(self, vuln_type): - new_vuln = Vulnerability() + new_vuln = VulnerabilityFactory.build() new_vuln.name = 'Test vuln' new_vuln.description = 'Test description' new_vuln.vuln_type = vuln_type - self.manager.get_workspace(self.workspace_name).session.add(new_vuln) - self.manager.get_workspace(self.workspace_name).session.commit() return new_vuln def test_count(self): - vuln_dao = VulnerabilityDAO(self.workspace_name) - res = vuln_dao.count() - expected = {'total_count': 0, 'web_vuln_count': 0, 'vuln_count': 0} - self.assertEquals(expected, res) - self._new_vuln('Vulnerability') - self._new_vuln('VulnerabilityWeb') - res = vuln_dao.count() - expected = {'total_count': 2, 'web_vuln_count': 1, 'vuln_count': 1} - self.assertEquals(expected, res) + with app.app_context(): + vuln_dao = VulnerabilityDAO(self.workspace_name) + res = vuln_dao.count() + expected = {'total_count': 0, 'web_vuln_count': 0, 'vuln_count': 0} + self.assertEquals(expected, res) + self._new_vuln('Vulnerability') + self._new_vuln('VulnerabilityWeb') + res = vuln_dao.count() + expected = {'total_count': 2, 'web_vuln_count': 1, 'vuln_count': 1} + self.assertEquals(expected, res) From 9c9db71b33fd53751a99eb7a7239fd3e41991d93 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 9 Aug 2017 18:37:50 -0300 Subject: [PATCH 0063/1506] Set the version of flask security --- requirements_server.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_server.txt b/requirements_server.txt index e82eec74467..5ad45c1a9f1 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -8,6 +8,6 @@ sqlalchemy>=1.0.12 pyopenssl>=16.0.0 service_identity>=16.0.0 pyasn1-modules -flask-security +Flask-Security==3.0.0 bcrypt Flask-SQLAlchemy==2.2 From 8ece5f9fd0c38f5fdc3e3920584097310ba06172 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 10 Aug 2017 15:13:59 -0300 Subject: [PATCH 0064/1506] Update unit tests to use workspace db model --- server/dao/base.py | 7 ++++-- server/dao/vuln.py | 19 ++++++++++++---- test_cases/dao/vuln.py | 50 +++++++++++++++++++++++++++++------------ test_cases/factories.py | 16 ++++++++++--- 4 files changed, 69 insertions(+), 23 deletions(-) diff --git a/server/dao/base.py b/server/dao/base.py index 0efe10d31f0..9cd931953b2 100644 --- a/server/dao/base.py +++ b/server/dao/base.py @@ -5,7 +5,7 @@ import server.database import server.utils.logger -from server.models import db, EntityMetadata +from server.models import db, Workspace, EntityMetadata class FaradayDAO(object): @@ -16,10 +16,13 @@ def __init__(self, workspace): self._logger = server.utils.logger.get_logger(self) self._session = db.session self._couchdb = None + self.workspace = workspace + if not getattr(workspace, 'name', None): + self.workspace = db.session.query(Workspace).filter_by(name=workspace) def get_all(self): self.__check_valid_operation() - return self._session.query(self.MAPPED_ENTITY).all() + return self._session.query(self.MAPPED_ENTITY).filter_by(workspace=self.workspace) def __check_valid_operation(self): if self.MAPPED_ENTITY is None: diff --git a/server/dao/vuln.py b/server/dao/vuln.py index a2376ba2c29..a55bd012607 100644 --- a/server/dao/vuln.py +++ b/server/dao/vuln.py @@ -5,12 +5,23 @@ import json from server.dao.base import FaradayDAO -from server.utils.database import paginate, sort_results, apply_search_filter, get_count +from server.utils.database import ( + paginate, + sort_results, + apply_search_filter, + get_count +) from sqlalchemy import case from sqlalchemy.sql import func from sqlalchemy.orm.query import Bundle -from server.models import Host, Interface, Service, Vulnerability, EntityMetadata +from server.models import ( + Host, + Interface, + Service, + Vulnerability, + EntityMetadata +) class VulnerabilityDAO(FaradayDAO): @@ -97,8 +108,8 @@ def __query_database(self, search=None, page=0, page_size=0, order_by=None, orde .outerjoin(EntityMetadata, EntityMetadata.id == Vulnerability.entity_metadata_id)\ .outerjoin(Service, Service.id == Vulnerability.service_id)\ .outerjoin(Host, Host.id == Vulnerability.host_id)\ - .join(Interface, Interface.host_id == Host.id) - + .join(Interface, Interface.host_id == Host.id)\ + .filter_by(workspace=self.workspace) # Apply pagination, sorting and filtering options to the query query = self.__specialized_sort(query, order_by, order_dir) query = apply_search_filter(query, self.COLUMNS_MAP, search, vuln_filter, self.STRICT_FILTERING) diff --git a/test_cases/dao/vuln.py b/test_cases/dao/vuln.py index 13cdd735b61..718ed2ad4cf 100644 --- a/test_cases/dao/vuln.py +++ b/test_cases/dao/vuln.py @@ -5,7 +5,7 @@ import random import unittest from server.web import app -from server.models import Vulnerability, Workspace +from server.models import db, Vulnerability, Workspace from server.dao.vuln import VulnerabilityDAO from test_cases.factories import WorkspaceFactory, VulnerabilityFactory @@ -14,24 +14,46 @@ class VulnerabilityDAOTestCases(unittest.TestCase): def setUp(self): - self.workspace_name = ''.join(random.choice(string.ascii_lowercase) for _ in range(15)) - self.workspace = WorkspaceFactory.build() + with app.app_context(): + db.create_all() + self.workspace = WorkspaceFactory.build() + db.session.commit() - def _new_vuln(self, vuln_type): - new_vuln = VulnerabilityFactory.build() - new_vuln.name = 'Test vuln' - new_vuln.description = 'Test description' - new_vuln.vuln_type = vuln_type - return new_vuln + def tearDown(self): + with app.app_context(): + db.drop_all() - def test_count(self): + def test_vulnerability_count_per_workspace_is_filtered(self): + """ + Verifies that the dao return the correct count from each workspace + """ with app.app_context(): - vuln_dao = VulnerabilityDAO(self.workspace_name) + another_workspace = WorkspaceFactory.build() + vuln_dao = VulnerabilityDAO(self.workspace) + another_vuln_dao = VulnerabilityDAO(another_workspace) + vuln_1 = VulnerabilityFactory.build(vuln_type='Vulnerability', workspace=self.workspace) + vuln_2 = VulnerabilityFactory.build(vuln_type='Vulnerability', workspace=another_workspace) + db.session.add(vuln_1) + db.session.add(vuln_2) + db.session.commit() + ws_count = vuln_dao.count() + another_ws_count = another_vuln_dao.count() + ws_expected = {'total_count': 1, 'web_vuln_count': 0, 'vuln_count': 1} + another_expected = {'total_count': 1, 'web_vuln_count': 0, 'vuln_count': 1} + assert ws_count == ws_expected + assert another_ws_count == another_expected + + def test_count_by_type(self): + with app.app_context(): + vuln_dao = VulnerabilityDAO(self.workspace) res = vuln_dao.count() expected = {'total_count': 0, 'web_vuln_count': 0, 'vuln_count': 0} self.assertEquals(expected, res) - self._new_vuln('Vulnerability') - self._new_vuln('VulnerabilityWeb') + vuln = VulnerabilityFactory.build(vuln_type='Vulnerability', workspace=self.workspace) + vuln_web = VulnerabilityFactory.build(vuln_type='VulnerabilityWeb', workspace=self.workspace) + db.session.add(vuln) + db.session.add(vuln_web) + db.session.commit() res = vuln_dao.count() expected = {'total_count': 2, 'web_vuln_count': 1, 'vuln_count': 1} - self.assertEquals(expected, res) + assert expected == res diff --git a/test_cases/factories.py b/test_cases/factories.py index f38b81ab963..a8f05010d7c 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -1,18 +1,28 @@ import factory +from factory.fuzzy import FuzzyText from pytest_factoryboy import register -from server.models import Vulnerability, Workspace +from server.models import db, Vulnerability, Workspace -class VulnerabilityFactory(factory.Factory): +class VulnerabilityFactory(factory.alchemy.SQLAlchemyModelFactory): + + id = factory.Sequence(lambda n: n) + name = FuzzyText() + description = FuzzyText() class Meta: model = Vulnerability + sqlalchemy_session = db.session + +class WorkspaceFactory(factory.alchemy.SQLAlchemyModelFactory): -class WorkspaceFactory(factory.Factory): + id = factory.Sequence(lambda n: n) + name = FuzzyText() class Meta: model = Workspace + sqlalchemy_session = db.session register(VulnerabilityFactory) From 21300604683f2ce6d4ad58d11a466c0b5de79170 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 10 Aug 2017 15:14:20 -0300 Subject: [PATCH 0065/1506] Fix minor PEP8 --- server/dao/base.py | 1 - server/dao/vuln.py | 18 ++++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/server/dao/base.py b/server/dao/base.py index 9cd931953b2..1cbf0bf7fe8 100644 --- a/server/dao/base.py +++ b/server/dao/base.py @@ -38,4 +38,3 @@ def get_by_couchdb_id(self, couchdb_id): def save(self, obj): self._session.add(obj) self._session.commit() - diff --git a/server/dao/vuln.py b/server/dao/vuln.py index a55bd012607..2ff57162471 100644 --- a/server/dao/vuln.py +++ b/server/dao/vuln.py @@ -29,7 +29,7 @@ class VulnerabilityDAO(FaradayDAO): COLUMNS_MAP = { "couchid": [EntityMetadata.couchdb_id], "id": [Vulnerability.id], - "date": [EntityMetadata.create_time], # TODO: fix search for this field + "date": [EntityMetadata.create_time], # TODO: fix search for this field "confirmed": [Vulnerability.confirmed], "name": [Vulnerability.name], "severity": [Vulnerability.severity], @@ -83,7 +83,7 @@ def __query_database(self, search=None, page=0, page_size=0, order_by=None, orde # Instead of using SQLAlchemy ORM facilities to fetch rows, we bundle involved columns for # organizational and MAINLY performance reasons. Doing it this way, we improve retrieving # times from large workspaces almost 2x. - vuln_bundle = Bundle('vuln', Vulnerability.id.label('server_id'),Vulnerability.name.label('v_name'),\ + vuln_bundle = Bundle('vuln', Vulnerability.id.label('server_id'), Vulnerability.name.label('v_name'),\ Vulnerability.confirmed, Vulnerability.data,\ Vulnerability.description, Vulnerability.easeofresolution, Vulnerability.impact_accountability,\ Vulnerability.impact_availability, Vulnerability.impact_confidentiality, Vulnerability.impact_integrity,\ @@ -130,12 +130,12 @@ def __specialized_sort(self, query, order_by, order_dir): # instead of a lexicographycally one column_map = { 'severity': [case( - { 'unclassified': 0, + {'unclassified': 0, 'info': 1, 'low': 2, 'med': 3, 'high': 4, - 'critical': 5 }, + 'critical': 5}, value=Vulnerability.severity )] } @@ -147,6 +147,7 @@ def __specialized_sort(self, query, order_by, order_dir): def __get_vuln_data(self, vuln, service, host, hostnames): def get_own_id(couchdb_id): return couchdb_id.split('.')[-1] + def get_parent_id(couchdb_id): return '.'.join(couchdb_id.split('.')[:-1]) @@ -207,14 +208,15 @@ def get_parent_id(couchdb_id): def count(self, group_by=None, search=None, vuln_filter={}): query = self._session.query(Vulnerability.vuln_type, func.count())\ + .filter_by(workspace=self.workspace)\ .group_by(Vulnerability.vuln_type) query = apply_search_filter(query, self.COLUMNS_MAP, search, vuln_filter) total_count = dict(query.all()) # Return total amount of services if no group-by field was provided - result_count = { 'total_count': sum(total_count.values()), - 'web_vuln_count': total_count.get('VulnerabilityWeb', 0), - 'vuln_count': total_count.get('Vulnerability', 0), } + result_count = {'total_count': sum(total_count.values()), + 'web_vuln_count': total_count.get('VulnerabilityWeb', 0), + 'vuln_count': total_count.get('Vulnerability', 0)} if group_by is None: return result_count @@ -234,7 +236,7 @@ def count(self, group_by=None, search=None, vuln_filter={}): query = apply_search_filter(query, self.COLUMNS_MAP, search, vuln_filter, self.STRICT_FILTERING) result = query.all() - result_count['groups'] = [ { group_by: value[1], 'count': count } for value, count in result ] + result_count['groups'] = [{group_by: value[1], 'count': count} for value, count in result] return result_count From b8b33a6b57639cee8591ab0ddb2992287ffe5bf3 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 10 Aug 2017 16:57:14 -0300 Subject: [PATCH 0066/1506] Fix vuln dao list. update factories --- server/dao/vuln.py | 4 +-- test_cases/dao/vuln.py | 8 ++++- test_cases/factories.py | 72 ++++++++++++++++++++++++++++++++++++----- 3 files changed, 73 insertions(+), 11 deletions(-) diff --git a/server/dao/vuln.py b/server/dao/vuln.py index 2ff57162471..a53f1829ff3 100644 --- a/server/dao/vuln.py +++ b/server/dao/vuln.py @@ -109,7 +109,8 @@ def __query_database(self, search=None, page=0, page_size=0, order_by=None, orde .outerjoin(Service, Service.id == Vulnerability.service_id)\ .outerjoin(Host, Host.id == Vulnerability.host_id)\ .join(Interface, Interface.host_id == Host.id)\ - .filter_by(workspace=self.workspace) + .filter(Vulnerability.workspace == self.workspace) + # Apply pagination, sorting and filtering options to the query query = self.__specialized_sort(query, order_by, order_dir) query = apply_search_filter(query, self.COLUMNS_MAP, search, vuln_filter, self.STRICT_FILTERING) @@ -239,4 +240,3 @@ def count(self, group_by=None, search=None, vuln_filter={}): result_count['groups'] = [{group_by: value[1], 'count': count} for value, count in result] return result_count - diff --git a/test_cases/dao/vuln.py b/test_cases/dao/vuln.py index 718ed2ad4cf..ea068b60c45 100644 --- a/test_cases/dao/vuln.py +++ b/test_cases/dao/vuln.py @@ -23,7 +23,7 @@ def tearDown(self): with app.app_context(): db.drop_all() - def test_vulnerability_count_per_workspace_is_filtered(self): + def test_vulnerability_count_and_list_per_workspace_is_filtered(self): """ Verifies that the dao return the correct count from each workspace """ @@ -42,6 +42,12 @@ def test_vulnerability_count_per_workspace_is_filtered(self): another_expected = {'total_count': 1, 'web_vuln_count': 0, 'vuln_count': 1} assert ws_count == ws_expected assert another_ws_count == another_expected + ws_list = vuln_dao.list() + assert vuln_1 in ws_list['vulnerabilities'] + another_ws_list = another_vuln_dao.list() + assert vuln_2 in another_ws_list['vulnerabilities'] + + def test_count_by_type(self): with app.app_context(): diff --git a/test_cases/factories.py b/test_cases/factories.py index a8f05010d7c..5ef18aca1a5 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -1,29 +1,85 @@ import factory -from factory.fuzzy import FuzzyText +from factory.fuzzy import ( + FuzzyText, + FuzzyChoice +) from pytest_factoryboy import register -from server.models import db, Vulnerability, Workspace +from server.models import ( + db, + Host, + Service, + Interface, + Workspace, + Vulnerability, + EntityMetadata, +) -class VulnerabilityFactory(factory.alchemy.SQLAlchemyModelFactory): +class FaradayFactory(factory.alchemy.SQLAlchemyModelFactory): id = factory.Sequence(lambda n: n) + + +class WorkspaceFactory(FaradayFactory): + + name = FuzzyText() + + class Meta: + model = Workspace + sqlalchemy_session = db.session + + +class HostFactory(FaradayFactory): name = FuzzyText() description = FuzzyText() + os = FuzzyChoice(['Linux', 'Windows', 'OSX', 'Android', 'iOS']) class Meta: - model = Vulnerability + model = Host sqlalchemy_session = db.session -class WorkspaceFactory(factory.alchemy.SQLAlchemyModelFactory): +class EntityMetadataFactory(FaradayFactory): - id = factory.Sequence(lambda n: n) + class Meta: + model = EntityMetadata + sqlalchemy_session = db.session + + +class InterfaceFactory(FaradayFactory): + + class Meta: + model = Interface + sqlalchemy_session = db.session + + +class ServiceFactory(FaradayFactory): name = FuzzyText() + description = FuzzyText() + ports = FuzzyChoice(['443', '80', '22']) class Meta: - model = Workspace + model = Service + sqlalchemy_session = db.session + + +class VulnerabilityFactory(FaradayFactory): + + name = FuzzyText() + description = FuzzyText() + host = factory.SubFactory(HostFactory) + entity_metadata = factory.SubFactory(EntityMetadataFactory) + service = factory.SubFactory(ServiceFactory) + workspace = factory.SubFactory(WorkspaceFactory) + vuln_type = FuzzyChoice(['Vulnerability', 'VulnerabilityWeb']) + + class Meta: + model = Vulnerability sqlalchemy_session = db.session -register(VulnerabilityFactory) register(WorkspaceFactory) +register(HostFactory) +register(ServiceFactory) +register(InterfaceFactory) +register(VulnerabilityFactory) From d59054c409cbd221112a0d80f98e5f19e8ed2865 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 11 Aug 2017 19:03:10 -0300 Subject: [PATCH 0067/1506] Fix vuln dao Updated the unit test and factories. Obs: Interface will be removed in future. The real issue here was that factories didn't produce the correct relationship in the model and the join with host <-> interface was not possible --- server/dao/vuln.py | 1 - test_cases/dao/vuln.py | 5 +++-- test_cases/factories.py | 11 +++++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/server/dao/vuln.py b/server/dao/vuln.py index a53f1829ff3..5f512fbae51 100644 --- a/server/dao/vuln.py +++ b/server/dao/vuln.py @@ -108,7 +108,6 @@ def __query_database(self, search=None, page=0, page_size=0, order_by=None, orde .outerjoin(EntityMetadata, EntityMetadata.id == Vulnerability.entity_metadata_id)\ .outerjoin(Service, Service.id == Vulnerability.service_id)\ .outerjoin(Host, Host.id == Vulnerability.host_id)\ - .join(Interface, Interface.host_id == Host.id)\ .filter(Vulnerability.workspace == self.workspace) # Apply pagination, sorting and filtering options to the query diff --git a/test_cases/dao/vuln.py b/test_cases/dao/vuln.py index ea068b60c45..c944ad527e8 100644 --- a/test_cases/dao/vuln.py +++ b/test_cases/dao/vuln.py @@ -43,9 +43,10 @@ def test_vulnerability_count_and_list_per_workspace_is_filtered(self): assert ws_count == ws_expected assert another_ws_count == another_expected ws_list = vuln_dao.list() - assert vuln_1 in ws_list['vulnerabilities'] + + assert vuln_1.id == ws_list['vulnerabilities'][0]['_id'] another_ws_list = another_vuln_dao.list() - assert vuln_2 in another_ws_list['vulnerabilities'] + assert vuln_2.id == another_ws_list['vulnerabilities'][0]['_id'] diff --git a/test_cases/factories.py b/test_cases/factories.py index 5ef18aca1a5..e469b075c76 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -40,6 +40,7 @@ class Meta: class EntityMetadataFactory(FaradayFactory): + couchdb_id = factory.Sequence(lambda n: '{0}.1.2'.format(n)) class Meta: model = EntityMetadata @@ -47,6 +48,10 @@ class Meta: class InterfaceFactory(FaradayFactory): + name = FuzzyText() + description = FuzzyText() + mac = FuzzyText() + host = factory.SubFactory(HostFactory) class Meta: model = Interface @@ -57,6 +62,8 @@ class ServiceFactory(FaradayFactory): name = FuzzyText() description = FuzzyText() ports = FuzzyChoice(['443', '80', '22']) + interface = factory.SubFactory(InterfaceFactory) + host = factory.SubFactory(HostFactory) class Meta: model = Service @@ -72,6 +79,10 @@ class VulnerabilityFactory(FaradayFactory): service = factory.SubFactory(ServiceFactory) workspace = factory.SubFactory(WorkspaceFactory) vuln_type = FuzzyChoice(['Vulnerability', 'VulnerabilityWeb']) + attachments = '[]' + policyviolations = '[]' + refs = '[]' + class Meta: model = Vulnerability From 36b8cdc533155709459d4a0d6ce4ca56946c81a0 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 14 Aug 2017 15:16:22 -0300 Subject: [PATCH 0068/1506] Fix Workspace import from CouchDB The workspaces were imported multiple times generating duplicated entries in the DB. --- server/importer.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/server/importer.py b/server/importer.py index 5b22ed11499..3080856b8cb 100644 --- a/server/importer.py +++ b/server/importer.py @@ -336,8 +336,7 @@ class WorkspaceImporter(object): @classmethod def update_from_document(cls, document): - workspace = Workspace() - workspace.name = document.get('name', None) + workspace, created = get_or_create(session, server.models.Workspace, name=document.get('name', None)) return workspace def add_relationships_from_dict(self, entity, entities): @@ -345,7 +344,6 @@ def add_relationships_from_dict(self, entity, entities): child_entity.workspace = entity - class FaradayEntityImporter(object): # Document Types: [u'Service', u'Communication', u'Vulnerability', u'CommandRunInformation', u'Reports', u'Host', u'Workspace', u'Interface'] @classmethod @@ -430,18 +428,11 @@ def _open_couchdb_conn(): def import_workspace_into_database(workspace_name, couchdb_server_conn): + workspace, created = get_or_create(session, server.models.Workspace, name=workspace_name) - try: - # import checks if the object exists. - # the import is idempotent - _import_from_couchdb(workspace, couchdb_server_conn) - session.commit() - except Exception as ex: - import traceback - traceback.print_exc() - logger.exception(ex) - session.rollback() - raise ex + + _import_from_couchdb(workspace, couchdb_server_conn) + session.commit() return created From 3af72830158b7246a35e9910066435f234eff440 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 14 Aug 2017 16:07:45 -0300 Subject: [PATCH 0069/1506] Add __repr__ for User model To improve debugging and testing --- server/models.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/models.py b/server/models.py index 8e01a1eab76..eaa95170c2b 100644 --- a/server/models.py +++ b/server/models.py @@ -302,3 +302,7 @@ def get_security_payload(self): "roles": [role.name for role in self.roles], # Deprectated "name": self.email } + + def __repr__(self): + return '<%sUser: %s>' % ('LDAP ' if self.is_ldap else '', + self.username) From 07af63040d30ba64f63dd0ea9e55d499e85b2b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 14 Aug 2017 17:53:59 -0300 Subject: [PATCH 0070/1506] Refactor server tests to use pytest fixtures --- test_cases/conftest.py | 42 +++++++++++++++++++++++++++++++++++++++ test_cases/test_server.py | 39 +++++++++++++++++------------------- 2 files changed, 60 insertions(+), 21 deletions(-) create mode 100644 test_cases/conftest.py diff --git a/test_cases/conftest.py b/test_cases/conftest.py new file mode 100644 index 00000000000..7e01cc81cc9 --- /dev/null +++ b/test_cases/conftest.py @@ -0,0 +1,42 @@ +import os +import sys +import tempfile +import pytest + +sys.path.append(os.path.abspath(os.getcwd())) +from server.app import create_app +from server.models import db + + +@pytest.fixture +def app(monkeypatch): + db_fd, db_name = tempfile.mkstemp() + db_path = 'sqlite:///' + db_name + app = create_app(db_connection_string=db_path, testing=True) + + # monkeypatch.setattr('flask.current_app', app) + # monkeypatch.setattr('flask_security.forms.current_app', app) + + with app.app_context(): + db.create_all() + yield app#.test_client() + os.close(db_fd) + os.unlink(db_name) + + +def create_user(app, username, email, password, **kwargs): + user = app.user_datastore.create_user(username=username, + email=email, + password=password, + **kwargs) + db.session.add(user) + db.session.commit() + return user + +@pytest.fixture +def user(app): + return create_user(app, 'test', 'user@test.com', 'password', is_ldap=False) + +@pytest.fixture +def ldap_user(app): + return create_user(app, 'ldap', 'ldap@test.com', 'password', is_ldap=True) diff --git a/test_cases/test_server.py b/test_cases/test_server.py index 8e305e1427e..a68276be96c 100644 --- a/test_cases/test_server.py +++ b/test_cases/test_server.py @@ -1,8 +1,10 @@ import os import sys -sys.path.append(os.path.abspath(os.getcwd())) import unittest import tempfile +import pytest + +sys.path.append(os.path.abspath(os.getcwd())) from server.app import create_app from flask_security import Security, SQLAlchemyUserDatastore from server.models import db, User, Role @@ -14,19 +16,13 @@ def endpoint(): class BaseAPITestCase(unittest.TestCase): - ENDPOINT_ROUTE = '/' - - def setUp(self): - self.db_fd, self.db_name = tempfile.mkstemp() - db_path = 'sqlite:///' + self.db_name - self.flask_app = create_app(db_connection_string=db_path, testing=True) - - self.app = self.flask_app.test_client() - self.flask_app.route(self.ENDPOINT_ROUTE)(endpoint) - def tearDown(self): - os.close(self.db_fd) - os.unlink(self.db_name) + @pytest.fixture(autouse=True) + def load_app(self, app): + """Use this to avoid having to use an app argument to every + function""" + self.flask_app = app + self.app = app.test_client() def login_as(self, user): with self.app.session_transaction() as sess: @@ -41,14 +37,15 @@ class TestAuthentication(BaseAPITestCase): """Tests related to allow/dissallow access depending of whether the user is logged in or not""" - def setUp(self): - super(TestAuthentication, self).setUp() - with self.flask_app.app_context(): - db.create_all() - self.user = self.flask_app.user_datastore.create_user( - email='user@test.net', password='password') - db.session.add(self.user) - db.session.commit() + ENDPOINT_ROUTE = '/' + + @pytest.fixture(autouse=True) + def load_user(self, user): + self.user = user + + @pytest.fixture(autouse=True) + def route_endpoint(self, app): + app.route(self.ENDPOINT_ROUTE)(endpoint) def test_403_when_getting_an_existent_view_and_not_logged(self): res = self.app.get('/') From 623df8ce00eb737079581abadb812d4799403635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 14 Aug 2017 17:54:37 -0300 Subject: [PATCH 0071/1506] Add is_ldap field to User model It will be used only on corporate versions but add it here to maintain the same schema between community and commercial versions --- server/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/models.py b/server/models.py index eaa95170c2b..f764b5c2497 100644 --- a/server/models.py +++ b/server/models.py @@ -275,6 +275,7 @@ class User(db.Model, UserMixin): email = Column(String(255), unique=True) username = Column(String(255)) password = Column(String(255)) + is_ldap = Column(Boolean(), nullable=False) last_login_at = Column(DateTime()) current_login_at = Column(DateTime()) last_login_ip = Column(String(100)) From de709cec5d0854ccbcde5c3320b8036744176701 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 14 Aug 2017 17:57:24 -0300 Subject: [PATCH 0072/1506] Update service dao to use workspace --- server/dao/service.py | 2 +- test_cases/dao/services.py | 44 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 test_cases/dao/services.py diff --git a/server/dao/service.py b/server/dao/service.py index a720b306682..5e38bdde74a 100644 --- a/server/dao/service.py +++ b/server/dao/service.py @@ -90,7 +90,7 @@ def __get_service_data(self, service): 'vulns': service.vuln_count} def count(self, group_by=None): - total_count = self._session.query(func.count(Service.id)).scalar() + total_count = self._session.query(func.count(Service.id)).filter_by(workspace=self.workspace).scalar() # Return total amount of services if no group-by field was provided if group_by is None: diff --git a/test_cases/dao/services.py b/test_cases/dao/services.py new file mode 100644 index 00000000000..e1ef6dd9420 --- /dev/null +++ b/test_cases/dao/services.py @@ -0,0 +1,44 @@ +import os +import sys +sys.path.append(os.path.abspath(os.getcwd())) +import string +import random +import unittest +from server.web import app +from server.models import db, Service + +from server.dao.service import ServiceDAO +from test_cases.factories import WorkspaceFactory, ServiceFactory + + +class VulnerabilityDAOTestCases(unittest.TestCase): + + def setUp(self): + with app.app_context(): + db.create_all() + self.workspace = WorkspaceFactory.build() + db.session.commit() + + def tearDown(self): + with app.app_context(): + db.drop_all() + + def test_(self): + with app.app_context(): + workspace = WorkspaceFactory.build() + service_dao = ServiceDAO(workspace) + expected = {'total_count': 0} + res = service_dao.count() + assert expected == res + new_service = ServiceFactory.build(workspace=workspace) + db.session.add(new_service) + db.session.commit() + res = service_dao.count() + expected = {'total_count': 1} + assert expected == res + another_workspace = WorkspaceFactory.build() + another_service = ServiceFactory.build(workspace=another_workspace) + db.session.add(another_service) + db.session.commit() + res = service_dao.count() + assert expected == res From 9f2e696e5a652541899df8fb9edd659c5ea52e24 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 14 Aug 2017 17:57:39 -0300 Subject: [PATCH 0073/1506] Fix PEP8 warnings --- server/dao/service.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/server/dao/service.py b/server/dao/service.py index 5e38bdde74a..38bed5e796b 100644 --- a/server/dao/service.py +++ b/server/dao/service.py @@ -7,9 +7,17 @@ from sqlalchemy.orm.query import Bundle from server.dao.base import FaradayDAO -from server.models import Host, Interface, Service, EntityMetadata, Vulnerability, Credential +from server.models import ( + Host, + Interface, + Service, + EntityMetadata, + Vulnerability, + Credential +) from server.utils.database import apply_search_filter + class ServiceDAO(FaradayDAO): MAPPED_ENTITY = Service COLUMNS_MAP = { @@ -94,7 +102,7 @@ def count(self, group_by=None): # Return total amount of services if no group-by field was provided if group_by is None: - return { 'total_count': total_count } + return {'total_count': total_count} # Otherwise return the amount of services grouped by the field specified if group_by not in ServiceDAO.COLUMNS_MAP: @@ -107,7 +115,5 @@ def count(self, group_by=None): res = query.all() - return { 'total_count': total_count, - 'groups': [ { group_by: value, 'count': count } for value, count in res ] } - - + return {'total_count': total_count, + 'groups': [{group_by: value, 'count': count} for value, count in res]} From 27669b03214f052136b9d2584d4f9f8f36f9a877 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 14 Aug 2017 18:29:01 -0300 Subject: [PATCH 0074/1506] Update service dao list to use workspace filter --- server/dao/service.py | 1 + test_cases/dao/services.py | 28 +++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/server/dao/service.py b/server/dao/service.py index 38bed5e796b..489927f851d 100644 --- a/server/dao/service.py +++ b/server/dao/service.py @@ -55,6 +55,7 @@ def list(self, service_filter={}): outerjoin(Credential, (Credential.service_id == Service.id) and (Credential.host_id == None)).\ outerjoin(Host, Host.id == Interface.host_id) + query = query.filter(Service.workspace == self.workspace) query = apply_search_filter(query, self.COLUMNS_MAP, None, service_filter, self.STRICT_FILTERING) # 'LIKE' for search services started by hostId.%.% diff --git a/test_cases/dao/services.py b/test_cases/dao/services.py index e1ef6dd9420..2d4e50fad1f 100644 --- a/test_cases/dao/services.py +++ b/test_cases/dao/services.py @@ -23,7 +23,33 @@ def tearDown(self): with app.app_context(): db.drop_all() - def test_(self): + def test_list_with_multiple_workspace(self): + with app.app_context(): + workspace = WorkspaceFactory.build() + service_dao = ServiceDAO(workspace) + expected = {'services': []} + + res = service_dao.list() + assert expected == res + + new_service = ServiceFactory.build(workspace=workspace) + db.session.add(new_service) + db.session.commit() + + res = service_dao.list() + expected = {'services': [{'value': {'status': None, 'protocol': None, 'description': new_service.description, '_rev': None, 'owned': None, 'owner': None, 'credentials': 0, 'name': new_service.name, 'version': None, '_id': None, 'ports': [int(new_service.ports)], 'metadata': {'update_time': None, 'create_time': None, 'update_user': None, 'update_action': None, 'creator': None, 'owner': None, 'update_controller_action': None, 'command_id': None}}, '_id': 2, 'id': None, 'key': None, 'vulns': 0}]} + + assert expected == res + + another_workspace = WorkspaceFactory.build() + another_service = ServiceFactory.build(workspace=another_workspace) + db.session.add(another_service) + db.session.commit() + res = service_dao.list() + assert len(res['services']) == 1 + assert expected == res + + def test_count_with_multiple_workspaces(self): with app.app_context(): workspace = WorkspaceFactory.build() service_dao = ServiceDAO(workspace) From 511d009c9c96ae7ad7c7ee6b2a301145574ecbfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 14 Aug 2017 18:34:09 -0300 Subject: [PATCH 0075/1506] Continue refactor of server tests Add a common logged_user fixture that will be useful for future test cases, and refactor test_200_when_logged_in to use it. --- test_cases/conftest.py | 19 +++++++++++++++++++ test_cases/test_server.py | 40 +++++++++++++++++---------------------- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 7e01cc81cc9..05447fe15ae 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -23,6 +23,10 @@ def app(monkeypatch): os.close(db_fd) os.unlink(db_name) +@pytest.fixture +def test_client(app): + return app.test_client() + def create_user(app, username, email, password, **kwargs): user = app.user_datastore.create_user(username=username, @@ -33,10 +37,25 @@ def create_user(app, username, email, password, **kwargs): db.session.commit() return user + @pytest.fixture def user(app): return create_user(app, 'test', 'user@test.com', 'password', is_ldap=False) + @pytest.fixture def ldap_user(app): return create_user(app, 'ldap', 'ldap@test.com', 'password', is_ldap=True) + + +def login_as(test_client, user): + with test_client.session_transaction() as sess: + # Without this line the test breaks. Taken from + # http://pythonhosted.org/Flask-Testing/#testing-with-sqlalchemy + db.session.add(user) + sess['user_id'] = user.id + +@pytest.fixture +def logged_user(test_client, user): + login_as(test_client, user) + return user diff --git a/test_cases/test_server.py b/test_cases/test_server.py index a68276be96c..0d3994dce38 100644 --- a/test_cases/test_server.py +++ b/test_cases/test_server.py @@ -15,29 +15,15 @@ def endpoint(): return 'OK' -class BaseAPITestCase(unittest.TestCase): +class BaseAPITestCase: + ENDPOINT_ROUTE = '/' @pytest.fixture(autouse=True) - def load_app(self, app): + def load_app(self, app, test_client): """Use this to avoid having to use an app argument to every function""" self.flask_app = app - self.app = app.test_client() - - def login_as(self, user): - with self.app.session_transaction() as sess: - # Without this line the test breaks. Taken from - # http://pythonhosted.org/Flask-Testing/#testing-with-sqlalchemy - db.session.add(self.user) - - sess['user_id'] = user.id - - -class TestAuthentication(BaseAPITestCase): - """Tests related to allow/dissallow access depending of whether - the user is logged in or not""" - - ENDPOINT_ROUTE = '/' + self.app = test_client @pytest.fixture(autouse=True) def load_user(self, user): @@ -47,6 +33,11 @@ def load_user(self, user): def route_endpoint(self, app): app.route(self.ENDPOINT_ROUTE)(endpoint) + +class TestAuthentication(BaseAPITestCase, unittest.TestCase): + """Tests related to allow/dissallow access depending of whether + the user is logged in or not""" + def test_403_when_getting_an_existent_view_and_not_logged(self): res = self.app.get('/') self.assertEqual(res.status_code, 403) @@ -59,11 +50,6 @@ def test_403_when_accessing_a_non_existent_view_and_not_logged(self): res = self.app.post('/dfsdfsdd', 'data') self.assertEqual(res.status_code, 403) - def test_200_when_logged_in(self): - self.login_as(self.user) - res = self.app.get('/') - self.assertEqual(res.status_code, 200) - def test_200_when_not_logged_but_endpoint_is_public(self): endpoint.is_public = True res = self.app.get('/') @@ -87,5 +73,13 @@ def test_403_when_logged_user_is_deleted(self): self.assertEqual(res.status_code, 403) +class TestAuthenticationPytest(BaseAPITestCase): + + @pytest.mark.usefixtures('logged_user') + def test_200_when_logged_in(self, test_client): + res = test_client.get('/') + assert res.status_code == 200 + + if __name__ == '__main__': unittest.main() From e00315e4bc492468fdfc1f27ad170bd9182795ab Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 14 Aug 2017 19:04:28 -0300 Subject: [PATCH 0076/1506] Update flask sqlalchemt fixtures and change service test to use pytest fixtures --- server/app.py | 2 + test_cases/conftest.py | 61 ++++++++++++++++----- test_cases/dao/__init__.py | 1 + test_cases/dao/services.py | 105 ++++++++++++++++--------------------- 4 files changed, 97 insertions(+), 72 deletions(-) diff --git a/server/app.py b/server/app.py index c97d153f81d..ed28f00471f 100644 --- a/server/app.py +++ b/server/app.py @@ -25,6 +25,8 @@ def create_app(db_connection_string=None, testing=None): app.config['SECURITY_USER_IDENTITY_ATTRIBUTES'] = ['username'] app.config['SECURITY_POST_LOGIN_VIEW'] = '/_api/session' app.config['SECURITY_POST_LOGOUT_VIEW'] = '/_api/login' + if testing: + app.config['TESTING'] = testing from server.models import db db.init_app(app) diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 7e01cc81cc9..4493fdf3dc3 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -1,6 +1,5 @@ import os import sys -import tempfile import pytest sys.path.append(os.path.abspath(os.getcwd())) @@ -8,20 +7,54 @@ from server.models import db -@pytest.fixture -def app(monkeypatch): - db_fd, db_name = tempfile.mkstemp() - db_path = 'sqlite:///' + db_name - app = create_app(db_connection_string=db_path, testing=True) +@pytest.fixture(scope='session') +def app(request): + # we use sqlite memory for tests + test_conn_string = 'sqlite://' + app = create_app(db_connection_string=test_conn_string, testing=True) + + # Establish an application context before running the tests. + ctx = app.app_context() + ctx.push() + + def teardown(): + ctx.pop() + + request.addfinalizer(teardown) + return app + + +@pytest.fixture(scope='session') +def database(app, request): + """Session-wide test database.""" + + def teardown(): + db.drop_all() + + db.app = app + db.create_all() - # monkeypatch.setattr('flask.current_app', app) - # monkeypatch.setattr('flask_security.forms.current_app', app) + request.addfinalizer(teardown) + return db - with app.app_context(): - db.create_all() - yield app#.test_client() - os.close(db_fd) - os.unlink(db_name) + +@pytest.fixture(scope='function') +def session(database, request): + connection = db.engine.connect() + transaction = connection.begin() + + options = {"bind": connection, 'binds': {}} + session = db.create_scoped_session(options=options) + + db.session = session + + def teardown(): + transaction.rollback() + connection.close() + session.remove() + + request.addfinalizer(teardown) + return session def create_user(app, username, email, password, **kwargs): @@ -33,10 +66,12 @@ def create_user(app, username, email, password, **kwargs): db.session.commit() return user + @pytest.fixture def user(app): return create_user(app, 'test', 'user@test.com', 'password', is_ldap=False) + @pytest.fixture def ldap_user(app): return create_user(app, 'ldap', 'ldap@test.com', 'password', is_ldap=True) diff --git a/test_cases/dao/__init__.py b/test_cases/dao/__init__.py index e69de29bb2d..8b137891791 100644 --- a/test_cases/dao/__init__.py +++ b/test_cases/dao/__init__.py @@ -0,0 +1 @@ + diff --git a/test_cases/dao/services.py b/test_cases/dao/services.py index 2d4e50fad1f..f2bfcffb467 100644 --- a/test_cases/dao/services.py +++ b/test_cases/dao/services.py @@ -4,67 +4,54 @@ import string import random import unittest -from server.web import app -from server.models import db, Service from server.dao.service import ServiceDAO from test_cases.factories import WorkspaceFactory, ServiceFactory -class VulnerabilityDAOTestCases(unittest.TestCase): - - def setUp(self): - with app.app_context(): - db.create_all() - self.workspace = WorkspaceFactory.build() - db.session.commit() - - def tearDown(self): - with app.app_context(): - db.drop_all() - - def test_list_with_multiple_workspace(self): - with app.app_context(): - workspace = WorkspaceFactory.build() - service_dao = ServiceDAO(workspace) - expected = {'services': []} - - res = service_dao.list() - assert expected == res - - new_service = ServiceFactory.build(workspace=workspace) - db.session.add(new_service) - db.session.commit() - - res = service_dao.list() - expected = {'services': [{'value': {'status': None, 'protocol': None, 'description': new_service.description, '_rev': None, 'owned': None, 'owner': None, 'credentials': 0, 'name': new_service.name, 'version': None, '_id': None, 'ports': [int(new_service.ports)], 'metadata': {'update_time': None, 'create_time': None, 'update_user': None, 'update_action': None, 'creator': None, 'owner': None, 'update_controller_action': None, 'command_id': None}}, '_id': 2, 'id': None, 'key': None, 'vulns': 0}]} - - assert expected == res - - another_workspace = WorkspaceFactory.build() - another_service = ServiceFactory.build(workspace=another_workspace) - db.session.add(another_service) - db.session.commit() - res = service_dao.list() - assert len(res['services']) == 1 - assert expected == res - - def test_count_with_multiple_workspaces(self): - with app.app_context(): - workspace = WorkspaceFactory.build() - service_dao = ServiceDAO(workspace) - expected = {'total_count': 0} - res = service_dao.count() - assert expected == res - new_service = ServiceFactory.build(workspace=workspace) - db.session.add(new_service) - db.session.commit() - res = service_dao.count() - expected = {'total_count': 1} - assert expected == res - another_workspace = WorkspaceFactory.build() - another_service = ServiceFactory.build(workspace=another_workspace) - db.session.add(another_service) - db.session.commit() - res = service_dao.count() - assert expected == res +def test_list_with_multiple_workspace(app, session): + with app.app_context(): + workspace = WorkspaceFactory.build() + service_dao = ServiceDAO(workspace) + expected = {'services': []} + + res = service_dao.list() + assert expected == res + + new_service = ServiceFactory.build(workspace=workspace) + session.add(new_service) + session.commit() + + res = service_dao.list() + expected = {'services': [{'value': {'status': None, 'protocol': None, 'description': new_service.description, '_rev': None, 'owned': None, 'owner': None, 'credentials': 0, 'name': new_service.name, 'version': None, '_id': None, 'ports': [int(new_service.ports)], 'metadata': {'update_time': None, 'create_time': None, 'update_user': None, 'update_action': None, 'creator': None, 'owner': None, 'update_controller_action': None, 'command_id': None}}, '_id': new_service.id, 'id': None, 'key': None, 'vulns': 0}]} + + assert expected == res + + another_workspace = WorkspaceFactory.build() + another_service = ServiceFactory.build(workspace=another_workspace) + session.add(another_service) + session.commit() + res = service_dao.list() + assert len(res['services']) == 1 + assert expected == res + + +def test_count_with_multiple_workspaces(app, session): + with app.app_context(): + workspace = WorkspaceFactory.build() + service_dao = ServiceDAO(workspace) + expected = {'total_count': 0} + res = service_dao.count() + assert expected == res + new_service = ServiceFactory.build(workspace=workspace) + session.add(new_service) + session.commit() + res = service_dao.count() + expected = {'total_count': 1} + assert expected == res + another_workspace = WorkspaceFactory.build() + another_service = ServiceFactory.build(workspace=another_workspace) + session.add(another_service) + session.commit() + res = service_dao.count() + assert expected == res From c09308549fadf4eece7615eee86868c238a6c325 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 14 Aug 2017 19:11:25 -0300 Subject: [PATCH 0077/1506] Fix test_server tests --- test_cases/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_cases/conftest.py b/test_cases/conftest.py index c7c7da869a1..4e2737c9145 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -72,12 +72,12 @@ def create_user(app, username, email, password, **kwargs): @pytest.fixture -def user(app): +def user(app, session): return create_user(app, 'test', 'user@test.com', 'password', is_ldap=False) @pytest.fixture -def ldap_user(app): +def ldap_user(app, session): return create_user(app, 'ldap', 'ldap@test.com', 'password', is_ldap=True) From ae1ff26ac633f3c56b5574e37739af1bdd732152 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 15 Aug 2017 11:52:41 -0300 Subject: [PATCH 0078/1506] Use workspace for list in dao --- server/dao/host.py | 1 + test_cases/dao/host.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 test_cases/dao/host.py diff --git a/server/dao/host.py b/server/dao/host.py index 9bc00ad72de..cf02bceba0e 100644 --- a/server/dao/host.py +++ b/server/dao/host.py @@ -63,6 +63,7 @@ def __query_database(self, search=None, page=0, page_size=0, order_by=None, orde .outerjoin(Credential, (Credential.host_id == Host.id) & Credential.service_id == None)\ .group_by(Host.id) + query = query.filter(Host.workspace == self.workspace) # Apply pagination, sorting and filtering options to the query query = sort_results(query, self.COLUMNS_MAP, order_by, order_dir, default=Host.id) query = apply_search_filter(query, self.COLUMNS_MAP, search, host_filter, self.STRICT_FILTERING) diff --git a/test_cases/dao/host.py b/test_cases/dao/host.py new file mode 100644 index 00000000000..0fa43105650 --- /dev/null +++ b/test_cases/dao/host.py @@ -0,0 +1,36 @@ +import os +import sys +sys.path.append(os.path.abspath(os.getcwd())) +import string +import random +import unittest + +from server.dao.host import HostDAO +from test_cases.factories import WorkspaceFactory, HostFactory + + +def test_list_with_multiple_workspace(app, session): + with app.app_context(): + workspace = WorkspaceFactory.build() + + hosts_dao = HostDAO(workspace) + expected = {'rows': [], 'total_rows': 0} + + res = hosts_dao.list() + assert expected == res + + host = HostFactory.build(workspace=workspace) + session.add(host) + session.commit() + expected = {'rows': [{'value': {'description': host.description, 'default_gateway': [None, None], 'vulns': 0, '_rev': None, 'owned': None, 'owner': None, 'services': 0, 'credentials': 0, 'name': host.name, '_id': None, 'os': host.os, 'interfaces': [], 'metadata': {'update_time': None, 'create_time': None, 'update_user': None, 'update_action': None, 'creator': None, 'owner': None, 'update_controller_action': None, 'command_id': None}}, '_id': host.id, 'id': None, 'key': None}], 'total_rows': 1} + res = hosts_dao.list() + assert expected == res + + another_workspace = WorkspaceFactory.build() + + another_host = HostFactory.build(workspace=another_workspace) + session.add(another_host) + session.commit() + + res = hosts_dao.list() + assert expected == res From 4cab02b88f92ed3d8e4840522e752e3c3298694c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 15 Aug 2017 11:52:53 -0300 Subject: [PATCH 0079/1506] Some code style improvements --- server/dao/host.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/server/dao/host.py b/server/dao/host.py index cf02bceba0e..ef47af78e36 100644 --- a/server/dao/host.py +++ b/server/dao/host.py @@ -3,12 +3,24 @@ # See the file "doc/LICENSE" for the license information from server.dao.base import FaradayDAO -from server.utils.database import paginate, sort_results, apply_search_filter, get_count +from server.utils.database import ( + paginate, + sort_results, + apply_search_filter, + get_count +) from sqlalchemy import distinct from sqlalchemy.orm.query import Bundle from sqlalchemy.sql import func -from server.models import Host, Interface, Service, Vulnerability, EntityMetadata, Credential +from server.models import ( + Host, + Interface, + Service, + Vulnerability, + EntityMetadata, + Credential +) class HostDAO(FaradayDAO): @@ -126,4 +138,4 @@ def count(self, group_by=None): result_count["groups"] = [{group_by: value, "count": count} for value, count in res] - return result_count \ No newline at end of file + return result_count From be568d839e33fc90927403feae29fe3868df8ce8 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 15 Aug 2017 11:56:31 -0300 Subject: [PATCH 0080/1506] Fix hosts dao count to use workspace --- server/dao/host.py | 2 +- test_cases/dao/host.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/server/dao/host.py b/server/dao/host.py index ef47af78e36..cc56eb47487 100644 --- a/server/dao/host.py +++ b/server/dao/host.py @@ -120,7 +120,7 @@ def __get_host_data(self, host): }} def count(self, group_by=None): - total_count = self._session.query(func.count(Host.id)).scalar() + total_count = self._session.query(func.count(Host.id)).filter_by(workspace=self.workspace).scalar() # Return total amount of services if no group-by field was provided result_count = {"total_count": total_count} diff --git a/test_cases/dao/host.py b/test_cases/dao/host.py index 0fa43105650..0258a6f60f3 100644 --- a/test_cases/dao/host.py +++ b/test_cases/dao/host.py @@ -34,3 +34,29 @@ def test_list_with_multiple_workspace(app, session): res = hosts_dao.list() assert expected == res + + +def test_count_with_multiple_workspace(app, session): + with app.app_context(): + workspace = WorkspaceFactory.build() + + hosts_dao = HostDAO(workspace) + expected = {'total_count': 0} + + res = hosts_dao.count() + assert expected == res + + host = HostFactory.build(workspace=workspace) + session.add(host) + session.commit() + + another_workspace = WorkspaceFactory.build() + + another_host = HostFactory.build(workspace=another_workspace) + session.add(another_host) + session.commit() + + expected = {'total_count': 1} + + res = hosts_dao.count() + assert expected == res From 6ca39aa4a997e4b9ec2437a104578699ced18da9 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 15 Aug 2017 12:21:21 -0300 Subject: [PATCH 0081/1506] Fix credential dao to use workspace --- server/dao/credential.py | 15 +++++++++----- test_cases/dao/credential.py | 38 ++++++++++++++++++++++++++++++++++++ test_cases/factories.py | 11 ++++++++++- 3 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 test_cases/dao/credential.py diff --git a/server/dao/credential.py b/server/dao/credential.py index 2688e5b75a9..3d8c123dccd 100644 --- a/server/dao/credential.py +++ b/server/dao/credential.py @@ -34,15 +34,20 @@ def list(self, search=None, cred_filter={}): return result def __query_database(self, search=None, cred_filter={}): - creds_bundle = Bundle('cred', Credential.username, Credential.password, Credential.name, - Credential.description, Credential.owned, EntityMetadata.couchdb_id,\ - EntityMetadata.revision, EntityMetadata.update_time, EntityMetadata.update_user,\ - EntityMetadata.update_action, EntityMetadata.creator, EntityMetadata.create_time,\ - EntityMetadata.update_controller_action, EntityMetadata.owner, EntityMetadata.command_id) + creds_bundle = Bundle( + 'cred', Credential.username, Credential.password, + Credential.name, EntityMetadata.couchdb_id, + Credential.description, Credential.owned, + EntityMetadata.revision, EntityMetadata.update_time, + EntityMetadata.update_user, EntityMetadata.create_time, + EntityMetadata.update_action, EntityMetadata.creator, + EntityMetadata.update_controller_action, EntityMetadata.owner, + EntityMetadata.command_id) query = self._session.query(creds_bundle)\ .outerjoin(EntityMetadata, EntityMetadata.id == Credential.entity_metadata_id) + query = query.filter(Credential.workspace == self.workspace) # Apply filtering options to the query query = apply_search_filter(query, self.COLUMNS_MAP, search, cred_filter, self.STRICT_FILTERING) diff --git a/test_cases/dao/credential.py b/test_cases/dao/credential.py new file mode 100644 index 00000000000..d5463e697be --- /dev/null +++ b/test_cases/dao/credential.py @@ -0,0 +1,38 @@ +import os +import sys +sys.path.append(os.path.abspath(os.getcwd())) +import string +import random +import unittest + +from server.dao.credential import CredentialDAO +from test_cases.factories import WorkspaceFactory, CredentialFactory + + +def test_list_with_multiple_workspace(app, session): + with app.app_context(): + workspace = WorkspaceFactory.build() + + credentials_dao = CredentialDAO(workspace) + expected = {'rows': []} + + res = credentials_dao.list() + assert expected == res + + credential = CredentialFactory.build(workspace=workspace) + session.add(credential) + session.commit() + expected = {} + res = credentials_dao.list() + expected = {'rows': [{'value': {'username': credential.username, 'password': credential.password, 'description': None, 'couchid': None, 'owner': None, '_id': None, 'metadata': {'update_time': None, 'create_time': None, 'update_user': None, 'update_action': None, 'creator': None, 'owner': None, 'update_controller_action': None, 'command_id': None}, 'owned': None, 'name': None}, 'id': None, 'key': None}]} + + assert expected == res + + another_workspace = WorkspaceFactory.build() + + another_credential = CredentialFactory.build(workspace=another_workspace) + session.add(another_credential) + session.commit() + + res = credentials_dao.list() + assert expected == res diff --git a/test_cases/factories.py b/test_cases/factories.py index e469b075c76..a94e4c56ac2 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -10,6 +10,7 @@ Service, Interface, Workspace, + Credential, Vulnerability, EntityMetadata, ) @@ -83,14 +84,22 @@ class VulnerabilityFactory(FaradayFactory): policyviolations = '[]' refs = '[]' - class Meta: model = Vulnerability sqlalchemy_session = db.session +class CredentialFactory(FaradayFactory): + username = FuzzyText() + password = FuzzyText() + + class Meta: + model = Credential + sqlalchemy_session = db.session + register(WorkspaceFactory) register(HostFactory) register(ServiceFactory) register(InterfaceFactory) register(VulnerabilityFactory) +register(CredentialFactory) From 78bb0ef87c138493e2d7de512e19e02d2a796356 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 15 Aug 2017 12:37:11 -0300 Subject: [PATCH 0082/1506] Fix command dao to use workspace --- server/dao/command.py | 3 ++- test_cases/dao/command.py | 37 +++++++++++++++++++++++++++++++++++++ test_cases/factories.py | 10 ++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 test_cases/dao/command.py diff --git a/server/dao/command.py b/server/dao/command.py index 8f74600db88..db56f030d40 100644 --- a/server/dao/command.py +++ b/server/dao/command.py @@ -42,6 +42,7 @@ def __query_database(self, search=None, page=0, page_size=0, command_filter={}): .outerjoin(EntityMetadata, EntityMetadata.id == Command.entity_metadata_id) # Apply filtering options to the query + query = query.filter(Command.workspace == self.workspace) query = apply_search_filter(query, self.COLUMNS_MAP, None, command_filter, self.STRICT_FILTERING) if page_size: @@ -62,6 +63,6 @@ def __get_command_data(self, command): "hostname": command.hostname, "command": command.command, "user": command.user, - "workspace": command.workspace, + "workspace": command.workspace_id, "duration": command.duration, "params": command.params}} diff --git a/test_cases/dao/command.py b/test_cases/dao/command.py new file mode 100644 index 00000000000..9ddab8fdde0 --- /dev/null +++ b/test_cases/dao/command.py @@ -0,0 +1,37 @@ +import os +import sys +sys.path.append(os.path.abspath(os.getcwd())) +import string +import random +import unittest + +from server.dao.command import CommandDAO +from test_cases.factories import WorkspaceFactory, CommandFactory + + +def test_list_with_multiple_workspace(app, session): + with app.app_context(): + workspace = WorkspaceFactory.build() + + commands_dao = CommandDAO(workspace) + expected = {'commands': []} + + res = commands_dao.list() + assert expected == res + + command = CommandFactory.build(workspace=workspace) + session.add(command) + session.commit() + expected = {'commands': [{'value': {'itime': None, 'command': command.command, 'user': None, 'workspace': 0, 'params': None, 'duration': None, 'ip': None, '_id': None, 'hostname': None}, 'id': None, 'key': None}]} + res = commands_dao.list() + assert expected == res + + another_workspace = WorkspaceFactory.build() + + another_command = CommandFactory.build(workspace=another_workspace) + session.add(another_command) + session.commit() + + res = commands_dao.list() + assert len(res['commands']) == 1 + assert expected == res diff --git a/test_cases/factories.py b/test_cases/factories.py index a94e4c56ac2..195ba120cc4 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -7,6 +7,7 @@ from server.models import ( db, Host, + Command, Service, Interface, Workspace, @@ -97,6 +98,15 @@ class Meta: model = Credential sqlalchemy_session = db.session + +class CommandFactory(FaradayFactory): + command = FuzzyText() + + class Meta: + model = Command + sqlalchemy_session = db.session + + register(WorkspaceFactory) register(HostFactory) register(ServiceFactory) From a20cf685b548a7e8ef0b9f62053be903470a600e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 15 Aug 2017 12:37:25 -0300 Subject: [PATCH 0083/1506] Fix some pep8 warns --- server/dao/command.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/dao/command.py b/server/dao/command.py index db56f030d40..9ef1ec1ddd7 100644 --- a/server/dao/command.py +++ b/server/dao/command.py @@ -8,6 +8,7 @@ from server.models import Command, EntityMetadata from server.utils.database import apply_search_filter, paginate + class CommandDAO(FaradayDAO): MAPPED_ENTITY = Command COLUMNS_MAP = { @@ -18,7 +19,7 @@ class CommandDAO(FaradayDAO): def list(self, search=None, page=0, page_size=0, command_filter={}): results = self.__query_database(search, page, page_size, command_filter) - rows = [ self.__get_command_data(result.command) for result in results ] + rows = [self.__get_command_data(result.command) for result in results] result = { 'commands': rows @@ -33,8 +34,8 @@ def __query_database(self, search=None, page=0, page_size=0, command_filter={}): Command.hostname, Command.command, Command.user, - Command.workspace_id, Command.duration, + Command.workspace_id, Command.params, EntityMetadata.couchdb_id) From 095b0c17aa816aac17572bfbac72fd7ccfec374d Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 15 Aug 2017 18:26:57 -0300 Subject: [PATCH 0084/1506] update test to a more pytest friendly code --- test_cases/dao/vuln.py | 95 +++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 53 deletions(-) diff --git a/test_cases/dao/vuln.py b/test_cases/dao/vuln.py index c944ad527e8..eecdab185b0 100644 --- a/test_cases/dao/vuln.py +++ b/test_cases/dao/vuln.py @@ -11,56 +11,45 @@ from test_cases.factories import WorkspaceFactory, VulnerabilityFactory -class VulnerabilityDAOTestCases(unittest.TestCase): - - def setUp(self): - with app.app_context(): - db.create_all() - self.workspace = WorkspaceFactory.build() - db.session.commit() - - def tearDown(self): - with app.app_context(): - db.drop_all() - - def test_vulnerability_count_and_list_per_workspace_is_filtered(self): - """ - Verifies that the dao return the correct count from each workspace - """ - with app.app_context(): - another_workspace = WorkspaceFactory.build() - vuln_dao = VulnerabilityDAO(self.workspace) - another_vuln_dao = VulnerabilityDAO(another_workspace) - vuln_1 = VulnerabilityFactory.build(vuln_type='Vulnerability', workspace=self.workspace) - vuln_2 = VulnerabilityFactory.build(vuln_type='Vulnerability', workspace=another_workspace) - db.session.add(vuln_1) - db.session.add(vuln_2) - db.session.commit() - ws_count = vuln_dao.count() - another_ws_count = another_vuln_dao.count() - ws_expected = {'total_count': 1, 'web_vuln_count': 0, 'vuln_count': 1} - another_expected = {'total_count': 1, 'web_vuln_count': 0, 'vuln_count': 1} - assert ws_count == ws_expected - assert another_ws_count == another_expected - ws_list = vuln_dao.list() - - assert vuln_1.id == ws_list['vulnerabilities'][0]['_id'] - another_ws_list = another_vuln_dao.list() - assert vuln_2.id == another_ws_list['vulnerabilities'][0]['_id'] - - - - def test_count_by_type(self): - with app.app_context(): - vuln_dao = VulnerabilityDAO(self.workspace) - res = vuln_dao.count() - expected = {'total_count': 0, 'web_vuln_count': 0, 'vuln_count': 0} - self.assertEquals(expected, res) - vuln = VulnerabilityFactory.build(vuln_type='Vulnerability', workspace=self.workspace) - vuln_web = VulnerabilityFactory.build(vuln_type='VulnerabilityWeb', workspace=self.workspace) - db.session.add(vuln) - db.session.add(vuln_web) - db.session.commit() - res = vuln_dao.count() - expected = {'total_count': 2, 'web_vuln_count': 1, 'vuln_count': 1} - assert expected == res +def test_vulnerability_count_and_list_per_workspace_is_filtered(app, session): + """ + Verifies that the dao return the correct count from each workspace + """ + with app.app_context(): + workspace = WorkspaceFactory.build() + another_workspace = WorkspaceFactory.build() + vuln_dao = VulnerabilityDAO(workspace) + another_vuln_dao = VulnerabilityDAO(another_workspace) + vuln_1 = VulnerabilityFactory.build(vuln_type='Vulnerability', workspace=workspace) + vuln_2 = VulnerabilityFactory.build(vuln_type='Vulnerability', workspace=another_workspace) + db.session.add(vuln_1) + db.session.add(vuln_2) + db.session.commit() + ws_count = vuln_dao.count() + another_ws_count = another_vuln_dao.count() + ws_expected = {'total_count': 1, 'web_vuln_count': 0, 'vuln_count': 1} + another_expected = {'total_count': 1, 'web_vuln_count': 0, 'vuln_count': 1} + assert ws_count == ws_expected + assert another_ws_count == another_expected + ws_list = vuln_dao.list() + + assert vuln_1.id == ws_list['vulnerabilities'][0]['_id'] + another_ws_list = another_vuln_dao.list() + assert vuln_2.id == another_ws_list['vulnerabilities'][0]['_id'] + + +def test_count_by_type(app, session): + with app.app_context(): + workspace = WorkspaceFactory.build() + vuln_dao = VulnerabilityDAO(workspace) + res = vuln_dao.count() + expected = {'total_count': 0, 'web_vuln_count': 0, 'vuln_count': 0} + assert expected == res + vuln = VulnerabilityFactory.build(vuln_type='Vulnerability', workspace=workspace) + vuln_web = VulnerabilityFactory.build(vuln_type='VulnerabilityWeb', workspace=workspace) + db.session.add(vuln) + db.session.add(vuln_web) + db.session.commit() + res = vuln_dao.count() + expected = {'total_count': 2, 'web_vuln_count': 1, 'vuln_count': 1} + assert expected == res From c79d07655bdc4cf85a829c9d02631152c42b1c8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 17 Aug 2017 14:54:03 -0300 Subject: [PATCH 0085/1506] Don't import workspaces from CouchDB if and env var is set FARADAY_DONT_IMPORT --- server/importer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/importer.py b/server/importer.py index e0eff1613ef..587552863ad 100644 --- a/server/importer.py +++ b/server/importer.py @@ -4,6 +4,7 @@ import sys import json +import os import server.app import server.utils.logger @@ -438,6 +439,8 @@ def import_workspace_into_database(workspace_name, couchdb_server_conn): def _import_from_couchdb(workspace, couchdb_conn): + if 'FARADAY_DONT_IMPORT' in os.environ: + return couchdb_workspace = server.couchdb.CouchDBWorkspace(workspace.name, couchdb_server_conn=couchdb_conn) total_amount = couchdb_workspace.get_total_amount_of_documents() processed_docs, progress = 0, 0 From 33d05596f222596ff869ca5fb1c28f8f0e34b502 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 17 Aug 2017 17:08:22 -0300 Subject: [PATCH 0086/1506] remove old code for api views import. we use blueprints now --- server/api/__init__.py | 24 ------------------------ server/app.py | 5 ----- 2 files changed, 29 deletions(-) diff --git a/server/api/__init__.py b/server/api/__init__.py index 375acddd1e7..e4f24fb396f 100644 --- a/server/api/__init__.py +++ b/server/api/__init__.py @@ -1,27 +1,3 @@ # Faraday Penetration Test IDE # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information - -import os.path -import glob -import sys - -# Add handlers modules directory to sys.path -modules_path = os.path.join(os.path.dirname(__file__), 'modules') -sys.path.append(modules_path) - -# Get modules path, compiled or not -modules_files = glob.glob(os.path.join(modules_path, '*.py')) -modules_files += glob.glob(os.path.join(modules_path, '*.pyc')) - -# Remove duplicate names -extract_module_name = lambda module_path: os.path.splitext(os.path.basename(module_path))[0] -modules = set(map(extract_module_name, modules_files)) - -# Import and add to handlers namespace every module found in modules -for handler_name in modules: - globals()[handler_name] = __import__(handler_name) - - -def get_handlers(): - return BaseHandler.BaseHandler.get_handlers() diff --git a/server/app.py b/server/app.py index ed28f00471f..77e56e56eee 100644 --- a/server/app.py +++ b/server/app.py @@ -102,8 +102,3 @@ class MiniJSONEncoder(JSONEncoder): app.json_encoder = MiniJSONEncoder app.config['JSONIFY_PRETTYPRINT_REGULAR'] = False - - -# Load APIs -import server.api -import server.modules.info From 7e44dd5465ab03e81e66372578ad27ca0282ca2f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 17 Aug 2017 17:10:59 -0300 Subject: [PATCH 0087/1506] Remove import * to fix PEP8 warnings --- faraday.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/faraday.py b/faraday.py index a16172c8364..f62773ea9d4 100755 --- a/faraday.py +++ b/faraday.py @@ -17,7 +17,22 @@ import sys from config.configuration import getInstanceConfiguration -from config.globals import * +from config.globals import ( + CONST_USER_HOME, + CONST_FARADAY_HOME_PATH, + CONST_FARADAY_PLUGINS_PATH, + CONST_FARADAY_PLUGINS_REPO_PATH, + CONST_FARADAY_IMAGES, + CONST_FARADAY_USER_CFG, + CONST_FARADAY_BASE_CFG, + CONST_USER_ZSHRC, + CONST_FARADAY_ZSHRC, + CONST_ZSH_PATH, + CONST_FARADAY_ZSH_FARADAY, + CONST_VERSION_FILE, + CONST_REQUIREMENTS_FILE, + CONST_FARADAY_FOLDER_LIST, +) from utils import dependencies from utils.logs import getLogger, setUpLogger from utils.profilehooks import profile @@ -478,7 +493,7 @@ def checkCouchUrl(): to set the path of the cert """ sys.exit(-1) - except Exception as e: + except Exception: # Non fatal error pass @@ -496,7 +511,7 @@ def checkVersion(): getInstanceConfiguration().setVersion(f_version) f.close() - except Exception as e: + except Exception: getLogger("launcher").error( "It seems that something's wrong with your version\nPlease contact customer support") sys.exit(-1) From b46f1088738962d1c79f9fbe899fb89ef5f7e51a Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 17 Aug 2017 17:20:07 -0300 Subject: [PATCH 0088/1506] rename test for pytest --- test_cases/dao/{command.py => test_command.py} | 0 test_cases/dao/{credential.py => test_credential.py} | 0 test_cases/dao/{host.py => test_host.py} | 0 test_cases/dao/{services.py => test_services.py} | 0 test_cases/dao/{vuln.py => test_vuln.py} | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename test_cases/dao/{command.py => test_command.py} (100%) rename test_cases/dao/{credential.py => test_credential.py} (100%) rename test_cases/dao/{host.py => test_host.py} (100%) rename test_cases/dao/{services.py => test_services.py} (100%) rename test_cases/dao/{vuln.py => test_vuln.py} (100%) diff --git a/test_cases/dao/command.py b/test_cases/dao/test_command.py similarity index 100% rename from test_cases/dao/command.py rename to test_cases/dao/test_command.py diff --git a/test_cases/dao/credential.py b/test_cases/dao/test_credential.py similarity index 100% rename from test_cases/dao/credential.py rename to test_cases/dao/test_credential.py diff --git a/test_cases/dao/host.py b/test_cases/dao/test_host.py similarity index 100% rename from test_cases/dao/host.py rename to test_cases/dao/test_host.py diff --git a/test_cases/dao/services.py b/test_cases/dao/test_services.py similarity index 100% rename from test_cases/dao/services.py rename to test_cases/dao/test_services.py diff --git a/test_cases/dao/vuln.py b/test_cases/dao/test_vuln.py similarity index 100% rename from test_cases/dao/vuln.py rename to test_cases/dao/test_vuln.py From 4c7f625895de692e18d34bae04b0c48bb601820f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 17 Aug 2017 19:28:55 -0300 Subject: [PATCH 0089/1506] change status code to 401 --- server/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/app.py b/server/app.py index 77e56e56eee..691488a5cb1 100644 --- a/server/app.py +++ b/server/app.py @@ -80,10 +80,10 @@ def create_app(db_connection_string=None, testing=None): app.register_blueprint(session_api) # We are exposing a RESTful API, so don't redirect a user to a login page in - # case of being unauthorized, raise a 403 error instead + # case of being unauthorized, raise a 401 error instead @app.login_manager.unauthorized_handler def unauthorized(): - flask.abort(403) + flask.abort(401) @app.before_request def default_login_required(): From a90c99bdc29c588d29b0e0c804ca88315f6c641f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 17 Aug 2017 20:05:33 -0300 Subject: [PATCH 0090/1506] Start to use interceptor to redirect to login or forbidden page --- server/www/scripts/app.js | 12 ++++-------- server/www/scripts/auth/services/interceptor.js | 11 +++++++++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/server/www/scripts/app.js b/server/www/scripts/app.js index 3435bd800b2..8bb7e551535 100644 --- a/server/www/scripts/app.js +++ b/server/www/scripts/app.js @@ -271,6 +271,10 @@ faradayApp.config(['$routeProvider', 'ngClipProvider', '$uibTooltipProvider', controller: 'commercialCtrl', title: 'Methodologies | ' }). + when('/forbidden', { + templateUrl: 'scripts/auth/partials/forbidden.html', + title: ' Forbidden |' + }). otherwise({ templateUrl: 'scripts/commons/partials/home.html' }); @@ -282,12 +286,4 @@ faradayApp.run(['$location', '$rootScope', 'loginSrv', function($location, $root $rootScope.title = current.$$route.title; } }); - $rootScope.$on('$routeChangeStart', function(event){ - // Require in all routes (except the login one) - // Taken from http://stackoverflow.com/questions/26145871/redirect-on-all-routes-to-login-if-not-authenticated - // I don't know why this doesn't cause an infinite loop - loginSrv.isAuthenticated().then(function(auth){ - if(!auth) $location.path('/login'); - }); - }); }]); diff --git a/server/www/scripts/auth/services/interceptor.js b/server/www/scripts/auth/services/interceptor.js index fc643a679af..99b72dd82df 100644 --- a/server/www/scripts/auth/services/interceptor.js +++ b/server/www/scripts/auth/services/interceptor.js @@ -8,9 +8,9 @@ angular.module('faradayApp'). response: function(response){ return response; }, - + responseError: function(response) { - if(response.status === 401 || response.status === 403){ + if(response.status === 401){ var deferred = $q.defer(); loginSrv.isAuthenticated().then(function(auth){ if(!auth) { @@ -20,9 +20,16 @@ angular.module('faradayApp'). return deferred.reject(response); }); return deferred.promise; + }else if (response.status === 403) { + $location.path('/forbidden'); }else{ return $q.reject(response); } } } +}]); + +angular.module('faradayApp'). + config(['$httpProvider', function($httpProvider) { + $httpProvider.interceptors.push('AuthInterceptor'); }]); From 1b04e6147519e4c87fd9ec7de59ef7cf7b58e5e6 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 18 Aug 2017 11:34:54 -0300 Subject: [PATCH 0091/1506] Add forbidden page --- server/www/scripts/auth/partials/forbidden.html | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 server/www/scripts/auth/partials/forbidden.html diff --git a/server/www/scripts/auth/partials/forbidden.html b/server/www/scripts/auth/partials/forbidden.html new file mode 100644 index 00000000000..f39c7f455ee --- /dev/null +++ b/server/www/scripts/auth/partials/forbidden.html @@ -0,0 +1,10 @@ +
    +
    +
    + +
    +

    Forbidden

    +
    +
    +
    +
    From 795fdc3966794d8cee1e274a065525853b173fa6 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 18 Aug 2017 12:39:36 -0300 Subject: [PATCH 0092/1506] Disable SQLALCHEMY_TRACK_MODIFICATIONS We are not using flask-sqlalchemy events, but we will use sqlalchemy builtin events. SQLALCHEMY_TRACK_MODIFICATIONS is set to false to suppress warnings --- server/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/app.py b/server/app.py index 77e56e56eee..432ccda75e4 100644 --- a/server/app.py +++ b/server/app.py @@ -25,6 +25,7 @@ def create_app(db_connection_string=None, testing=None): app.config['SECURITY_USER_IDENTITY_ATTRIBUTES'] = ['username'] app.config['SECURITY_POST_LOGIN_VIEW'] = '/_api/session' app.config['SECURITY_POST_LOGOUT_VIEW'] = '/_api/login' + app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False if testing: app.config['TESTING'] = testing From 21f88728e2ff2672cc5a8ca9ea890babab9e024d Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 18 Aug 2017 14:20:11 -0300 Subject: [PATCH 0093/1506] add test_ prefix to test files. fix nexpose test --- test_cases/plugins/{acunetix.py => test_acunetix.py} | 0 test_cases/plugins/{burp.py => test_burp.py} | 0 test_cases/plugins/{nessus.py => test_nessus.py} | 0 .../plugins/{nexpose_full.py => test_nexpose_full.py} | 10 +++++----- test_cases/plugins/{nmap.py => test_nmap.py} | 0 test_cases/plugins/{ping.py => test_ping.py} | 0 test_cases/plugins/{telnet.py => test_telnet.py} | 0 test_cases/plugins/{whois.py => test_whois.py} | 0 8 files changed, 5 insertions(+), 5 deletions(-) rename test_cases/plugins/{acunetix.py => test_acunetix.py} (100%) rename test_cases/plugins/{burp.py => test_burp.py} (100%) rename test_cases/plugins/{nessus.py => test_nessus.py} (100%) rename test_cases/plugins/{nexpose_full.py => test_nexpose_full.py} (85%) rename test_cases/plugins/{nmap.py => test_nmap.py} (100%) rename test_cases/plugins/{ping.py => test_ping.py} (100%) rename test_cases/plugins/{telnet.py => test_telnet.py} (100%) rename test_cases/plugins/{whois.py => test_whois.py} (100%) diff --git a/test_cases/plugins/acunetix.py b/test_cases/plugins/test_acunetix.py similarity index 100% rename from test_cases/plugins/acunetix.py rename to test_cases/plugins/test_acunetix.py diff --git a/test_cases/plugins/burp.py b/test_cases/plugins/test_burp.py similarity index 100% rename from test_cases/plugins/burp.py rename to test_cases/plugins/test_burp.py diff --git a/test_cases/plugins/nessus.py b/test_cases/plugins/test_nessus.py similarity index 100% rename from test_cases/plugins/nessus.py rename to test_cases/plugins/test_nessus.py diff --git a/test_cases/plugins/nexpose_full.py b/test_cases/plugins/test_nexpose_full.py similarity index 85% rename from test_cases/plugins/nexpose_full.py rename to test_cases/plugins/test_nexpose_full.py index 3143fa211b8..30588f034f4 100644 --- a/test_cases/plugins/nexpose_full.py +++ b/test_cases/plugins/test_nexpose_full.py @@ -52,12 +52,12 @@ def test_Plugin_creates_apropiate_objects(self): self.assertEqual(action[2].name, "192.168.1.1") for i in range(131): action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDVULNHOST) + if type(action[2]) in [Vuln, VulnWeb]: + assert action[0] == modelactions.ADDVULNHOST + elif type(action[-1]) == Service: + assert action[0] == modelactions.ADDSERVICEINT action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDSERVICEINT) - for i in range(15): - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDVULNSRV) + assert action[0] == modelactions.ADDVULNSRV if __name__ == '__main__': diff --git a/test_cases/plugins/nmap.py b/test_cases/plugins/test_nmap.py similarity index 100% rename from test_cases/plugins/nmap.py rename to test_cases/plugins/test_nmap.py diff --git a/test_cases/plugins/ping.py b/test_cases/plugins/test_ping.py similarity index 100% rename from test_cases/plugins/ping.py rename to test_cases/plugins/test_ping.py diff --git a/test_cases/plugins/telnet.py b/test_cases/plugins/test_telnet.py similarity index 100% rename from test_cases/plugins/telnet.py rename to test_cases/plugins/test_telnet.py diff --git a/test_cases/plugins/whois.py b/test_cases/plugins/test_whois.py similarity index 100% rename from test_cases/plugins/whois.py rename to test_cases/plugins/test_whois.py From 088d87d286d4a852f6783303f249c05d1c24d84c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 18 Aug 2017 14:48:59 -0300 Subject: [PATCH 0094/1506] update test to be executed with pytest --- test_cases/server_models.py | 21 ------------------- test_cases/{common.py => test_common.py} | 8 +++---- test_cases/{models.py => test_models.py} | 0 .../{server_io.py => test_server_io.py} | 2 +- 4 files changed, 5 insertions(+), 26 deletions(-) delete mode 100644 test_cases/server_models.py rename test_cases/{common.py => test_common.py} (81%) rename test_cases/{models.py => test_models.py} (100%) rename test_cases/{server_io.py => test_server_io.py} (99%) diff --git a/test_cases/server_models.py b/test_cases/server_models.py deleted file mode 100644 index 25f1bed5550..00000000000 --- a/test_cases/server_models.py +++ /dev/null @@ -1,21 +0,0 @@ -import os -import sys -import json -import unittest -sys.path.append(os.path.abspath(os.getcwd())) - -from server.models import Interface - -INTERFACE_TEST_CASE_1 = {"network_segment": "", "description": "", "_rev": "1-ffffffffffffffffbcb43323dfeeeeee", "owned": False, "mac": "00:00:00:00:00:00", "hostnames": None, "owner": "", "name": "192.168.1.1", "ipv4": {"mask": "0.0.0.0", "gateway": "0.0.0.0", "DNS": [], "address": "192.168.1.1"}, "ipv6": {"prefix": "00", "gateway": "0000:0000:0000:0000:0000:0000:0000:0000", "DNS": [], "address": "0000:0000:0000:0000:0000:0000:0000:0000"}, "_id": "90aa44756bd2f4fc2390f903a6f25f43216b0790.0e9d8e8deab983df5e8af607f00901e089174881", "type": "Interface", "metadata": {"update_time": 1498579348.26915, "update_user": "leonardo", "update_action": 0, "creator": "Metasploit", "create_time": 1498579348.26915, "update_controller_action": "No model controller call", "owner": "leonardo", "command_id": "f09ea09db5264f2185a6d142ecd794f2"}} - - -class ModelsTest(unittest.TestCase): - - def test_(self): - interface = Interface(INTERFACE_TEST_CASE_1) - interface.update_from_document(INTERFACE_TEST_CASE_1) - self.assertEquals(interface.hostname, '') - - -if __name__ == '__main__': - unittest.main() diff --git a/test_cases/common.py b/test_cases/test_common.py similarity index 81% rename from test_cases/common.py rename to test_cases/test_common.py index 8626fbd15f4..aa183554fb6 100644 --- a/test_cases/common.py +++ b/test_cases/test_common.py @@ -34,13 +34,13 @@ def create_host_vuln(self, host, name, desc, severity): return vuln def create_int_vuln(self, host, interface, name, desc, severity): - vuln = Vulnerability(name, desc, severity) - self.model_controller.addVulnToInterfaceSYNC( host.getID(), interface.getID(), vuln) + vuln = Vulnerability(name=name, description=desc, severity=severity) + self.model_controller.addVulnToInterfaceSYNC(host.getID(), interface.getID(), vuln) return vuln def create_serv_vuln(self, host, service, name, desc, severity): - vuln = Vulnerability(name, desc, severity) - self.model_controller.addVulnToServiceSYNC( host.getID(), service.getID(), vuln) + vuln = Vulnerability(name=name, description=desc, severity=severity) + self.model_controller.addVulnToServiceSYNC(host.getID(), service.getID(), vuln) return vuln diff --git a/test_cases/models.py b/test_cases/test_models.py similarity index 100% rename from test_cases/models.py rename to test_cases/test_models.py diff --git a/test_cases/server_io.py b/test_cases/test_server_io.py similarity index 99% rename from test_cases/server_io.py rename to test_cases/test_server_io.py index 907ccbfa828..6fedde59d9f 100644 --- a/test_cases/server_io.py +++ b/test_cases/test_server_io.py @@ -45,7 +45,7 @@ def test_create_server_get_ws_names_url(self): @responses.activate def test_raise_conflict_in_database(self): url = "http://just_raise_conflict.com" - responses.add(responses.PUT, url, body='{"name": "betcha"}', status=409, + responses.add(responses.PUT, url, status=409, content_type="application/json", json={'error': 'conflict'}) with self.assertRaises(server_io_exceptions.ConflictInDatabase): server._unsafe_io_with_server(requests.put, 200, url, json={"name": "betcha"}) From 7c88ebc84c0410c458448d4e1c8bebda52a02ea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 22 Aug 2017 16:04:31 -0300 Subject: [PATCH 0095/1506] Add script to import users from CouchDB Not idempotent yet --- import_users_from_couch.py | 95 ++++++++++++++++++++++ server/app.py | 11 +++ test_cases/test_import_users_from_couch.py | 61 ++++++++++++++ 3 files changed, 167 insertions(+) create mode 100644 import_users_from_couch.py create mode 100644 test_cases/test_import_users_from_couch.py diff --git a/import_users_from_couch.py b/import_users_from_couch.py new file mode 100644 index 00000000000..5939a94d20c --- /dev/null +++ b/import_users_from_couch.py @@ -0,0 +1,95 @@ +from __future__ import print_function +import argparse +import requests +from urlparse import urljoin + +from binascii import unhexlify +from passlib.utils.binary import ab64_encode +from passlib.hash import pbkdf2_sha1 + +from flask_script import Manager +from server.web import app +from server.models import db + + +COUCHDB_USER_PREFIX = 'org.couchdb.user:' +COUCHDB_PASSWORD_PREXFIX = '-pbkdf2-' + + +def modular_crypt_pbkdf2_sha1(checksum, salt, iterations=1000): + return '$pbkdf2${iterations}${salt}${checksum}'.format( + iterations=iterations, + salt=ab64_encode(salt), + checksum=ab64_encode(unhexlify(checksum)), + ) + + +def convert_couchdb_hash(original_hash): + if not original_hash.startswith(COUCHDB_PASSWORD_PREXFIX): + # Should be a plaintext password + return original_hash + checksum, salt, iterations = original_hash[ + len(COUCHDB_PASSWORD_PREXFIX):].split(',') + iterations = int(iterations) + return modular_crypt_pbkdf2_sha1(checksum, salt, iterations) + + +def get_hash_from_document(doc): + scheme = doc.get('password_scheme', 'unset') + if scheme != 'pbkdf2': + raise ValueError('Unknown password scheme: %s' % scheme) + return modular_crypt_pbkdf2_sha1(doc['derived_key'], doc['salt'], + doc['iterations']) + + +def parse_all_docs(doc): + return [row['doc'] for row in doc['rows']] + + +def import_users(admins, all_users): + + manager = Manager(app) + with app.app_context(): + + # Import admin users + for (username, password) in admins.items(): + print("Creating user", username) + app.user_datastore.create_user( + username=username, + email=username + '@test.com', + password=convert_couchdb_hash(password), + is_ldap=False + ) + + # Import non admin users + for user in all_users: + if not user['_id'].startswith(COUCHDB_USER_PREFIX): + # It can be a view or something other than a user + continue + if user['name'] in admins.keys(): + # This is an already imported admin user, skip + continue + print("Importing", user['name']) + app.user_datastore.create_user( + username=user['name'], + email=user['name'] + '@test.com', + password=get_hash_from_document(user), + is_ldap=False + ) + db.session.commit() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--couch-url', default='http://localhost:5984') + parser.add_argument('username') + parser.add_argument('password') + args = parser.parse_args() + + auth = (args.username, args.password) + admins_url = urljoin(args.couch_url, + '/_config/admins') + users_url = urljoin(args.couch_url, + '/_users/_all_docs?include_docs=true') + import_users(requests.get(admins_url, auth=auth).json(), + parse_all_docs(requests.get(users_url, auth=auth).json())) diff --git a/server/app.py b/server/app.py index 77e56e56eee..5a45870b599 100644 --- a/server/app.py +++ b/server/app.py @@ -25,6 +25,17 @@ def create_app(db_connection_string=None, testing=None): app.config['SECURITY_USER_IDENTITY_ATTRIBUTES'] = ['username'] app.config['SECURITY_POST_LOGIN_VIEW'] = '/_api/session' app.config['SECURITY_POST_LOGOUT_VIEW'] = '/_api/login' + app.config['SECURITY_PASSWORD_SCHEMES'] = [ + 'bcrypt', # This should be the default value + # 'des_crypt', + 'pbkdf2_sha1', # Used by CouchDB passwords + # 'pbkdf2_sha256', + # 'pbkdf2_sha512', + # 'sha256_crypt', + # 'sha512_crypt', + # And always last one... + 'plaintext' # TODO: remove + ] if testing: app.config['TESTING'] = testing diff --git a/test_cases/test_import_users_from_couch.py b/test_cases/test_import_users_from_couch.py new file mode 100644 index 00000000000..91385fcb688 --- /dev/null +++ b/test_cases/test_import_users_from_couch.py @@ -0,0 +1,61 @@ +import pytest +from passlib.hash import pbkdf2_sha1 +import import_users_from_couch + +NON_ADMIN_DOC = { + "_id": "org.couchdb.user:removeme2", + "_rev": "0-00000000000000000000000000000000", + "password_scheme": "pbkdf2", + "iterations": 10, + "name": "removeme2", + "roles": [ + "pentester" + ], + "type": "user", + "derived_key": "d2061cb98f85b5e14eda97da556c1625906e5c2b", + "salt": "60604061640065389e9e90ca12c83b8d" +} + +ADMIN_DOC = { + "_id": "org.couchdb.user:removeme", + "_rev": "1-00000000000000000000000000000000", + "name": "removeme", + "password": None, + "roles": [], + "type": "user" +} + +def test_import_encrypted_password_from_admin_user(): + original_hash = ('-pbkdf2-eeea435c505e74d33a8c1b55c39d8dd355db4c2d,' + 'aedeef5a01f96a84360d2719fc521b9f,10') + new_hash = import_users_from_couch.convert_couchdb_hash(original_hash) + assert pbkdf2_sha1.verify('12345', new_hash) + + +def test_import_plaintext_password_from_admin_user(): + assert import_users_from_couch.convert_couchdb_hash('12345') == '12345' + + +def test_import_non_admin_from_document(): + new_hash = import_users_from_couch.get_hash_from_document(NON_ADMIN_DOC) + assert pbkdf2_sha1.verify('12345', new_hash) + + +def test_import_admin_from_document_fails(): + with pytest.raises(ValueError): + import_users_from_couch.get_hash_from_document(ADMIN_DOC) + + +def test_parse_all_docs_response_succeeds(): + doc_with_metadata = { + "id": "org.couchdb.user:removeme", + "key": "org.couchdb.user:removeme", + "value": {"rev": "1-00000000000000000000000000000000"}, + "doc": ADMIN_DOC + } + data = { + "total_rows": 15, + "offset": 0, + "rows": [doc_with_metadata] * 15 + } + assert import_users_from_couch.parse_all_docs(data) == [ADMIN_DOC] * 15 From 92d6855d123efe2ecf0c67299ad5af64bcfd6285 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 22 Aug 2017 16:06:00 -0300 Subject: [PATCH 0096/1506] Add WorkspaceDAO Also modifies the ws wndpoint to use the new DAO. --- server/api/modules/workspaces.py | 35 ++++++++-- server/dao/workspace.py | 114 +++++++++++++++++++++++++++++++ server/models.py | 13 +++- 3 files changed, 154 insertions(+), 8 deletions(-) create mode 100644 server/dao/workspace.py diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index 488eeebd1f6..476f06d147f 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -12,12 +12,16 @@ from server.dao.service import ServiceDAO from server.dao.interface import InterfaceDAO from server.dao.note import NoteDAO +from server.dao.workspace import WorkspaceDAO +from server.utils.logger import get_logger from server.utils.web import ( - gzipped, - validate_workspace, + build_bad_request_response, + filter_request_args, get_basic_auth, + get_integer_parameter, + gzipped, validate_admin_perm, - build_bad_request_response + validate_workspace ) from server.couchdb import ( list_workspaces_as_user, @@ -27,13 +31,30 @@ workspace_api = Blueprint('workspace_api', __name__) - @workspace_api.route('/ws', methods=['GET']) @gzipped def workspace_list(): - return flask.jsonify( - list_workspaces_as_user( - flask.request.cookies, get_basic_auth())) + get_logger(__name__).debug("Request parameters: {!r}" + .format(flask.request.args)) + + page = get_integer_parameter('page', default=0) + page_size = get_integer_parameter('page_size', default=0) + search = flask.request.args.get('search') + order_by = flask.request.args.get('sort') + order_dir = flask.request.args.get('sort_dir') + + ws_filter = filter_request_args('page', 'page_size', 'search', 'sort', 'sort_dir') + + ws_dao = WorkspaceDAO() + + result = ws_dao.list(search=search, + page=page, + page_size=page_size, + order_by=order_by, + order_dir=order_dir, + workspace_filter=ws_filter) + + return flask.jsonify(result) @workspace_api.route('/ws//summary', methods=['GET']) diff --git a/server/dao/workspace.py b/server/dao/workspace.py new file mode 100644 index 00000000000..52a6bd7fa74 --- /dev/null +++ b/server/dao/workspace.py @@ -0,0 +1,114 @@ +# Faraday Penetration Test IDE +# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) +# See the file "doc/LICENSE" for the license information + +from server.dao.base import FaradayDAO +from server.utils.database import ( + apply_search_filter, + get_count, + paginate +) +from server.utils import logger + +from sqlalchemy.orm.query import Bundle +from server.models import db, Workspace + + +class WorkspaceDAO(object): + + MAPPED_ENTITY = Workspace + + COLUMNS_MAP = { + "create_date": [Workspace.create_date], + "creator": [Workspace.creator], + "customer": [Workspace.customer], + "description": [Workspace.description], + "disabled": [Workspace.disabled], + "end_date": [Workspace.end_date], + "name": [Workspace.name], + "public": [Workspace.public], + "scope": [Workspace.scope], + "start_date": [Workspace.start_date], + "update_date": [Workspace.update_date] + } + + STRICT_FILTERING = ["name", "creator", "customer", "disabled", + "public", "update_date"] + + def __init__(self): + self._logger = logger.get_logger(self) + self._session = db.session + self._couchdb = None + + def get_all(self): + self.__check_valid_operation() + return self._session.query(self.MAPPED_ENTITY) + + def __check_valid_operation(self): + if self.MAPPED_ENTITY is None: + raise RuntimeError('Invalid operation') + + def save(self, obj): + self._session.add(obj) + self._session.commit() + + def list(self, search=None, page=0, page_size=0, order_by=None, + order_dir=None, workspace_filter={}): + + results, count = self.__query_database(search, page, page_size, + order_by, order_dir, + workspace_filter) + rows = [self.__get_workspace_data(result.workspace) + for result in results] + + result = { + "total_rows": count, + "rows": rows + } + + return result + + def __query_database(self, search=None, page=0, page_size=0, + order_by=None, order_dir=None, workspace_filter={}): + + workspace_bundle = Bundle('workspace', + Workspace.create_date, + Workspace.creator, + Workspace.customer, + Workspace.description, + Workspace.disabled, + Workspace.end_date, + Workspace.name, + Workspace.public, + Workspace.scope, + Workspace.start_date, + Workspace.update_date) + + query = self._session.query(workspace_bundle) + + query = apply_search_filter(query, self.COLUMNS_MAP, search, + workspace_filter, self.STRICT_FILTERING) + count = get_count(query, count_col=Workspace.name) + + if page_size: + query = paginate(query, page, page_size) + + results = query.all() + + return results, count + + def __get_workspace_data(self, workspace): + + return { + "create_date": workspace.create_date, + "creator": workspace.creator, + "customer": workspace.customer, + "description": workspace.description, + "disabled": workspace.disabled, + "end_date": workspace.end_date, + "name": workspace.name, + "public": workspace.public, + "scope": workspace.scope, + "start_date": workspace.start_date, + "update_date": workspace.update_date + } diff --git a/server/models.py b/server/models.py index d676a9ec993..0d16bd111e9 100644 --- a/server/models.py +++ b/server/models.py @@ -244,7 +244,18 @@ class Command(db.Model): class Workspace(db.Model): __tablename__ = 'workspace' id = Column(Integer, primary_key=True) - name = Column(String(250), nullable=True) + # TODO: change nullable=True for appropriate fields + create_date = Column(DateTime(), nullable=True) + creator = Column(Integer(), nullable=True) + customer = Column(String(250), nullable=True) + description = Column(Text(), nullable=True) + disabled = Column(Boolean(), nullable=True) + end_date = Column(DateTime(), nullable=True) + name = Column(String(250), nullable=True, unique=True) + public = Column(Boolean(), nullable=True) + scope = Column(Text(), nullable=True) + start_date = Column(DateTime(), nullable=True) + update_date = Column(DateTime(), nullable=True) def is_valid_workspace(workspace_name): From f018c937a5aa56bdc48b81c83c6e622b8b3dd0bb Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 22 Aug 2017 17:28:01 -0300 Subject: [PATCH 0097/1506] add fields to new model --- server/models.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/server/models.py b/server/models.py index d676a9ec993..71196feb11d 100644 --- a/server/models.py +++ b/server/models.py @@ -244,7 +244,7 @@ class Command(db.Model): class Workspace(db.Model): __tablename__ = 'workspace' id = Column(Integer, primary_key=True) - name = Column(String(250), nullable=True) + name = Column(String(250), nullable=False, unique=True) def is_valid_workspace(workspace_name): @@ -266,25 +266,27 @@ class Role(db.Model, RoleMixin): __tablename__ = 'role' id = Column(Integer(), primary_key=True) name = Column(String(80), unique=True) - description = Column(String(255)) + description = Column(String(255), nullable=True) class User(db.Model, UserMixin): __tablename__ = 'user' id = Column(Integer, primary_key=True) - email = Column(String(255), unique=True) - username = Column(String(255)) - password = Column(String(255)) - is_ldap = Column(Boolean(), nullable=False) - last_login_at = Column(DateTime()) - current_login_at = Column(DateTime()) - last_login_ip = Column(String(100)) - current_login_ip = Column(String(100)) - login_count = Column(Integer) - active = Column(Boolean()) + username = Column(String(255), unique=True, nullable=False) + password = Column(String(255), nullable=True) + email = Column(String(255), unique=True, nullable=True) # TBI + name = Column(String(255), nullalbe=True) # TBI + is_ldap = Column(Boolean(), nullable=False, default=False) + last_login_at = Column(DateTime()) # flask-security + current_login_at = Column(DateTime()) # flask-security + last_login_ip = Column(String(100)) # flask-security + current_login_ip = Column(String(100)) # flask-security + login_count = Column(Integer) # flask-security + active = Column(Boolean(), default=True, nullable=False) # TBI flask-security confirmed_at = Column(DateTime()) roles = relationship('Role', secondary='roles_users', backref=backref('users', lazy='dynamic')) + # TODO: add many to many relationship to add permission to workspace @property def role(self): From 998cd96ef67c1d6de0953dabf0353efe776ab415 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 22 Aug 2017 18:32:35 -0300 Subject: [PATCH 0098/1506] Add new model changes --- server/models.py | 92 ++++++++++++++++++++++++++++++------------------ 1 file changed, 57 insertions(+), 35 deletions(-) diff --git a/server/models.py b/server/models.py index 8ede9825877..563d11cf2f8 100644 --- a/server/models.py +++ b/server/models.py @@ -143,31 +143,55 @@ class Service(db.Model): workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) +class Reference(db.Model): + __tablename__ = 'reference' + id = Column(Integer, primary_key=True) + name = Column(String(250)) + + workspace = relationship('Workspace') + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + + vulnerability = relationship('Vulnerability') + vulnerability_id = Column(Integer, ForeignKey('vulnerbility.id'), index=True) + + +class PolicyViolation(db.Model): + __tablename__ = 'policiy_violation' + id = Column(Integer, primary_key=True) + name = Column(String(250)) + + workspace = relationship('Workspace') + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + + vulnerability = relationship('Vulnerability', backref='policy_violations') + vulnerability_id = Column(Integer, ForeignKey('vulnerbility.id'), index=True) + + class Vulnerability(db.Model): + # TODO: add unique constraint to -> name, description, severity, parent, method, pname, path, website + # revisar plugin nexpose, netspark para terminar de definir uniques. asegurar que se carguen bien __tablename__ = 'vulnerability' id = Column(Integer, primary_key=True) name = Column(String(250), nullable=False) description = Column(Text(), nullable=False) - confirmed = Column(Boolean) - vuln_type = Column(String(250)) + confirmed = Column(Boolean, default=False) + vuln_type = Column(String(250)) # TODO: add enum data = Column(Text()) - easeofresolution = Column(String(50)) - refs = Column(Text()) - resolution = Column(Text()) - severity = Column(String(50)) - owned = Column(Boolean) - attachments = Column(Text(), nullable=True) - policyviolations = Column(Text()) - - impact_accountability = Column(Boolean) - impact_availability = Column(Boolean) - impact_confidentiality = Column(Boolean) - impact_integrity = Column(Boolean) - - method = Column(String(50)) - params = Column(String(500)) - path = Column(String(500)) + ease_of_resolution = Column(String(50)) # TODO: add enum + resolution = Column(Text(), nullable=True) + severity = Column(String(50), nullalbe=False) # TODO: add enum + # TODO add evidence + + impact_accountability = Column(Boolean, default=False) + impact_availability = Column(Boolean, default=False) + impact_confidentiality = Column(Boolean, default=False) + impact_integrity = Column(Boolean, default=False) + + method = Column(String(50), nullable=True) + params = Column(String(500), nullable=True) + path = Column(String(500), nullable=True) + # REVIEW FROM FROM HERE pname = Column(String(250)) query = Column(Text()) request = Column(Text()) @@ -190,6 +214,7 @@ class Vulnerability(db.Model): class Note(db.Model): + # TODO: review this model __tablename__ = 'note' id = Column(Integer, primary_key=True) name = Column(String(250), nullable=False) @@ -208,18 +233,18 @@ class Credential(db.Model): id = Column(Integer, primary_key=True) username = Column(String(250), nullable=False) password = Column(Text(), nullable=False) - owned = Column(Boolean) description = Column(Text(), nullable=True) name = Column(String(250), nullable=True) entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) - host_id = Column(Integer, ForeignKey(Host.id), index=True) + host_id = Column(Integer, ForeignKey(Host.id), index=True, nullalbe=False) host = relationship('Host', back_populates='credentials') - service_id = Column(Integer, ForeignKey(Service.id), index=True) + service_id = Column(Integer, ForeignKey(Service.id), index=True, nullalbe=False) service = relationship('Service', back_populates='credentials') + workspace = relationship('Workspace') workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) @@ -227,16 +252,16 @@ class Credential(db.Model): class Command(db.Model): __tablename__ = 'command' id = Column(Integer, primary_key=True) - command = Column(String(250), nullable=True) - duration = Column(Float, nullable=True) - itime = Column(Float, nullable=True) - ip = Column(String(250), nullable=True) - hostname = Column(String(250), nullable=True) + command = Column(String(250), nullable=False) + start_date = Column(DateTime, nullable=False) + end_date = Column(DateTime, nullable=False) + ip = Column(String(250), nullable=False) # where the command was executed + hostname = Column(String(250), nullable=False) # where the command was executed params = Column(String(250), nullable=True) - user = Column(String(250), nullable=True) + user = Column(String(250), nullable=True) # where the command was executed workspace = relationship('Workspace') workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) - + # TODO: add Tool relationship and report_attachment entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) @@ -245,17 +270,14 @@ class Workspace(db.Model): __tablename__ = 'workspace' id = Column(Integer, primary_key=True) # TODO: change nullable=True for appropriate fields - create_date = Column(DateTime(), nullable=True) - creator = Column(Integer(), nullable=True) - customer = Column(String(250), nullable=True) + customer = Column(String(250), nullable=True) # TBI description = Column(Text(), nullable=True) - disabled = Column(Boolean(), nullable=True) + active = Column(Boolean(), nullable=False, default=True) # TBI end_date = Column(DateTime(), nullable=True) - name = Column(String(250), nullable=True, unique=True) - public = Column(Boolean(), nullable=True) + name = Column(String(250), nullable=False, unique=True) + public = Column(Boolean(), nullable=False, default=True) # TBI scope = Column(Text(), nullable=True) start_date = Column(DateTime(), nullable=True) - update_date = Column(DateTime(), nullable=True) def is_valid_workspace(workspace_name): From 5f0aaa7e5a2deb38e06deac8a3d58078b51a13fc Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 22 Aug 2017 19:39:24 -0300 Subject: [PATCH 0099/1506] Add initdb command to create the database --- manage.py | 13 +++++++++++++ requirements_server.txt | 1 + 2 files changed, 14 insertions(+) create mode 100644 manage.py diff --git a/manage.py b/manage.py new file mode 100644 index 00000000000..654c5e5dae5 --- /dev/null +++ b/manage.py @@ -0,0 +1,13 @@ +# manage.py + +from flask_script import Manager + +from server.web import app +from server.commands import InitDB + +manager = Manager(app) + + +if __name__ == "__main__": + manager.add_command('initdb', InitDB()) + manager.run() diff --git a/requirements_server.txt b/requirements_server.txt index 5ad45c1a9f1..4e95cbcb3e0 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -11,3 +11,4 @@ pyasn1-modules Flask-Security==3.0.0 bcrypt Flask-SQLAlchemy==2.2 +Flask-Script==2.0.5 From 9f99d010974b31054f1b2cbbe34e4b08fcaa8850 Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 23 Aug 2017 14:45:07 -0300 Subject: [PATCH 0100/1506] Update models to match new Relational Model Missing UNIQUE constraints, ENUMS and other details. No work was done on EntityMetadata object. --- server/models.py | 166 +++++++++++++++++++++++------------------------ 1 file changed, 81 insertions(+), 85 deletions(-) diff --git a/server/models.py b/server/models.py index 563d11cf2f8..6dca6afbaa6 100644 --- a/server/models.py +++ b/server/models.py @@ -56,119 +56,106 @@ class EntityMetadata(db.Model): class Host(db.Model): + # TODO: add unique constraint -> ip, workspace __tablename__ = 'host' id = Column(Integer, primary_key=True) - name = Column(String(250), nullable=False) - description = Column(Text(), nullable=False) - os = Column(String(250), nullable=False) + ip = Column(Text, nullable=False) # IP v4 or v6 + description = Column(Text, nullable=True) + os = Column(Text, nullable=True) - owned = Column(Boolean) + owned = Column(Boolean, nullable=False, default=False) - default_gateway_ip = Column(String(250)) - default_gateway_mac = Column(String(250)) + default_gateway_ip = Column(Text, nullable=True) + default_gateway_mac = Column(Text, nullable=True) + + mac = Column(Text, nullable=True) + net_segment = Column(Text, nullable=True) entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) - interfaces = relationship('Interface') - services = relationship('Service') - vulnerabilities = relationship('Vulnerability') - credentials = relationship('Credential') - - workspace = relationship('Workspace') + workspace = relationship('Workspace', backref='hosts') workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) -class Interface(db.Model): - __tablename__ = 'interface' +class Hostname(db.Model): + # TODO: add unique constraint -> name, host, workspace + __tablename__ = 'hostname' id = Column(Integer, primary_key=True) - name = Column(String(250), nullable=False) - description = Column(String(250), nullable=False) - mac = Column(String(250), nullable=False) - owned = Column(Boolean) - - hostnames = Column(String(250)) - network_segment = Column(String(250)) + name = Column(Text, nullable=False) - ipv4_address = Column(String(250)) - ipv4_gateway = Column(String(250)) - ipv4_dns = Column(String(250)) - ipv4_mask = Column(String(250)) - - ipv6_address = Column(String(250)) - ipv6_gateway = Column(String(250)) - ipv6_dns = Column(String(250)) - ipv6_prefix = Column(String(250)) - - ports_filtered = Column(Integer) - ports_opened = Column(Integer) - ports_closed = Column(Integer) - - entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) - entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) - - host_id = Column(Integer, ForeignKey(Host.id), index=True) - host = relationship('Host', back_populates='interfaces') - workspace = relationship('Workspace') - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) - - services = relationship('Service') + host = relationship('Host', backref='hostnames') + host_id = Column(Integer, ForeignKey('host.id'), index=True) class Service(db.Model): - # Table schema + # TODO: add unique constraint to -> port, protocol, host_id, workspace __tablename__ = 'service' id = Column(Integer, primary_key=True) - name = Column(String(250), nullable=False) - description = Column(String(250), nullable=False) - ports = Column(String(250), nullable=False) - owned = Column(Boolean) + name = Column(Text, nullable=True) + description = Column(Text, nullable=True) + port = Column(Integer, nullable=False) + owned = Column(Boolean, nullable=False, default=False) - protocol = Column(String(250)) - status = Column(String(250)) - version = Column(String(250)) + protocol = Column(Text, nullable=False) + # open, closed, filtered + status = Column(Text, nullable=True) # TODO: add enum + version = Column(Text, nullable=True) + + banner = Column(Text, nullable=True) entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) - host_id = Column(Integer, ForeignKey(Host.id), index=True) - host = relationship('Host', back_populates='services') - - interface_id = Column(Integer, ForeignKey(Interface.id), index=True) - interface = relationship('Interface', back_populates='services') + host = relationship('Host', backref='services') + host_id = Column(Integer, ForeignKey('host.id'), index=True) - vulnerabilities = relationship('Vulnerability') - credentials = relationship('Credential') - workspace = relationship('Workspace') + workspace = relationship('Workspace', backref='services') workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) class Reference(db.Model): __tablename__ = 'reference' id = Column(Integer, primary_key=True) - name = Column(String(250)) + name = Column(Text, nullable=False) - workspace = relationship('Workspace') - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace = relationship('Workspace', backref='references') + workspace_id = Column( + Integer, + ForeignKey('workspace.id'), + index=True + ) - vulnerability = relationship('Vulnerability') - vulnerability_id = Column(Integer, ForeignKey('vulnerbility.id'), index=True) + vulnerability = relationship('Vulnerability', backref='references') + vulnerability_id = Column( + Integer, + ForeignKey('vulnerbility.id'), + index=True + ) class PolicyViolation(db.Model): - __tablename__ = 'policiy_violation' + __tablename__ = 'policy_violation' id = Column(Integer, primary_key=True) - name = Column(String(250)) + name = Column(Text, nullable=False) - workspace = relationship('Workspace') - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace = relationship('Workspace', backref='policy_violations') + workspace_id = Column( + Integer, + ForeignKey('workspace.id'), + index=True + ) vulnerability = relationship('Vulnerability', backref='policy_violations') - vulnerability_id = Column(Integer, ForeignKey('vulnerbility.id'), index=True) + vulnerability_id = Column( + Integer, + ForeignKey('vulnerbility.id'), + index=True + ) class Vulnerability(db.Model): - # TODO: add unique constraint to -> name, description, severity, parent, method, pname, path, website + # TODO: add unique constraint to -> name, description, severity, parent, method, pname, path, website, workspace # revisar plugin nexpose, netspark para terminar de definir uniques. asegurar que se carguen bien __tablename__ = 'vulnerability' id = Column(Integer, primary_key=True) @@ -189,16 +176,17 @@ class Vulnerability(db.Model): impact_integrity = Column(Boolean, default=False) method = Column(String(50), nullable=True) - params = Column(String(500), nullable=True) + parameters = Column(String(500), nullable=True) path = Column(String(500), nullable=True) # REVIEW FROM FROM HERE - pname = Column(String(250)) - query = Column(Text()) - request = Column(Text()) - response = Column(Text()) - website = Column(String(250)) + parameter_name = Column(String(250), nullable=True) + query = Column(Text(), nullable=True) + request = Column(Text(), nullable=True) + response = Column(Text(), nullable=True) + website = Column(String(250), nullable=True) - status = Column(String(250)) + # open, closed, re-opened, risk-accepted + status = Column(String(250), nullable=False, default="open") # TODO: add enum entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) @@ -210,7 +198,12 @@ class Vulnerability(db.Model): service = relationship('Service', back_populates='vulnerabilities') workspace = relationship('Workspace') - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace_id = Column( + Integer, + ForeignKey('workspace.id'), + index=True, + back_populates='vulnerabilities' + ) class Note(db.Model): @@ -229,6 +222,8 @@ class Note(db.Model): class Credential(db.Model): + # TODO: add unique constraint -> username, host o service y workspace + # TODO: add constraint host y service, uno o el otro __tablename__ = 'credential' id = Column(Integer, primary_key=True) username = Column(String(250), nullable=False) @@ -239,14 +234,14 @@ class Credential(db.Model): entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) - host_id = Column(Integer, ForeignKey(Host.id), index=True, nullalbe=False) - host = relationship('Host', back_populates='credentials') + host = relationship('Host', backref='credentials') + host_id = Column(Integer, ForeignKey(Host.id), index=True, nullalbe=True) - service_id = Column(Integer, ForeignKey(Service.id), index=True, nullalbe=False) - service = relationship('Service', back_populates='credentials') + service = relationship('Service', backref='credentials') + service_id = Column(Integer, ForeignKey(Service.id), index=True, nullalbe=True) - workspace = relationship('Workspace') - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace = relationship('Workspace', backref='credentials') + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) class Command(db.Model): @@ -262,6 +257,7 @@ class Command(db.Model): workspace = relationship('Workspace') workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) # TODO: add Tool relationship and report_attachment + entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) From 89fbea4cafc929b5f0f3ce45fdf1ae5857652491 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 23 Aug 2017 16:44:10 -0300 Subject: [PATCH 0101/1506] Add enum to the model --- server/models.py | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/server/models.py b/server/models.py index 6dca6afbaa6..53e14f1c232 100644 --- a/server/models.py +++ b/server/models.py @@ -10,7 +10,8 @@ Float, Text, UniqueConstraint, - DateTime + DateTime, + Enum, ) from sqlalchemy.orm import relationship, backref from flask_sqlalchemy import SQLAlchemy @@ -59,7 +60,7 @@ class Host(db.Model): # TODO: add unique constraint -> ip, workspace __tablename__ = 'host' id = Column(Integer, primary_key=True) - ip = Column(Text, nullable=False) # IP v4 or v6 + ip = Column(Text, nullable=False) # IP v4 or v6 description = Column(Text, nullable=True) os = Column(Text, nullable=True) @@ -90,6 +91,11 @@ class Hostname(db.Model): class Service(db.Model): # TODO: add unique constraint to -> port, protocol, host_id, workspace + STATUSES = [ + 'open', + 'closed', + 'filtered' + ] __tablename__ = 'service' id = Column(Integer, primary_key=True) name = Column(Text, nullable=True) @@ -98,8 +104,7 @@ class Service(db.Model): owned = Column(Boolean, nullable=False, default=False) protocol = Column(Text, nullable=False) - # open, closed, filtered - status = Column(Text, nullable=True) # TODO: add enum + status = Column(Enum(*STATUSES), nullable=True) version = Column(Text, nullable=True) banner = Column(Text, nullable=True) @@ -157,17 +162,34 @@ class PolicyViolation(db.Model): class Vulnerability(db.Model): # TODO: add unique constraint to -> name, description, severity, parent, method, pname, path, website, workspace # revisar plugin nexpose, netspark para terminar de definir uniques. asegurar que se carguen bien + VULN_TYPES = [ + 'vuln', + 'vuln_web', + 'source_code'] + EASE_OF_RESOLUTIONS = [ + 'trivial', + 'simple', + 'moderate', + 'difficult', + 'infeasible' + ] + STATUSES = [ + 'open', + 'closed', + 're-opened', + 'risk-accepted' + ] __tablename__ = 'vulnerability' id = Column(Integer, primary_key=True) name = Column(String(250), nullable=False) description = Column(Text(), nullable=False) confirmed = Column(Boolean, default=False) - vuln_type = Column(String(250)) # TODO: add enum + vuln_type = Column(Enum(*VULN_TYPES)) data = Column(Text()) - ease_of_resolution = Column(String(50)) # TODO: add enum + ease_of_resolution = Column(Enum(*EASE_OF_RESOLUTIONS)) resolution = Column(Text(), nullable=True) - severity = Column(String(50), nullalbe=False) # TODO: add enum + severity = Column(String(50), nullalbe=False) # TODO add evidence impact_accountability = Column(Boolean, default=False) @@ -186,7 +208,7 @@ class Vulnerability(db.Model): website = Column(String(250), nullable=True) # open, closed, re-opened, risk-accepted - status = Column(String(250), nullable=False, default="open") # TODO: add enum + status = Column(Enum(*STATUSES), nullable=False, default="open") entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) From 617228545c2c4a894e84581e3f16911ab9fb3a21 Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 23 Aug 2017 17:36:41 -0300 Subject: [PATCH 0102/1506] Add models for Task and Methodology --- server/models.py | 96 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/server/models.py b/server/models.py index 53e14f1c232..c8cce59f991 100644 --- a/server/models.py +++ b/server/models.py @@ -360,3 +360,99 @@ def get_security_payload(self): def __repr__(self): return '<%sUser: %s>' % ('LDAP ' if self.is_ldap else '', self.username) + + +class MethodologyTemplate(db.Model): + # TODO: reset template_id in methodologies when deleting meth template + __tablename__ = 'methodology_template' + id = Column(Integer, primary_key=True) + name = Column(Text, nullable=False) + + +class Methodology(db.Model): + # TODO: add unique constraint -> name, workspace + __tablename__ = 'methodology' + id = Column(Integer, primary_key=True) + name = Column(Text, nullable=False) + + entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) + entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) + + template = relationship('MethodologyTemplate', backref='methodologies') + template_id = Column( + Integer, + ForeignKey('methodology_template.id'), + index=True, + nullable=True, + ) + + workspace = relationship('Workspace', backref='methodologies') + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + + +class TaskABC(db.Model): + __tablename__ = 'task_abc' + id = Column(Integer, primary_key=True) + name = Column(Text, nullable=False) + description = Column(Text, nullable=False) + + +class TaskTemplate(TaskABC): + __tablename__ = 'task_template' + id = Column(Integer, primary_key=True) + + __mapper_args__ = { + 'concrete': True + } + + template = relationship('MethodologyTemplate', backref='tasks') + template_id = Column( + Integer, + ForeignKey('methodology_template.id'), + index=True, + nullable=False, + ) + + +class Task(TaskABC): + STATUSES = [ + 'new', + 'in progress', + 'review', + 'completed', + ] + + __tablename__ = 'task' + id = Column(Integer, primary_key=True) + + due_date = Column(DateTime, nullable=True) + status = Column(Enum(*STATUSES), nullable=True) + + __mapper_args__ = { + 'concrete': True + } + + entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) + entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) + + assigned_to = relationship('User', backref='assigned_tasks') + assigned_to_id = Column(Integer, ForeignKey('user.id'), nullable=True) + + methodology = relationship('Methodology', backref='tasks') + methodology_id = Column( + Integer, + ForeignKey('methodology.id'), + index=True, + nullable=False, + ) + + template = relationship('TaskTemplate', backref='tasks') + template_id = Column( + Integer, + ForeignKey('task_template.id'), + index=True, + nullable=True, + ) + + workspace = relationship('Workspace', backref='tasks') + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) From d49a5173751c44887b9d117af087d557aba502b6 Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 23 Aug 2017 17:59:51 -0300 Subject: [PATCH 0103/1506] Add models for several objects that were missing Removes Notes model. Adds License, Tag, TagObject association class, Comment. --- server/models.py | 60 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/server/models.py b/server/models.py index c8cce59f991..12e5db11786 100644 --- a/server/models.py +++ b/server/models.py @@ -228,21 +228,6 @@ class Vulnerability(db.Model): ) -class Note(db.Model): - # TODO: review this model - __tablename__ = 'note' - id = Column(Integer, primary_key=True) - name = Column(String(250), nullable=False) - text = Column(Text(), nullable=True) - description = Column(Text(), nullable=True) - owned = Column(Boolean) - - entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) - entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) - workspace = relationship('Workspace') - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) - - class Credential(db.Model): # TODO: add unique constraint -> username, host o service y workspace # TODO: add constraint host y service, uno o el otro @@ -456,3 +441,48 @@ class Task(TaskABC): workspace = relationship('Workspace', backref='tasks') workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + + +class License(db.Model): + __tablename__ = 'license' + id = Column(Integer, primary_key=True) + product = Column(Text, nullable=False) + start_date = Column(DateTime, nullable=False) + end_date = Column(DateTime, nullable=False) + + type = Column(Text, nullable=True) + notes = Column(Text, nullable=True) + + +class Tag(db.Model): + __tablename__ = 'tag' + id = Column(Integer, primary_key=True) + name = Column(Text, nullable=False, unique=True) + slug = Column(Text, nullable=False, unique=True) + + +class TagObject(db.Model): + __tablename__ = 'tag_object' + id = Column(Integer, primary_key=True) + + object_id = Column(Integer, nullable=False) + object_type = Column(Text, nullable=False) + + tag = relationship('Tag', backref='tagged_objects') + tag_id = Column(Integer, ForeignKey('tag.id'), index=True) + + +class Comment(db.Model): + __tablename__ = 'comment' + id = Column(Integer, primary_key=True) + + text = Column(Text, nullable=False) + + reply_to = relationship('Comment', backref='replies') + reply_to_id = Column(Integer, ForeignKey('comment.id')) + + object_id = Column(Integer, nullable=False) + object_type = Column(Text, nullable=False) + + workspace = relationship('Workspace') + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) From a3c11eeb5be51c1a9908fa9ea45f20217b7c29e6 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 24 Aug 2017 12:33:59 -0300 Subject: [PATCH 0104/1506] add server command class --- server/commands/__init__.py | 56 +++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 server/commands/__init__.py diff --git a/server/commands/__init__.py b/server/commands/__init__.py new file mode 100644 index 00000000000..232ee31a008 --- /dev/null +++ b/server/commands/__init__.py @@ -0,0 +1,56 @@ +import sys +import getpass +from subprocess import Popen, PIPE + +try: + from configparser import ConfigParser, NoSectionError, NoOptionError +except ImportError: + from ConfigParser import ConfigParser, NoSectionError, NoOptionError + +from flask import current_app +from flask_script import Command + +from server.config import LOCAL_CONFIG_FILE + + +class InitDB(Command): + + def run(self): + database_name = raw_input('Please enter the database name (press enter to use "faraday"): ') or 'faraday' + username = raw_input('Please enter the database user (press enter to use "faraday"): ') or 'faraday' + postgres_command = ['sudo', '-u', 'postgres'] + password = None + while not password: + password = getpass.getpass(prompt='Please enter the password for the postgreSQL username: ') + if not password: + print('Please type a valid password') + command = postgres_command + ['psql', '-c', 'CREATE ROLE {0} WITH LOGIN PASSWORD \'{1}\';'.format(username, password)] + p = Popen(command) + p.wait() + db_server = raw_input('Enter the postgresql address (press enter for localhost): ') or 'localhost' + print('Creating database {0}'.format(database_name)) + command = postgres_command + ['createdb', '-O', username, database_name] + p = Popen(command, stdin=PIPE) + p.wait() + + print('Saving database credentials file in {0}'.format(LOCAL_CONFIG_FILE)) + config = ConfigParser() + config.read(LOCAL_CONFIG_FILE) + try: + config.get('database', 'connection_string') + except NoSectionError: + config.add_section('database') + + conn_string = 'postgresql+psycopg2://{username}:{password}@{server}'.format( + username=username, + password=password, + server=db_server, + ) + config.set('database', 'connection_string', conn_string) + with open(LOCAL_CONFIG_FILE, 'w') as configfile: + config.write(configfile) + + print('Creating tables') + from server.models import db + current_app.config['SQLALCHEMY_DATABASE_URI'] = conn_string + db.create_all() From 6a6ac87373e66d226fe186f5b18fd2e60ffcc1b8 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 24 Aug 2017 14:35:51 -0300 Subject: [PATCH 0105/1506] some improvements on dbinit and code cleanup --- manage.py | 3 +- server/commands/__init__.py | 63 ++++++++++++++++++++++++++++--------- 2 files changed, 51 insertions(+), 15 deletions(-) mode change 100644 => 100755 manage.py diff --git a/manage.py b/manage.py old mode 100644 new mode 100755 index 654c5e5dae5..e8c2ddd696f --- a/manage.py +++ b/manage.py @@ -1,4 +1,5 @@ -# manage.py +#!/usr/bin/env python + from flask_script import Manager diff --git a/server/commands/__init__.py b/server/commands/__init__.py index 232ee31a008..c06cbc3873a 100644 --- a/server/commands/__init__.py +++ b/server/commands/__init__.py @@ -1,3 +1,4 @@ +import os import sys import getpass from subprocess import Popen, PIPE @@ -10,46 +11,80 @@ from flask import current_app from flask_script import Command +from config.globals import CONST_FARADAY_HOME_PATH from server.config import LOCAL_CONFIG_FILE class InitDB(Command): + def _check_current_config(self, config): + try: + config.get('database', 'connection_string') + reconfigure = None + while not reconfigure: + reconfigure = raw_input('Database section \e[31m already found. \e[30m Do you want to reconfigure database? (yes/no) ') + if reconfigure.lower() == 'no': + return False + elif reconfigure.lower() == 'yes': + continue + else: + reconfigure = None + except NoSectionError: + config.add_section('database') + + return True + def run(self): - database_name = raw_input('Please enter the database name (press enter to use "faraday"): ') or 'faraday' - username = raw_input('Please enter the database user (press enter to use "faraday"): ') or 'faraday' + config = ConfigParser() + config.read(LOCAL_CONFIG_FILE) + if not self._check_current_config(config): + return + faraday_path_conf = os.path.expanduser(CONST_FARADAY_HOME_PATH) + psql_log_filename = os.path.join(faraday_path_conf, 'logs', 'psql_log.log') + with open(psql_log_filename, 'a+') as psql_log_file: + username, password = self._configure_postgres(psql_log_file) + database_name = self._create_database(username, psql_log_file) + conn_string = self._save_config(config, username, password, database_name) + self._create_tables(conn_string) + + def _configure_postgres(self, psql_log_file): + username = raw_input('Please enter the \e[21m database user \e[0m (press enter to use "faraday"): ') or 'faraday' postgres_command = ['sudo', '-u', 'postgres'] password = None while not password: - password = getpass.getpass(prompt='Please enter the password for the postgreSQL username: ') + password = getpass.getpass(prompt='Please enter the \e[21m password for the postgreSQL username \e[0m: ') if not password: print('Please type a valid password') command = postgres_command + ['psql', '-c', 'CREATE ROLE {0} WITH LOGIN PASSWORD \'{1}\';'.format(username, password)] - p = Popen(command) + p = Popen(command, stderr=psql_log_file, stdout=psql_log_file) p.wait() - db_server = raw_input('Enter the postgresql address (press enter for localhost): ') or 'localhost' + return username, password + + def _create_database(self, username, psql_log_file): + postgres_command = ['sudo', '-u', 'postgres'] + database_name = raw_input('Please enter the \e[21m database name \e[0m (press enter to use "faraday"): ') or 'faraday' print('Creating database {0}'.format(database_name)) command = postgres_command + ['createdb', '-O', username, database_name] - p = Popen(command, stdin=PIPE) + p = Popen(command, stderr=psql_log_file, stdout=psql_log_file) p.wait() + return database_name - print('Saving database credentials file in {0}'.format(LOCAL_CONFIG_FILE)) - config = ConfigParser() - config.read(LOCAL_CONFIG_FILE) - try: - config.get('database', 'connection_string') - except NoSectionError: - config.add_section('database') + def _save_config(self, config, username, password, database_name): + db_server = 'localhost' + print('\e[5m\e[39mSaving \e[30m database credentials file in {0}'.format(LOCAL_CONFIG_FILE)) - conn_string = 'postgresql+psycopg2://{username}:{password}@{server}'.format( + conn_string = 'postgresql+psycopg2://{username}:{password}@{server}/{database_name}'.format( username=username, password=password, server=db_server, + database_name=database_name ) config.set('database', 'connection_string', conn_string) with open(LOCAL_CONFIG_FILE, 'w') as configfile: config.write(configfile) + return conn_string + def _create_tables(self, conn_string): print('Creating tables') from server.models import db current_app.config['SQLALCHEMY_DATABASE_URI'] = conn_string From f86a58f088577e70e9008e9a67a38d834263fbae Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 24 Aug 2017 14:43:45 -0300 Subject: [PATCH 0106/1506] Add colors and catch keyboard interrupt exception --- server/commands/__init__.py | 39 ++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/server/commands/__init__.py b/server/commands/__init__.py index c06cbc3873a..9e3c09b8c55 100644 --- a/server/commands/__init__.py +++ b/server/commands/__init__.py @@ -10,9 +10,13 @@ from flask import current_app from flask_script import Command +from colorama import init +from colorama import Fore, Back, Style + from config.globals import CONST_FARADAY_HOME_PATH from server.config import LOCAL_CONFIG_FILE +init() class InitDB(Command): @@ -22,7 +26,7 @@ def _check_current_config(self, config): config.get('database', 'connection_string') reconfigure = None while not reconfigure: - reconfigure = raw_input('Database section \e[31m already found. \e[30m Do you want to reconfigure database? (yes/no) ') + reconfigure = raw_input('Database section {red} already found.{white} Do you want to reconfigure database? (yes/no) '.format(red=Fore.RED, white=Fore.WHITE)) if reconfigure.lower() == 'no': return False elif reconfigure.lower() == 'yes': @@ -35,24 +39,27 @@ def _check_current_config(self, config): return True def run(self): - config = ConfigParser() - config.read(LOCAL_CONFIG_FILE) - if not self._check_current_config(config): - return - faraday_path_conf = os.path.expanduser(CONST_FARADAY_HOME_PATH) - psql_log_filename = os.path.join(faraday_path_conf, 'logs', 'psql_log.log') - with open(psql_log_filename, 'a+') as psql_log_file: - username, password = self._configure_postgres(psql_log_file) - database_name = self._create_database(username, psql_log_file) - conn_string = self._save_config(config, username, password, database_name) - self._create_tables(conn_string) + try: + config = ConfigParser() + config.read(LOCAL_CONFIG_FILE) + if not self._check_current_config(config): + return + faraday_path_conf = os.path.expanduser(CONST_FARADAY_HOME_PATH) + psql_log_filename = os.path.join(faraday_path_conf, 'logs', 'psql_log.log') + with open(psql_log_filename, 'a+') as psql_log_file: + username, password = self._configure_postgres(psql_log_file) + database_name = self._create_database(username, psql_log_file) + conn_string = self._save_config(config, username, password, database_name) + self._create_tables(conn_string) + except KeyboardInterrupt: + print('User cancelled.') def _configure_postgres(self, psql_log_file): - username = raw_input('Please enter the \e[21m database user \e[0m (press enter to use "faraday"): ') or 'faraday' + username = raw_input('Please enter the {red} database user {white} (press enter to use "faraday"): '.format(red=Fore.RED, white=Fore.WHITE)) or 'faraday' postgres_command = ['sudo', '-u', 'postgres'] password = None while not password: - password = getpass.getpass(prompt='Please enter the \e[21m password for the postgreSQL username \e[0m: ') + password = getpass.getpass(prompt='Please enter the {red} password for the postgreSQL username {white}: '.format(red=Fore.RED, white=Fore.WHITE)) if not password: print('Please type a valid password') command = postgres_command + ['psql', '-c', 'CREATE ROLE {0} WITH LOGIN PASSWORD \'{1}\';'.format(username, password)] @@ -62,7 +69,7 @@ def _configure_postgres(self, psql_log_file): def _create_database(self, username, psql_log_file): postgres_command = ['sudo', '-u', 'postgres'] - database_name = raw_input('Please enter the \e[21m database name \e[0m (press enter to use "faraday"): ') or 'faraday' + database_name = raw_input('Please enter the {red} database name {white} (press enter to use "faraday"): '.format(red=Fore.RED, white=Fore.WHITE)) or 'faraday' print('Creating database {0}'.format(database_name)) command = postgres_command + ['createdb', '-O', username, database_name] p = Popen(command, stderr=psql_log_file, stdout=psql_log_file) @@ -71,7 +78,7 @@ def _create_database(self, username, psql_log_file): def _save_config(self, config, username, password, database_name): db_server = 'localhost' - print('\e[5m\e[39mSaving \e[30m database credentials file in {0}'.format(LOCAL_CONFIG_FILE)) + print('{red}Saving {white} database credentials file in {0}'.format(LOCAL_CONFIG_FILE, red=Fore.RED, white=Fore.WHITE)) conn_string = 'postgresql+psycopg2://{username}:{password}@{server}/{database_name}'.format( username=username, From a886dc51cdd777113990a7a8fef6bbd0045ec603 Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 24 Aug 2017 18:20:41 -0300 Subject: [PATCH 0107/1506] Add VulnerabilityABC class - Incomplete commit Removes table names from abstract classes and other changes. --- server/models.py | 55 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/server/models.py b/server/models.py index 12e5db11786..5b54007159a 100644 --- a/server/models.py +++ b/server/models.py @@ -159,13 +159,14 @@ class PolicyViolation(db.Model): ) -class Vulnerability(db.Model): +class VulnerabilityABC(db.Model): # TODO: add unique constraint to -> name, description, severity, parent, method, pname, path, website, workspace # revisar plugin nexpose, netspark para terminar de definir uniques. asegurar que se carguen bien VULN_TYPES = [ 'vuln', 'vuln_web', - 'source_code'] + 'source_code' + ] EASE_OF_RESOLUTIONS = [ 'trivial', 'simple', @@ -179,17 +180,20 @@ class Vulnerability(db.Model): 're-opened', 'risk-accepted' ] - __tablename__ = 'vulnerability' + + __abstract__ = True + id = Column(Integer, primary_key=True) - name = Column(String(250), nullable=False) - description = Column(Text(), nullable=False) - - confirmed = Column(Boolean, default=False) - vuln_type = Column(Enum(*VULN_TYPES)) - data = Column(Text()) - ease_of_resolution = Column(Enum(*EASE_OF_RESOLUTIONS)) - resolution = Column(Text(), nullable=True) - severity = Column(String(50), nullalbe=False) + + confirmed = Column(Boolean, nullable=False, default=False) + data = Column(Text, nullable=True) + description = Column(Text, nullable=False) + ease_of_resolution = Column(Enum(*EASE_OF_RESOLUTIONS), nullable=True) + name = Column(Text, nullable=False) + resolution = Column(Text, nullable=True) + severity = Column(String(50), nullable=False) + status = Column(Enum(*STATUSES), nullable=False, default="open") + vuln_type = Column(Enum(*VULN_TYPES), nullable=False) # TODO add evidence impact_accountability = Column(Boolean, default=False) @@ -197,18 +201,28 @@ class Vulnerability(db.Model): impact_confidentiality = Column(Boolean, default=False) impact_integrity = Column(Boolean, default=False) + # Web Vulns method = Column(String(50), nullable=True) parameters = Column(String(500), nullable=True) - path = Column(String(500), nullable=True) - # REVIEW FROM FROM HERE parameter_name = Column(String(250), nullable=True) + path = Column(String(500), nullable=True) query = Column(Text(), nullable=True) request = Column(Text(), nullable=True) response = Column(Text(), nullable=True) website = Column(String(250), nullable=True) - # open, closed, re-opened, risk-accepted - status = Column(Enum(*STATUSES), nullable=False, default="open") + # Code Vulns + line = Column(Integer, nullable=True) + + entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) + entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) + + +class Vulnerability(VulnerabilityABC): + __tablename__ = 'vulnerability' + id = Column(Integer, primary_key=True) + + vuln_type = Column(Enum(*VulnerabilityABC.VULN_TYPES), nullable=False) entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) @@ -228,6 +242,12 @@ class Vulnerability(db.Model): ) +class VulnerabilityTemplate(VulnerabilityABC): + __tablename__ = 'vulnerability_template' + + vuln_type = Column(Enum(*VULN_TYPES), nullable=False) + + class Credential(db.Model): # TODO: add unique constraint -> username, host o service y workspace # TODO: add constraint host y service, uno o el otro @@ -376,7 +396,8 @@ class Methodology(db.Model): class TaskABC(db.Model): - __tablename__ = 'task_abc' + __abstract__ = True + id = Column(Integer, primary_key=True) name = Column(Text, nullable=False) description = Column(Text, nullable=False) From a444be2e789ab6259e8c3ada39d6a997f559a4d3 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 25 Aug 2017 13:13:46 -0300 Subject: [PATCH 0108/1506] Set first database uri and then configure db for app --- server/app.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/server/app.py b/server/app.py index 432ccda75e4..94b530fac6b 100644 --- a/server/app.py +++ b/server/app.py @@ -28,6 +28,12 @@ def create_app(db_connection_string=None, testing=None): app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False if testing: app.config['TESTING'] = testing + try: + app.config['SQLALCHEMY_DATABASE_URI'] = db_connection_string or server.config.database.connection_string.strip("'") + except AttributeError: + print('Missing [database] section on server.ini. Please configure the database before running the server.') + except NoOptionError: + print('Missing connection_string on [database] section on server.ini. Please configure the database before running the server.') from server.models import db db.init_app(app) @@ -48,14 +54,6 @@ def create_app(db_connection_string=None, testing=None): for handler in LOGGING_HANDLERS: app.logger.addHandler(handler) - try: - app.config['SQLALCHEMY_DATABASE_URI'] = db_connection_string or server.config.database.connection_string.strip("'") - except AttributeError: - print('Missing [database] section on server.ini. Please configure the database before running the server.') - sys.exit(1) - except NoOptionError: - print('Missing connection_string on [database] section on server.ini. Please configure the database before running the server.') - sys.exit(1) from server.api.modules.workspaces import workspace_api from server.api.modules.doc import doc_api From 22d67d4f88cba596e63eedd4ac3103a09cbd76cd Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 25 Aug 2017 13:14:09 -0300 Subject: [PATCH 0109/1506] Add command and improve error checking Error checks includes detect missing database installation. --- server/commands/__init__.py | 46 ++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/server/commands/__init__.py b/server/commands/__init__.py index 9e3c09b8c55..1455f61686e 100644 --- a/server/commands/__init__.py +++ b/server/commands/__init__.py @@ -1,6 +1,7 @@ import os import sys import getpass +from tempfile import TemporaryFile from subprocess import Popen, PIPE try: @@ -39,22 +40,50 @@ def _check_current_config(self, config): return True def run(self): + """ + Main entry point that executes these steps: + * creates role in database. + * creates database. + * save new configuration on server.ini. + * creates tables. + """ + current_psql_output = TemporaryFile() try: config = ConfigParser() config.read(LOCAL_CONFIG_FILE) if not self._check_current_config(config): return faraday_path_conf = os.path.expanduser(CONST_FARADAY_HOME_PATH) + # we use psql_log_filename for historical saving. we will ask faraday users this file. + # current_psql_output is for checking psql command already known errors for each execution. psql_log_filename = os.path.join(faraday_path_conf, 'logs', 'psql_log.log') with open(psql_log_filename, 'a+') as psql_log_file: - username, password = self._configure_postgres(psql_log_file) - database_name = self._create_database(username, psql_log_file) + username, password = self._configure_postgres(current_psql_output) + current_psql_output.seek(0) + psql_output = current_psql_output.read() + psql_log_file.write(psql_output) + current_psql_output.seek(0) + psql_output = current_psql_output.read() + self._check_psql_output(psql_output) + database_name = self._create_database(username, current_psql_output) + self._check_psql_output(psql_output) + current_psql_output.close() conn_string = self._save_config(config, username, password, database_name) self._create_tables(conn_string) except KeyboardInterrupt: + current_psql_output.close() print('User cancelled.') + sys.exit(1) + + def _check_psql_output(self, psql_log_output): + if 'unknown user: postgres' in psql_log_output: + raise UserWarning('postgres user not found. did you installed postgresql?') def _configure_postgres(self, psql_log_file): + """ + This step will create the role on the database. + we return username and password and those values will be saved in the config file. + """ username = raw_input('Please enter the {red} database user {white} (press enter to use "faraday"): '.format(red=Fore.RED, white=Fore.WHITE)) or 'faraday' postgres_command = ['sudo', '-u', 'postgres'] password = None @@ -68,6 +97,9 @@ def _configure_postgres(self, psql_log_file): return username, password def _create_database(self, username, psql_log_file): + """ + This step uses the createdb command to add a new database. + """ postgres_command = ['sudo', '-u', 'postgres'] database_name = raw_input('Please enter the {red} database name {white} (press enter to use "faraday"): '.format(red=Fore.RED, white=Fore.WHITE)) or 'faraday' print('Creating database {0}'.format(database_name)) @@ -77,6 +109,9 @@ def _create_database(self, username, psql_log_file): return database_name def _save_config(self, config, username, password, database_name): + """ + This step saves database configuration to server.ini + """ db_server = 'localhost' print('{red}Saving {white} database credentials file in {0}'.format(LOCAL_CONFIG_FILE, red=Fore.RED, white=Fore.WHITE)) @@ -95,4 +130,9 @@ def _create_tables(self, conn_string): print('Creating tables') from server.models import db current_app.config['SQLALCHEMY_DATABASE_URI'] = conn_string - db.create_all() + try: + db.create_all() + except ImportError as ex: + if 'psycopg2' in ex: + print('Missing python depency {red}psycopg2{white}. Please install it with {green}pip install psycopg2'.format(red=Fore.RED, white=Fore.WHITE, green=Fore.GREEN)) + sys.exit(1) From 8f85febe09e28fed94b4f760b67bc0a679399e7e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 25 Aug 2017 15:11:37 -0300 Subject: [PATCH 0110/1506] Add command to generate db schemas --- manage.py | 14 ++++++++++ server/commands/__init__.py | 51 +++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100755 manage.py create mode 100644 server/commands/__init__.py diff --git a/manage.py b/manage.py new file mode 100755 index 00000000000..58aae122dcc --- /dev/null +++ b/manage.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python + + +from flask_script import Manager + +from server.web import app +from server.commands import DatabaseSchema + +manager = Manager(app) + + +if __name__ == "__main__": + manager.add_command('generate_database_schemas', DatabaseSchema()) + manager.run() diff --git a/server/commands/__init__.py b/server/commands/__init__.py new file mode 100644 index 00000000000..0b134acf51f --- /dev/null +++ b/server/commands/__init__.py @@ -0,0 +1,51 @@ +import sys +from flask_script import Command +from sqlalchemy import MetaData +try: + from sqlalchemy_schemadisplay import create_schema_graph + from sqlalchemy_schemadisplay import create_uml_graph +except ImportError: + print('Please install sqlalchemy_schemadisplay with "pip install sqlalchemy_schemadisplay"') + sys.exit(1) +from sqlalchemy.orm import class_mapper + +from server import models +import server.config + + +class DatabaseSchema(Command): + + def run(self): + self._draw_entity_diagrama() + self._draw_uml_class_diagram() + + def _draw_entity_diagrama(self): + # create the pydot graph object by autoloading all tables via a bound metadata object + graph = create_schema_graph( + metadata=MetaData(server.config.database.connection_string.strip("'")), + show_datatypes=False, # The image would get nasty big if we'd show the datatypes + show_indexes=False, # ditto for indexes + rankdir='LR', # From left to right (instead of top to bottom) + concentrate=False # Don't try to join the relation lines together + ) + graph.write_png('entity_dbschema.png') # write out the file + + def _draw_uml_class_diagram(self): + # lets find all the mappers in our model + mappers = [] + for attr in dir(models): + if attr[0] == '_': + continue + try: + cls = getattr(models, attr) + mappers.append(class_mapper(cls)) + except: + pass + + # pass them to the function and set some formatting options + graph = create_uml_graph( + mappers, + show_operations=False, # not necessary in this case + show_multiplicity_one=False # some people like to see the ones, some don't + ) + graph.write_png('uml_schema.png') # write out the file From ba7c1cde7e6ead96115eb0aaec9d18d7f5f42f91 Mon Sep 17 00:00:00 2001 From: micabot Date: Fri, 25 Aug 2017 15:15:52 -0300 Subject: [PATCH 0111/1506] Add Vuln types and SourceCode to models --- server/models.py | 83 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 20 deletions(-) diff --git a/server/models.py b/server/models.py index 5b54007159a..3366c1d43c0 100644 --- a/server/models.py +++ b/server/models.py @@ -56,6 +56,16 @@ class EntityMetadata(db.Model): document_type = Column(String(250)) +class SourceCode(db.Model): + # TODO: add unique constraint -> filename, workspace + __tablename__ = 'source_code' + id = Column(Integer, primary_key=True) + filename = Column(Text, nullable=False) + + workspace = relationship('Workspace', backref='source_codes') + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + + class Host(db.Model): # TODO: add unique constraint -> ip, workspace __tablename__ = 'host' @@ -162,11 +172,6 @@ class PolicyViolation(db.Model): class VulnerabilityABC(db.Model): # TODO: add unique constraint to -> name, description, severity, parent, method, pname, path, website, workspace # revisar plugin nexpose, netspark para terminar de definir uniques. asegurar que se carguen bien - VULN_TYPES = [ - 'vuln', - 'vuln_web', - 'source_code' - ] EASE_OF_RESOLUTIONS = [ 'trivial', 'simple', @@ -218,34 +223,72 @@ class VulnerabilityABC(db.Model): entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) -class Vulnerability(VulnerabilityABC): +class VulnerabilityGeneric(VulnerabilityABC): + VULN_TYPES = [ + 'vulnerability', + 'vulnerability_web', + 'vulnerability_code' + ] + __tablename__ = 'vulnerability' - id = Column(Integer, primary_key=True) - vuln_type = Column(Enum(*VulnerabilityABC.VULN_TYPES), nullable=False) + type = Column(Enum(*VULN_TYPES), nullable=False) + + workspace = relationship('Workspace', backref='vulnerabilities') + workspace_id = Column( + Integer, + ForeignKey('workspace.id'), + index=True, + ) + + __mapper_args__ = { + 'polymorphic_on': type + } + + +class Vulnerability(VulnerabilityGeneric): + id = Column(Integer, primary_key=True) entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) + host = relationship('Host', backref='vulnerabilities') host_id = Column(Integer, ForeignKey(Host.id), index=True) - host = relationship('Host', back_populates='vulnerabilities') + service = relationship('Service', backref='vulnerabilities') service_id = Column(Integer, ForeignKey(Service.id), index=True) - service = relationship('Service', back_populates='vulnerabilities') - workspace = relationship('Workspace') - workspace_id = Column( - Integer, - ForeignKey('workspace.id'), - index=True, - back_populates='vulnerabilities' - ) + __mapper_args__ = { + 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[0] + } -class VulnerabilityTemplate(VulnerabilityABC): - __tablename__ = 'vulnerability_template' +class VulnerabilityWeb(VulnerabilityGeneric): + id = Column(Integer, primary_key=True) - vuln_type = Column(Enum(*VULN_TYPES), nullable=False) + entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) + entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) + + service = relationship('Service', backref='vulnerabilities') + service_id = Column(Integer, ForeignKey(Service.id), index=True) + + __mapper_args__ = { + 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[1] + } + + +class VulnerabilityCode(VulnerabilityGeneric): + id = Column(Integer, primary_key=True) + + entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) + entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) + + source_code = relationship('SourceCode', backref='vulnerabilities') + source_code_id = Column(Integer, ForeignKey('SourceCode.id'), index=True) + + __mapper_args__ = { + 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[2] + } class Credential(db.Model): From 83cdef68eea1f8570c789c5a8ef76f0cc195bb5e Mon Sep 17 00:00:00 2001 From: micabot Date: Fri, 25 Aug 2017 15:22:14 -0300 Subject: [PATCH 0112/1506] Add VulnerabilityTemplate class to models Also removes type column from VulnerabilityGeneric class --- server/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index 3366c1d43c0..1316a86c67f 100644 --- a/server/models.py +++ b/server/models.py @@ -198,7 +198,6 @@ class VulnerabilityABC(db.Model): resolution = Column(Text, nullable=True) severity = Column(String(50), nullable=False) status = Column(Enum(*STATUSES), nullable=False, default="open") - vuln_type = Column(Enum(*VULN_TYPES), nullable=False) # TODO add evidence impact_accountability = Column(Boolean, default=False) @@ -223,6 +222,10 @@ class VulnerabilityABC(db.Model): entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) +class VulnerabilityTemplate(VulnerabilityABC): + __tablename__ = 'vulnerability_template' + + class VulnerabilityGeneric(VulnerabilityABC): VULN_TYPES = [ 'vulnerability', From f75227fa2552c6b78555d5ae87fcf2e267f789b7 Mon Sep 17 00:00:00 2001 From: micabot Date: Fri, 25 Aug 2017 15:25:40 -0300 Subject: [PATCH 0113/1506] Fix vuln fields in models Removes fields that don't belong in the VulnerabilityABC class and moves them to VulnerabilityGeneric so they won't be present in the Templates table. --- server/models.py | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/server/models.py b/server/models.py index 1316a86c67f..12b7a1e2c82 100644 --- a/server/models.py +++ b/server/models.py @@ -179,25 +179,16 @@ class VulnerabilityABC(db.Model): 'difficult', 'infeasible' ] - STATUSES = [ - 'open', - 'closed', - 're-opened', - 'risk-accepted' - ] __abstract__ = True - id = Column(Integer, primary_key=True) - confirmed = Column(Boolean, nullable=False, default=False) data = Column(Text, nullable=True) description = Column(Text, nullable=False) ease_of_resolution = Column(Enum(*EASE_OF_RESOLUTIONS), nullable=True) name = Column(Text, nullable=False) resolution = Column(Text, nullable=True) severity = Column(String(50), nullable=False) - status = Column(Enum(*STATUSES), nullable=False, default="open") # TODO add evidence impact_accountability = Column(Boolean, default=False) @@ -205,19 +196,6 @@ class VulnerabilityABC(db.Model): impact_confidentiality = Column(Boolean, default=False) impact_integrity = Column(Boolean, default=False) - # Web Vulns - method = Column(String(50), nullable=True) - parameters = Column(String(500), nullable=True) - parameter_name = Column(String(250), nullable=True) - path = Column(String(500), nullable=True) - query = Column(Text(), nullable=True) - request = Column(Text(), nullable=True) - response = Column(Text(), nullable=True) - website = Column(String(250), nullable=True) - - # Code Vulns - line = Column(Integer, nullable=True) - entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) @@ -227,6 +205,12 @@ class VulnerabilityTemplate(VulnerabilityABC): class VulnerabilityGeneric(VulnerabilityABC): + STATUSES = [ + 'open', + 'closed', + 're-opened', + 'risk-accepted' + ] VULN_TYPES = [ 'vulnerability', 'vulnerability_web', @@ -234,7 +218,8 @@ class VulnerabilityGeneric(VulnerabilityABC): ] __tablename__ = 'vulnerability' - + confirmed = Column(Boolean, nullable=False, default=False) + status = Column(Enum(*STATUSES), nullable=False, default="open") type = Column(Enum(*VULN_TYPES), nullable=False) workspace = relationship('Workspace', backref='vulnerabilities') @@ -268,6 +253,14 @@ class Vulnerability(VulnerabilityGeneric): class VulnerabilityWeb(VulnerabilityGeneric): id = Column(Integer, primary_key=True) + method = Column(String(50), nullable=True) + parameters = Column(String(500), nullable=True) + parameter_name = Column(String(250), nullable=True) + path = Column(String(500), nullable=True) + query = Column(Text(), nullable=True) + request = Column(Text(), nullable=True) + response = Column(Text(), nullable=True) + website = Column(String(250), nullable=True) entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) @@ -282,6 +275,7 @@ class VulnerabilityWeb(VulnerabilityGeneric): class VulnerabilityCode(VulnerabilityGeneric): id = Column(Integer, primary_key=True) + line = Column(Integer, nullable=True) entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) From 1e26cc657a358fa8005e9d3a39faa712116b2687 Mon Sep 17 00:00:00 2001 From: micabot Date: Fri, 25 Aug 2017 15:44:05 -0300 Subject: [PATCH 0114/1506] Remove metadata from Vulnerabilities --- server/models.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/server/models.py b/server/models.py index 12b7a1e2c82..2e379359def 100644 --- a/server/models.py +++ b/server/models.py @@ -2,30 +2,29 @@ # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information from sqlalchemy import ( + Boolean, Column, + DateTime, + Enum, + Float, + ForeignKey, Integer, String, - Boolean, - ForeignKey, - Float, Text, UniqueConstraint, - DateTime, - Enum, ) from sqlalchemy.orm import relationship, backref from flask_sqlalchemy import SQLAlchemy from flask_security import ( - UserMixin, RoleMixin, + UserMixin, ) import server.config db = SQLAlchemy() - -SCHEMA_VERSION = 'W.2.6.3' +SCHEMA_VERSION = 'W.3.0.0' class DatabaseMetadata(db.Model): @@ -196,9 +195,6 @@ class VulnerabilityABC(db.Model): impact_confidentiality = Column(Boolean, default=False) impact_integrity = Column(Boolean, default=False) - entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) - entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) - class VulnerabilityTemplate(VulnerabilityABC): __tablename__ = 'vulnerability_template' From 041956ebeee1a6d7dd0cca0b8e4c37b333eb50b7 Mon Sep 17 00:00:00 2001 From: micabot Date: Fri, 25 Aug 2017 15:58:12 -0300 Subject: [PATCH 0115/1506] Fix typos and make models work The file server/models.py is now clean and doesn't throw any errors on server initialization. --- server/models.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/server/models.py b/server/models.py index 2e379359def..37e9bf586ce 100644 --- a/server/models.py +++ b/server/models.py @@ -231,24 +231,22 @@ class VulnerabilityGeneric(VulnerabilityABC): class Vulnerability(VulnerabilityGeneric): - id = Column(Integer, primary_key=True) - - entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) - entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) - host = relationship('Host', backref='vulnerabilities') host_id = Column(Integer, ForeignKey(Host.id), index=True) service = relationship('Service', backref='vulnerabilities') service_id = Column(Integer, ForeignKey(Service.id), index=True) + __table_args__ = { + 'extend_existing': True + } + __mapper_args__ = { 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[0] } class VulnerabilityWeb(VulnerabilityGeneric): - id = Column(Integer, primary_key=True) method = Column(String(50), nullable=True) parameters = Column(String(500), nullable=True) parameter_name = Column(String(250), nullable=True) @@ -258,27 +256,28 @@ class VulnerabilityWeb(VulnerabilityGeneric): response = Column(Text(), nullable=True) website = Column(String(250), nullable=True) - entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) - entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) - - service = relationship('Service', backref='vulnerabilities') + service = relationship('Service', backref='vulnerabilities_web') service_id = Column(Integer, ForeignKey(Service.id), index=True) + __table_args__ = { + 'extend_existing': True + } + __mapper_args__ = { 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[1] } class VulnerabilityCode(VulnerabilityGeneric): - id = Column(Integer, primary_key=True) line = Column(Integer, nullable=True) - entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) - entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) - source_code = relationship('SourceCode', backref='vulnerabilities') source_code_id = Column(Integer, ForeignKey('SourceCode.id'), index=True) + __table_args__ = { + 'extend_existing': True + } + __mapper_args__ = { 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[2] } @@ -298,10 +297,10 @@ class Credential(db.Model): entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) host = relationship('Host', backref='credentials') - host_id = Column(Integer, ForeignKey(Host.id), index=True, nullalbe=True) + host_id = Column(Integer, ForeignKey(Host.id), index=True, nullable=True) service = relationship('Service', backref='credentials') - service_id = Column(Integer, ForeignKey(Service.id), index=True, nullalbe=True) + service_id = Column(Integer, ForeignKey(Service.id), index=True, nullable=True) workspace = relationship('Workspace', backref='credentials') workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) @@ -367,7 +366,7 @@ class User(db.Model, UserMixin): username = Column(String(255), unique=True, nullable=False) password = Column(String(255), nullable=True) email = Column(String(255), unique=True, nullable=True) # TBI - name = Column(String(255), nullalbe=True) # TBI + name = Column(String(255), nullable=True) # TBI is_ldap = Column(Boolean(), nullable=False, default=False) last_login_at = Column(DateTime()) # flask-security current_login_at = Column(DateTime()) # flask-security From f9a4994fced615f425497772524c704cbd8695c3 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 25 Aug 2017 18:05:34 -0300 Subject: [PATCH 0116/1506] Fix a bug, set SQLALCHEMY_DATABASE_URI before than init db --- server/app.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/server/app.py b/server/app.py index 432ccda75e4..6e79a079bc2 100644 --- a/server/app.py +++ b/server/app.py @@ -28,6 +28,14 @@ def create_app(db_connection_string=None, testing=None): app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False if testing: app.config['TESTING'] = testing + try: + app.config['SQLALCHEMY_DATABASE_URI'] = db_connection_string or server.config.database.connection_string.strip("'") + except AttributeError: + print('Missing [database] section on server.ini. Please configure the database before running the server.') + sys.exit(1) + except NoOptionError: + print('Missing connection_string on [database] section on server.ini. Please configure the database before running the server.') + sys.exit(1) from server.models import db db.init_app(app) @@ -48,15 +56,6 @@ def create_app(db_connection_string=None, testing=None): for handler in LOGGING_HANDLERS: app.logger.addHandler(handler) - try: - app.config['SQLALCHEMY_DATABASE_URI'] = db_connection_string or server.config.database.connection_string.strip("'") - except AttributeError: - print('Missing [database] section on server.ini. Please configure the database before running the server.') - sys.exit(1) - except NoOptionError: - print('Missing connection_string on [database] section on server.ini. Please configure the database before running the server.') - sys.exit(1) - from server.api.modules.workspaces import workspace_api from server.api.modules.doc import doc_api from server.api.modules.interfaces import interfaces_api From 05d799af2427b7ce9f750279beac79204455e737 Mon Sep 17 00:00:00 2001 From: micabot Date: Fri, 25 Aug 2017 18:31:51 -0300 Subject: [PATCH 0117/1506] Remove Interface logic to get the Server up There's still some in the repo, this commit only removes enough to avoid errors during Server bootstrap. --- apis/rest/api.py | 12 -- gui/gtk/dialogs.py | 210 ++++++------------- persistence/server/models.py | 342 +++++++++++++------------------ server/api/modules/interfaces.py | 28 --- server/api/modules/notes.py | 45 ---- server/api/modules/workspaces.py | 6 - server/app.py | 4 - server/dao/host.py | 12 +- server/dao/interface.py | 82 -------- server/dao/note.py | 76 ------- server/dao/service.py | 13 +- server/dao/vuln.py | 55 ++--- server/dao/workspace.py | 25 +-- server/importer.py | 54 +---- server/models.py | 153 +++++++++++--- 15 files changed, 361 insertions(+), 756 deletions(-) delete mode 100644 server/api/modules/interfaces.py delete mode 100644 server/api/modules/notes.py delete mode 100644 server/dao/interface.py delete mode 100644 server/dao/note.py diff --git a/apis/rest/api.py b/apis/rest/api.py index af0e86c39cb..3810bab056f 100644 --- a/apis/rest/api.py +++ b/apis/rest/api.py @@ -126,10 +126,6 @@ def getRoutes(self): view_func=self.listWebVulns, methods=['GET'])) - routes.append(Route(path='/model/interface', - view_func=self.createInterface, - methods=['PUT'])) - routes.append(Route(path='/model/service', view_func=self.createService, methods=['PUT'])) @@ -245,14 +241,6 @@ def createHost(self): self.controller.newHost, ['name', 'os']) - def createInterface(self): - return self._create( - self.controller.newInterface, - ['name', 'mac', 'ipv6_address', 'ipv4_mask', 'ipv4_gateway', - 'ipv4_dns', 'ipv6_address', 'ipv6_prefix', 'ipv6_gateway', - 'ipv6_dns', 'network_segment', 'hostname_resolution', - 'parent_id']) - def createService(self): return self._create( self.controller.newService, diff --git a/gui/gtk/dialogs.py b/gui/gtk/dialogs.py index 519db94a65b..96f96d047ed 100644 --- a/gui/gtk/dialogs.py +++ b/gui/gtk/dialogs.py @@ -566,7 +566,7 @@ def type_faraday_plugin_command(self, plugin): class HostInfoDialog(Gtk.Window): """Sets the blueprints for a simple host info window. It will display - basic information in labels as well as interfaces/services in a treeview. + basic information in labels as well as services in a treeview. While working in this class, keep in mind the distinction between selections (which are part of a model that holds data about an object as @@ -715,20 +715,15 @@ def create_vuln_list(self): def create_model(self, host): """Return a model for the given host. It holds quite a bit of info. It has 15 columns holding the host ID and name as parent, - all the information about the interfaces of that host and all the - information about the services of those interfaces. + all the information about the services. The model is difficult to draw because of its nested nature, but you can think of it like this, keeping in mind each node has several columns HOST - -----> INTERFACE1 - ------------> SERVICE1 - ------------> SERVICE2 - -----> INTERFACE2 - -----------> SERVICE1 - -----------> SERVICE2 + ------------> SERVICE1 + ------------> SERVICE2 And so on and so on, like Zizek says. """ @@ -758,60 +753,10 @@ def lst_to_str(lst): """Convenient function to avoid this long line everywhere""" return ', '.join([str(word) for word in lst if word]) - def add_interface_to_host_in_model(interface, host_pos, model): - """Append an interface to the host within a model. - Return the tree_iter represeting the position of the interface - within the model. Modifies the model. - """ - ipv4_dic = interface.getIPv4() - ipv6_dic = interface.getIPv6() - display_str = str(interface) - - position = model.append(host_pos, [interface.getID(), - interface.getName(), - interface.getDescription(), - interface.getMAC(), - ipv4_dic['mask'], - ipv4_dic['gateway'], - lst_to_str(ipv4_dic['DNS']), - ipv4_dic['address'], - ipv6_dic['prefix'], - ipv6_dic['gateway'], - lst_to_str(ipv6_dic['DNS']), - ipv6_dic['address'], - display_str]) - return position - - def add_service_to_interface_in_model(service, interface_pos, model): - """Append a service to an interface at interface_pos in the given - model. Return None. Modifies the model""" - display_str = service.getName() + " (" + str(len(service.getVulns())) + ")" - model.append(interface_pos, [service.getID(), - service.getName(), - service.getDescription(), - service.getProtocol(), - service.getStatus(), - lst_to_str(service.getPorts()), - service.getVersion(), - "Yes" if service.isOwned() else "No", - "", "", "", "", display_str]) - - interfaces = host.getAllInterfaces() - for interface in interfaces: - interface_position = add_interface_to_host_in_model(interface, - host_position, - model) - services = interface.getAllServices() - for service in services: - add_service_to_interface_in_model(service, interface_position, - model) - - return model - @scrollable(width=250) def create_main_tree_view(self, model): """Return a box containing the main tree (the one showing - Host/Interfaces/Services) as its content. + Host/Services) as its content. """ view = Gtk.TreeView(model) view.set_activate_on_single_click(True) @@ -820,7 +765,7 @@ def create_main_tree_view(self, model): view.expand_all() renderer = Gtk.CellRendererText() - column = Gtk.TreeViewColumn("Host/Interfaces/Services", + column = Gtk.TreeViewColumn("Host/Services", renderer, text=12) view.append_column(column) @@ -843,12 +788,12 @@ def on_main_tree_selection(self, tree_selection): object_info = model[tree_iter] iter_depth = model.iter_depth(tree_iter) - object_type = {0: 'Host', 1: 'Interface', 2: 'Service'}[iter_depth] + object_type = {0: 'Host', 1: 'Service'}[iter_depth] if object_type == 'Host': self.set_vuln_model(self.create_vuln_model(self.host)) - elif object_type == 'Interface' or object_type == 'Service': + elif object_type == 'Service': self.clear(self.specific_info) self.change_label_in_frame(self.specific_info_frame, object_type) prop_names = self.get_properties_names(object_type) @@ -999,38 +944,33 @@ def safe_wrapper(*args, **kwargs): return safe_wrapper object_id = selected_object[0] - if object_type == 'Interface': - # an interface is a direct child of a host - object_ = safely(self.host.getInterface)(object_id) - elif object_type == 'Service': - # a service is a grand-child of a host, so we should look - # for its parent interface and ask her about the child - parent_interface_iter = selected_object.get_parent() - parent_interface_id = parent_interface_iter[0] - parent_interface = safely(self.host.getInterface)(parent_interface_id) - if parent_interface: - object_ = safely(parent_interface.getService)(object_id) - else: - object_ = None + object_ = None + # TODO: fix this code to get services removing interface logic + # if object_type == 'Interface': + # # an interface is a direct child of a host + # object_ = safely(self.host.getInterface)(object_id) + # elif object_type == 'Service': + # # a service is a grand-child of a host, so we should look + # # for its parent interface and ask her about the child + # parent_interface_iter = selected_object.get_parent() + # parent_interface_id = parent_interface_iter[0] + # parent_interface = safely(self.host.getInterface)(parent_interface_id) + # if parent_interface: + # object_ = safely(parent_interface.getService)(object_id) + # else: + # object_ = None return object_ def get_properties_names(self, object_type): """Return a list with the property names for objects of type - Interface, Service, Vulnerability and VulnerabilityWeb (passed as a + Service, Vulnerability and VulnerabilityWeb (passed as a string). """ if object_type == "Host": property_names = ["Name: ", "OS: ", "Owned: ", "Vulnerabilities: "] - elif object_type == "Interface": - property_names = ["Name: ", "Description: ", "MAC: ", - "IPv4 Mask: ", "IPv4 Gateway: ", "IPv4 DNS: ", - "IPv4 Address: ", "IPv6 Prefix: ", - "IPv6 Gateway", "IPv6 DNS: ", - "IPv6 Address: "] - elif object_type == "Service": property_names = ["Name: ", "Description: ", "Protocol: ", "Status: ", "Port: ", "Version: ", "Is Owned?: "] @@ -1148,15 +1088,16 @@ def save(self, button, keeper): elif keeper == "left": n = 1 - # interface needs a special case, 'cause it's the only object - # which resolveConflict() will expect its solution to have a - # dicitionary inside the solution dictionary - if current_conflict_type != "Interface": - solution = {} - for row in self.current_conflict_model: - solution[row[0].lower()] = self.uncook(row[n], row[4]) - else: - solution = self.case_for_interfaces(self.current_conflict_model, n) + # TODO: fix this code to remove interface logic + # # interface needs a special case, 'cause it's the only object + # # which resolveConflict() will expect its solution to have a + # # dicitionary inside the solution dictionary + # if current_conflict_type != "Interface": + # solution = {} + # for row in self.current_conflict_model: + # solution[row[0].lower()] = self.uncook(row[n], row[4]) + # else: + # solution = self.case_for_interfaces(self.current_conflict_model, n) try: guiapi.resolveConflict(self.current_conflict, solution) @@ -1191,32 +1132,33 @@ def save(self, button, keeper): dialog.destroy() self._next_conflict_or_close() - def case_for_interfaces(self, model, n): - """The custom case for the interfaces. Plays a little - with the information in the given model to create a solution acceptable - by resolveConflict. n is the right or left view, should be - either 1 or 2 as integers""" - solution = {} - solution["ipv4"] = {} - solution["ipv6"] = {} - for row in model: - prop_name = row[0].lower() - if prop_name.startswith("ipv4"): - prop_name = prop_name.split(" ")[1] - if not prop_name.startswith("dns"): - solution["ipv4"][prop_name] = self.uncook(row[n], row[4]) - elif prop_name.startswith("dns"): - solution["ipv4"]["DNS"] = self.uncook(row[n], row[4]) - - elif prop_name.startswith("ipv6"): - prop_name = prop_name.split(" ")[1] - if not prop_name.startswith("dns"): - solution["ipv6"][prop_name] = self.uncook(row[n], row[4]) - elif prop_name.startswith("dns"): - solution["ipv6"]["DNS"] = self.uncook(row[n], row[4]) - else: - solution[prop_name] = self.uncook(row[n], row[4]) - return solution + # TODO: refactor or remove this code + # def case_for_interfaces(self, model, n): + # """The custom case for the interfaces. Plays a little + # with the information in the given model to create a solution acceptable + # by resolveConflict. n is the right or left view, should be + # either 1 or 2 as integers""" + # solution = {} + # solution["ipv4"] = {} + # solution["ipv6"] = {} + # for row in model: + # prop_name = row[0].lower() + # if prop_name.startswith("ipv4"): + # prop_name = prop_name.split(" ")[1] + # if not prop_name.startswith("dns"): + # solution["ipv4"][prop_name] = self.uncook(row[n], row[4]) + # elif prop_name.startswith("dns"): + # solution["ipv4"]["DNS"] = self.uncook(row[n], row[4]) + + # elif prop_name.startswith("ipv6"): + # prop_name = prop_name.split(" ")[1] + # if not prop_name.startswith("dns"): + # solution["ipv6"][prop_name] = self.uncook(row[n], row[4]) + # elif prop_name.startswith("dns"): + # solution["ipv6"]["DNS"] = self.uncook(row[n], row[4]) + # else: + # solution[prop_name] = self.uncook(row[n], row[4]) + # return solution def on_quit(self, button): """Exits the window""" @@ -1321,8 +1263,6 @@ def create_conflicts_models(self, conflicts): if conflict_type == "Service": self.fill_service_conflict_model(model, obj1, obj2) - elif conflict_type == "Interface": - self.fill_interface_conflict_model(model, obj1, obj2) elif conflict_type == "Host": self.fill_host_conflict_model(model, obj1, obj2) elif conflict_type == "Vulnerability": @@ -1371,34 +1311,6 @@ def fill_host_conflict_model(self, model, obj1, obj2): model = self.fill_model_from_props_and_attr(model, attr, props) return model - def fill_interface_conflict_model(self, model, obj1, obj2): - """ - Precondition: the model has 5 string columns, obj1 && obj2 are - interfaces - Will get a model and two objects and return a - model with all the appropiate information""" - attr = [] - for obj in [obj1, obj2]: - attr.append((obj.getName(), - obj.getDescription(), - obj.getHostnames(), - obj.getMAC(), - obj.getIPv4Address(), - obj.getIPv4Mask(), - obj.getIPv4Gateway(), - obj.getIPv4DNS(), - obj.getIPv6Address(), - obj.getIPv6Gateway(), - obj.getIPv6DNS(), - obj.isOwned())) - - props = ["Name", "Description", "Hostnames", "MAC", "IPv4 Address", - "IPv4 Mask", "IPv4 Gateway", "IPv4 DNS", "IPv6 Address", - "IPv6 Gateway", "IPv6 DNS", "Owned"] - - model = self.fill_model_from_props_and_attr(model, attr, props) - return model - def fill_vuln_conflict_model(self, model, obj1, obj2): """ Precondition: the model has 5 string columns, obj1 && obj2 are vulns diff --git a/persistence/server/models.py b/persistence/server/models.py index d0808f0a0b4..79a569b36b6 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -19,7 +19,6 @@ from persistence.server.utils import (force_unique, get_hash, get_host_properties, - get_interface_properties, get_service_properties, get_vuln_properties, get_vuln_web_properties, @@ -102,17 +101,16 @@ def _flatten_dictionary(dictionary): def _get_faraday_ready_objects(workspace_name, faraday_ready_object_dictionaries, faraday_object_name): """Takes a workspace name, a faraday object ('hosts', 'vulns', - 'interfaces' or 'services') a row_name (the name of the row where + or 'services') a row_name (the name of the row where the information about the objects live) and an arbitray number of params to customize to request. Return a list of faraday objects: either - Host, Interface, Service, Vuln, VulnWeb, Credential or Command. + Host, Service, Vuln, VulnWeb, Credential or Command. """ object_to_class = {'hosts': Host, 'vulns': Vuln, 'vulns_web': VulnWeb, - 'interfaces': Interface, 'services': Service, 'notes': Note, 'credentials': Credential, @@ -150,10 +148,6 @@ def _get_faraday_ready_services(workspace_name, services_dictionaries): """Return a list of Services created with the information found on services_dictionaries""" return _get_faraday_ready_objects(workspace_name, services_dictionaries, 'services') -def _get_faraday_ready_interfaces(workspace_name, interfaces_dictionaries): - """Return a list of Interfaces created with the information found on interfaces_dictionaries""" - return _get_faraday_ready_objects(workspace_name, interfaces_dictionaries, 'interfaces') - def _get_faraday_ready_credentials(workspace_name, credentials_dictionaries): """Return a list of Credentials created with the information found on credentials_dictionaries""" return _get_faraday_ready_objects(workspace_name, credentials_dictionaries, 'credentials') @@ -224,19 +218,6 @@ def get_web_vuln(workspace_name, vuln_id): """Return the WebVuln of id vuln_id. None if not found.""" return force_unique(get_web_vulns(workspace_name, couchid=vuln_id)) -def get_interfaces(workspace_name, **params): - """Take a workspace name and a arbitrary number of params to customize the - request. - - Return a list of Interfaces objects - """ - interfaces_dictionaries = server.get_interfaces(workspace_name, **params) - return _get_faraday_ready_interfaces(workspace_name, interfaces_dictionaries) - -def get_interface(workspace_name, interface_id): - """Return the Interface of id interface_id. None if not found.""" - return force_unique(get_interfaces(workspace_name, couchid=interface_id)) - def get_services(workspace_name, **params): """Take a workspace name and a arbitrary number of params to customize the request. @@ -300,13 +281,12 @@ def get_object(workspace_name, object_signature, object_id): about 'object_signature' objects matching the query. object_signature must be either 'Host', 'Vulnerability', 'VulnerabilityWeb', - 'Interface', 'Service', 'Cred', 'Note' or 'CommandRunInformation'. + 'Service', 'Cred', 'Note' or 'CommandRunInformation'. Will raise an WrongObjectSignature error if this condition is not met. """ object_to_func = {Host.class_signature: get_host, Vuln.class_signature: get_vuln, VulnWeb.class_signature: get_web_vuln, - Interface.class_signature: get_interface, Service.class_signature: get_service, Credential.class_signature: get_credential, Note.class_signature: get_note, @@ -342,23 +322,6 @@ def update_host(workspace_name, host): host_properties = get_host_properties(host) return server.update_host(workspace_name, **host_properties) -@_ignore_in_changes -def create_interface(workspace_name, interface): - """Take a workspace_name and an interface object and save it to the sever. - Return the server's json response as a dictionary. - """ - interface_properties = get_interface_properties(interface) - return server.create_interface(workspace_name, **interface_properties) - -@_ignore_in_changes -def update_interface(workspace_name, interface): - """Take a workspace_name and an interface object and update it in the sever. - - Return the server's json response as a dictionary. - """ - interface_properties = get_interface_properties(interface) - return server.update_interface(workspace_name, **interface_properties) - @_ignore_in_changes def create_service(workspace_name, service): """Take a workspace_name and a service object and save it to the sever. @@ -464,13 +427,12 @@ def create_object(workspace_name, object_signature, obj): object_signature must match the type of the object. object_signature must be either 'Host', 'Vulnerability', 'VulnerabilityWeb', - 'Interface', 'Service', 'Cred', 'Note' or 'CommandRunInformation'. + 'Service', 'Cred', 'Note' or 'CommandRunInformation'. Will raise an WrongObjectSignature error if this condition is not met. """ object_to_func = {Host.class_signature: create_host, Vuln.class_signature: create_vuln, VulnWeb.class_signature: create_vuln_web, - Interface.class_signature: create_interface, Service.class_signature: create_service, Credential.class_signature: create_credential, Note.class_signature: create_note, @@ -489,14 +451,13 @@ def update_object(workspace_name, object_signature, obj): object_signature must match the type of the object. object_signature must be either 'Host', 'Vulnerability', 'VulnerabilityWeb', - 'Interface', 'Service', 'Cred', 'Note' or 'CommandRunInformation'. + 'Service', 'Cred', 'Note' or 'CommandRunInformation'. Will raise an WrongObjectSignature error if this condition is not met. """ object_to_func = {Host.class_signature: update_host, Vuln.class_signature: update_vuln, VulnWeb.class_signature: update_vuln_web, - Interface.class_signature: update_interface, Service.class_signature: update_service, Credential.class_signature: update_credential, Note.class_signature: update_note, @@ -527,7 +488,7 @@ def get_workspace_summary(workspace_name): return server.get_workspace_summary(workspace_name) def get_workspace_numbers(workspace_name): - """Return a tuple with the number of hosts, interfaces, services and vulns + """Return a tuple with the number of hosts, services and vulns on the workspace of name workspace_name. """ return server.get_workspace_numbers(workspace_name) @@ -542,18 +503,13 @@ def get_services_number(workspace_name, **params): """ return server.get_services_number(workspace_name, **params) -def get_interfaces_number(workspace_name, **params): - """Return the number of interfaces found on the workspace of name workspace_name - """ - return server.get_interfaces_number(workspace_name, **params) - def get_vulns_number(workspace_name, **params): """Return the number of vulns found on the workspace of name workspace_name """ return server.get_vulns_number(workspace_name, **params) # NOTE: the delete functions are actually the same. -# there's no difference between delete_host and delete_interface +# there's no difference between delete_host and # except for their names. # maybe implement some kind of validation in the future? @@ -564,13 +520,6 @@ def delete_host(workspace_name, host_id): """ return server.delete_host(workspace_name, host_id) -@_ignore_in_changes -def delete_interface(workspace_name, interface_id): - """Delete the interface of id interface_id on workspace workspace_name. - Return the json response from the server. - """ - return server.delete_interface(workspace_name, interface_id) - @_ignore_in_changes def delete_service(workspace_name, service_id): """Delete the service of id service_id on workspace workspace_name. @@ -617,13 +566,12 @@ def delete_object(workspace_name, object_signature, obj_id): """Given a workspace name, an object_signature as string and an object id. object_signature must be either 'Host', 'Vulnerability', 'VulnerabilityWeb', - 'Interface', 'Service', 'Cred', 'Note' or 'CommandRunInformation'. + 'Service', 'Cred', 'Note' or 'CommandRunInformation'. Will raise an WrongObjectSignature error if this condition is not met. """ object_to_func = {Host.class_signature: delete_host, Vuln.class_signature: delete_vuln, VulnWeb.class_signature: delete_vuln_web, - Interface.class_signature: delete_interface, Service.class_signature: delete_service, Credential.class_signature: delete_credential, Note.class_signature: delete_note, @@ -809,147 +757,147 @@ def getID(self): return self.id def getDefaultGateway(self): return self.default_gateway def getVulns(self): return get_all_vulns(self._workspace_name, hostid=self._server_id) - def getInterface(self, interface_couch_id): - service = get_interfaces(self._workspace_name, couchid=interface_couch_id) - return service[0] - def getAllInterfaces(self): - return get_interfaces(self._workspace_name, host=self._server_id) + # def getInterface(self, interface_couch_id): + # service = get_interfaces(self._workspace_name, couchid=interface_couch_id) + # return service[0] + # def getAllInterfaces(self): + # return get_interfaces(self._workspace_name, host=self._server_id) def getServices(self): return get_services(self._workspace_name, hostid=self._server_id) -class Interface(ModelBase): - """A simple Interface class. Should implement all the methods of the - Interface object in Model.Host - Any method here more than a couple of lines long probably represent - a search the server is missing. - """ - class_signature = 'Interface' - - def __init__(self, interface, workspace_name): - ModelBase.__init__(self, interface, workspace_name) - self.hostnames = interface.get('hostnames', []) - - # NOTE. i don't know why this is like this - # probably a remnant of the old faraday style classes - try: - self.ipv4 = interface['ipv4'] - self.ipv6 = interface['ipv6'] - except KeyError: - self.ipv4 = {'address': interface['ipv4_address'], - 'gateway': interface['ipv4_gateway'], - 'mask': interface['ipv4_mask'], - 'DNS': interface['ipv4_dns']} - self.ipv6 = {'address': interface['ipv6_address'], - 'gateway': interface['ipv6_gateway'], - 'prefix': interface['ipv6_prefix'], - 'DNS': interface['ipv6_dns']} - self.mac = interface.get('mac') - self.network_segment = interface.get('network_segment') - self.ports = interface.get('ports') - - self.amount_ports_opened = 0 - self.amount_ports_closed = 0 - self.amount_ports_filtered = 0 - - def setID(self, parent_id): - try: - ipv4_address = self.ipv4_address - ipv6_address = self.ipv6_address - except AttributeError: - ipv4_address = self.ipv4['address'] - ipv6_address = self.ipv6['address'] - - ModelBase.setID(self, parent_id, self.network_segment, ipv4_address, ipv6_address) - - @staticmethod - def publicattrsrefs(): - publicattrs = dict(ModelBase.publicattrsrefs(), **{ - 'MAC Address' : 'mac', - 'IPV4 Settings' : 'ipv4', - 'IPV6 Settings' : 'ipv6', - 'Network Segment' : 'network_segment', - 'Hostnames' : 'hostnames' - }) - return publicattrs - - def tieBreakable(self, property_key): - """ - Return true if we can auto resolve this conflict. - """ - if property_key in ["hostnames"]: - return True - return False - - def tieBreak(self, key, prop1, prop2): - """ - Return the 'choosen one' - Return a tuple with prop1, prop2 if we cant resolve conflict. - """ - if key == "hostnames": - prop1.extend(prop2) - # Remove duplicated with set... - return list(set(prop1)) - - return (prop1, prop2) - - def updateAttributes(self, name=None, description=None, hostnames=None, mac=None, ipv4=None, ipv6=None, - network_segment=None, amount_ports_opened=None, amount_ports_closed=None, - amount_ports_filtered=None, owned=None): - - if name is not None: - self.name = name - if description is not None: - self.description = description - if hostnames is not None: - self.hostnames = hostnames - if mac is not None: - self.mac = mac - if ipv4 is not None: - self.ipv4 = ipv4 - if ipv6 is not None: - self.ipv6 = ipv6 - if network_segment is not None: - self.network_segment = network_segment - if amount_ports_opened is not None: - self.setPortsOpened(amount_ports_opened) - if amount_ports_closed is not None: - self.setPortsClosed(amount_ports_closed) - if amount_ports_filtered is not None: - self.setPortsFiltered(amount_ports_filtered) - if owned is not None: - self.owned = owned - - def setPortsOpened(self, ports_opened): - self.amount_ports_opened = ports_opened - - def setPortsClosed(self, ports_closed): - self.amount_ports_closed = ports_closed - - def setPortsFiltered(self, ports_filtered): - self.amount_ports_filtered = ports_filtered - - def __str__(self): return "{0}".format(self.name) - def getID(self): return self.id - def getHostnames(self): return self.hostnames - def getIPv4(self): return self.ipv4 - def getIPv6(self): return self.ipv6 - def getIPv4Address(self): return self.ipv4['address'] - def getIPv4Mask(self): return self.ipv4['mask'] - def getIPv4Gateway(self): return self.ipv4['gateway'] - def getIPv4DNS(self): return self.ipv4['DNS'] - def getIPv6Address(self): return self.ipv6['address'] - def getIPv6Gateway(self): return self.ipv6['gateway'] - def getIPv6DNS(self): return self.ipv6['DNS'] - def getMAC(self): return self.mac - def getNetworkSegment(self): return self.network_segment - - def getService(self, service_couch_id): - return get_service(self._workspace_name, service_couch_id) - def getAllServices(self): - return get_services(self._workspace_name, interface=self._server_id) - def getVulns(self): - return get_all_vulns(self._workspace_name, interfaceid=self._server_id) +# class Interface(ModelBase): +# """A simple Interface class. Should implement all the methods of the +# Interface object in Model.Host +# Any method here more than a couple of lines long probably represent +# a search the server is missing. +# """ +# class_signature = 'Interface' + +# def __init__(self, interface, workspace_name): +# ModelBase.__init__(self, interface, workspace_name) +# self.hostnames = interface.get('hostnames', []) + +# # NOTE. i don't know why this is like this +# # probably a remnant of the old faraday style classes +# try: +# self.ipv4 = interface['ipv4'] +# self.ipv6 = interface['ipv6'] +# except KeyError: +# self.ipv4 = {'address': interface['ipv4_address'], +# 'gateway': interface['ipv4_gateway'], +# 'mask': interface['ipv4_mask'], +# 'DNS': interface['ipv4_dns']} +# self.ipv6 = {'address': interface['ipv6_address'], +# 'gateway': interface['ipv6_gateway'], +# 'prefix': interface['ipv6_prefix'], +# 'DNS': interface['ipv6_dns']} +# self.mac = interface.get('mac') +# self.network_segment = interface.get('network_segment') +# self.ports = interface.get('ports') + +# self.amount_ports_opened = 0 +# self.amount_ports_closed = 0 +# self.amount_ports_filtered = 0 + +# def setID(self, parent_id): +# try: +# ipv4_address = self.ipv4_address +# ipv6_address = self.ipv6_address +# except AttributeError: +# ipv4_address = self.ipv4['address'] +# ipv6_address = self.ipv6['address'] + +# ModelBase.setID(self, parent_id, self.network_segment, ipv4_address, ipv6_address) + +# @staticmethod +# def publicattrsrefs(): +# publicattrs = dict(ModelBase.publicattrsrefs(), **{ +# 'MAC Address' : 'mac', +# 'IPV4 Settings' : 'ipv4', +# 'IPV6 Settings' : 'ipv6', +# 'Network Segment' : 'network_segment', +# 'Hostnames' : 'hostnames' +# }) +# return publicattrs + +# def tieBreakable(self, property_key): +# """ +# Return true if we can auto resolve this conflict. +# """ +# if property_key in ["hostnames"]: +# return True +# return False + +# def tieBreak(self, key, prop1, prop2): +# """ +# Return the 'choosen one' +# Return a tuple with prop1, prop2 if we cant resolve conflict. +# """ +# if key == "hostnames": +# prop1.extend(prop2) +# # Remove duplicated with set... +# return list(set(prop1)) + +# return (prop1, prop2) + +# def updateAttributes(self, name=None, description=None, hostnames=None, mac=None, ipv4=None, ipv6=None, +# network_segment=None, amount_ports_opened=None, amount_ports_closed=None, +# amount_ports_filtered=None, owned=None): + +# if name is not None: +# self.name = name +# if description is not None: +# self.description = description +# if hostnames is not None: +# self.hostnames = hostnames +# if mac is not None: +# self.mac = mac +# if ipv4 is not None: +# self.ipv4 = ipv4 +# if ipv6 is not None: +# self.ipv6 = ipv6 +# if network_segment is not None: +# self.network_segment = network_segment +# if amount_ports_opened is not None: +# self.setPortsOpened(amount_ports_opened) +# if amount_ports_closed is not None: +# self.setPortsClosed(amount_ports_closed) +# if amount_ports_filtered is not None: +# self.setPortsFiltered(amount_ports_filtered) +# if owned is not None: +# self.owned = owned + +# def setPortsOpened(self, ports_opened): +# self.amount_ports_opened = ports_opened + +# def setPortsClosed(self, ports_closed): +# self.amount_ports_closed = ports_closed + +# def setPortsFiltered(self, ports_filtered): +# self.amount_ports_filtered = ports_filtered + +# def __str__(self): return "{0}".format(self.name) +# def getID(self): return self.id +# def getHostnames(self): return self.hostnames +# def getIPv4(self): return self.ipv4 +# def getIPv6(self): return self.ipv6 +# def getIPv4Address(self): return self.ipv4['address'] +# def getIPv4Mask(self): return self.ipv4['mask'] +# def getIPv4Gateway(self): return self.ipv4['gateway'] +# def getIPv4DNS(self): return self.ipv4['DNS'] +# def getIPv6Address(self): return self.ipv6['address'] +# def getIPv6Gateway(self): return self.ipv6['gateway'] +# def getIPv6DNS(self): return self.ipv6['DNS'] +# def getMAC(self): return self.mac +# def getNetworkSegment(self): return self.network_segment + +# def getService(self, service_couch_id): +# return get_service(self._workspace_name, service_couch_id) +# def getAllServices(self): +# return get_services(self._workspace_name, interface=self._server_id) +# def getVulns(self): +# return get_all_vulns(self._workspace_name, interfaceid=self._server_id) class Service(ModelBase): diff --git a/server/api/modules/interfaces.py b/server/api/modules/interfaces.py deleted file mode 100644 index 9e6039b5b24..00000000000 --- a/server/api/modules/interfaces.py +++ /dev/null @@ -1,28 +0,0 @@ -# Faraday Penetration Test IDE -# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) -# See the file 'doc/LICENSE' for the license information - -import flask -from flask import Blueprint - -from server.utils.logger import get_logger -from server.utils.web import ( - gzipped, - validate_workspace, -) - -from server.dao.interface import InterfaceDAO - -interfaces_api = Blueprint('interface_api', __name__) - - -@gzipped -@interfaces_api.route('/ws//interfaces', methods=['GET']) -def list_interfaces(workspace=None): - validate_workspace(workspace) - get_logger(__name__).debug("Request parameters: {!r}".format(flask.request.args)) - - dao = InterfaceDAO(workspace) - result = dao.list(interface_filter=flask.request.args) - - return flask.jsonify(result) diff --git a/server/api/modules/notes.py b/server/api/modules/notes.py deleted file mode 100644 index f8d552fcf62..00000000000 --- a/server/api/modules/notes.py +++ /dev/null @@ -1,45 +0,0 @@ -# Faraday Penetration Test IDE -# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) -# See the file 'doc/LICENSE' for the license information - -from flask import request, jsonify, abort -from flask import Blueprint - -from server.utils.logger import get_logger -from server.utils.web import gzipped, validate_workspace, filter_request_args - -from server.dao.note import NoteDAO - -notes_api = Blueprint('notes_api', __name__) - - -@gzipped -@notes_api.route('/ws//notes', methods=['GET']) -def list_notes(workspace=None): - - validate_workspace(workspace) - get_logger(__name__).debug( - "Request parameters: {!r}".format(request.args)) - - note_filter = filter_request_args() - - dao = NoteDAO(workspace) - - result = dao.list(note_filter=note_filter) - - return jsonify(result) - - -@notes_api.route('/ws//notes/count', methods=['GET']) -@gzipped -def count_notes(workspace=None): - validate_workspace(workspace) - get_logger(__name__).debug("Request parameters: {!r}"\ - .format(request.args)) - - services_dao = NoteDAO(workspace) - result = services_dao.count() - if result is None: - abort(400) - - return jsonify(result) diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index 476f06d147f..2befd1cde85 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -10,8 +10,6 @@ from server.dao.host import HostDAO from server.dao.vuln import VulnerabilityDAO from server.dao.service import ServiceDAO -from server.dao.interface import InterfaceDAO -from server.dao.note import NoteDAO from server.dao.workspace import WorkspaceDAO from server.utils.logger import get_logger from server.utils.web import ( @@ -65,8 +63,6 @@ def workspace_summary(workspace=None): services_count = ServiceDAO(workspace).count() vuln_count = VulnerabilityDAO(workspace).count(vuln_filter=flask.request.args) host_count = HostDAO(workspace).count() - iface_count = InterfaceDAO(workspace).count() - note_count = NoteDAO(workspace).count() response = { 'stats': { @@ -75,8 +71,6 @@ def workspace_summary(workspace=None): 'web_vulns': vuln_count.get('web_vuln_count', 0), 'std_vulns': vuln_count.get('vuln_count', 0), 'hosts': host_count.get('total_count', 0), - 'interfaces': iface_count.get('total_count', 0), - 'notes': note_count.get('total_count', 0), } } diff --git a/server/app.py b/server/app.py index 432ccda75e4..34cbc7a5467 100644 --- a/server/app.py +++ b/server/app.py @@ -59,24 +59,20 @@ def create_app(db_connection_string=None, testing=None): from server.api.modules.workspaces import workspace_api from server.api.modules.doc import doc_api - from server.api.modules.interfaces import interfaces_api from server.api.modules.vuln_csv import vuln_csv_api from server.api.modules.hosts import host_api from server.api.modules.commandsrun import commandsrun_api from server.api.modules.services import services_api from server.api.modules.credentials import credentials_api - from server.api.modules.notes import notes_api from server.api.modules.session import session_api from server.modules.info import info_api app.register_blueprint(workspace_api) app.register_blueprint(doc_api) - app.register_blueprint(interfaces_api) app.register_blueprint(vuln_csv_api) app.register_blueprint(host_api) app.register_blueprint(commandsrun_api) app.register_blueprint(services_api) app.register_blueprint(credentials_api) - app.register_blueprint(notes_api) app.register_blueprint(info_api) app.register_blueprint(session_api) diff --git a/server/dao/host.py b/server/dao/host.py index cc56eb47487..75f48a776d9 100644 --- a/server/dao/host.py +++ b/server/dao/host.py @@ -15,7 +15,6 @@ from sqlalchemy.sql import func from server.models import ( Host, - Interface, Service, Vulnerability, EntityMetadata, @@ -29,7 +28,7 @@ class HostDAO(FaradayDAO): COLUMNS_MAP = { "couchid": [EntityMetadata.couchdb_id], - "name": [Host.name], + "ip": [Host.ip], "service": [Service.name], "services": ["open_services_count"], "vulns": ["vuln_count"], @@ -55,21 +54,19 @@ def list(self, search=None, page=0, page_size=0, order_by=None, order_dir=None, def __query_database(self, search=None, page=0, page_size=0, order_by=None, order_dir=None, host_filter={}): host_bundle = Bundle( - "host", Host.id, Host.name, Host.os, Host.description, Host.owned, + "host", Host.id, Host.ip, Host.os, Host.description, Host.owned, Host.default_gateway_ip, Host.default_gateway_mac, EntityMetadata.couchdb_id, EntityMetadata.revision, EntityMetadata.update_time, EntityMetadata.update_user, EntityMetadata.update_action, EntityMetadata.creator, EntityMetadata.create_time, EntityMetadata.update_controller_action, EntityMetadata.owner, EntityMetadata.command_id, - func.group_concat(distinct(Interface.id)).label("interfaces"), func.count(distinct(Vulnerability.id)).label("vuln_count"), func.count(distinct(Service.id)).label("open_services_count"), func.count(distinct(Credential.id)).label("credentials_count")) query = self._session.query(host_bundle)\ .outerjoin(EntityMetadata, EntityMetadata.id == Host.entity_metadata_id)\ - .outerjoin(Interface, Host.id == Interface.host_id)\ .outerjoin(Vulnerability, Host.id == Vulnerability.host_id)\ .outerjoin(Service, (Host.id == Service.host_id) & (Service.status.in_(("open", "running", "opened"))))\ .outerjoin(Credential, (Credential.host_id == Host.id) & Credential.service_id == None)\ @@ -97,7 +94,7 @@ def __get_host_data(self, host): "value": { "_id": host.couchdb_id, "_rev": host.revision, - "name": host.name, + "ip": host.ip, "os": host.os, "owned": host.owned, "owner": host.owner, @@ -115,7 +112,6 @@ def __get_host_data(self, host): }, "vulns": host.vuln_count, "services": host.open_services_count, - "interfaces": map(int, host.interfaces.split(",")) if host.interfaces else [], "credentials": host.credentials_count }} @@ -129,7 +125,7 @@ def count(self, group_by=None): # Otherwise return the amount of services grouped by the field specified # Strict restriction is applied for this entity - if group_by not in ["name", "os"]: + if group_by not in ["ip", "os"]: return None col = HostDAO.COLUMNS_MAP.get(group_by)[0] diff --git a/server/dao/interface.py b/server/dao/interface.py deleted file mode 100644 index 0bd6e918935..00000000000 --- a/server/dao/interface.py +++ /dev/null @@ -1,82 +0,0 @@ -# Faraday Penetration Test IDE -# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) -# See the file 'doc/LICENSE' for the license information - -from server.utils.database import apply_search_filter -from server.dao.base import FaradayDAO -from server.models import Interface, EntityMetadata -from sqlalchemy.orm.query import Bundle -from sqlalchemy.sql import func - -class InterfaceDAO(FaradayDAO): - MAPPED_ENTITY = Interface - COLUMNS_MAP = { - "host": [Interface.host_id], - "couchid": [EntityMetadata.couchdb_id], - } - STRICT_FILTERING = ["host", "couchid"] - - def count(self): - total_count = self._session.query(func.count(Interface.id)).scalar() - return { 'total_count': total_count } - - def list(self, interface_filter={}): - interface_bundle = Bundle('interface', - Interface.id, Interface.name, Interface.description, Interface.mac, - Interface.owned, Interface.hostnames, Interface.network_segment, Interface.ipv4_address, - Interface.ipv4_gateway, Interface.ipv4_dns, Interface.ipv4_mask, Interface.ipv6_address, - Interface.ipv6_gateway, Interface.ipv6_dns, Interface.ipv6_prefix, Interface.ports_filtered, - Interface.ports_opened, Interface.ports_closed, Interface.host_id, EntityMetadata.couchdb_id,\ - EntityMetadata.revision, EntityMetadata.update_time, EntityMetadata.update_user,\ - EntityMetadata.update_action, EntityMetadata.creator, EntityMetadata.create_time,\ - EntityMetadata.update_controller_action, EntityMetadata.owner, EntityMetadata.command_id) - - query = self._session.query(interface_bundle).\ - outerjoin(EntityMetadata, EntityMetadata.id == Interface.entity_metadata_id) - - query = apply_search_filter(query, self.COLUMNS_MAP, None, interface_filter, self.STRICT_FILTERING) - - raw_interfaces = query.all() - interfaces = [self.__get_interface_data(r.interface) for r in raw_interfaces] - result = {'interfaces': interfaces} - - return result - - def __get_interface_data(self, interface): - return { - 'id': interface.couchdb_id, - 'key': interface.couchdb_id, - '_id': interface.id, - 'value': { - '_id': interface.couchdb_id, - '_rev': interface.revision, - 'name': interface.name, - 'description': interface.description, - 'mac': interface.mac, - 'owned': interface.owned, - 'owner': interface.owner, - 'hostnames': interface.hostnames.split(',') if interface.hostnames else [], - 'network_segment': interface.network_segment, - 'ipv4': {'address': interface.ipv4_address, - 'gateway': interface.ipv4_gateway, - 'DNS': interface.ipv4_dns.split(',') if interface.ipv4_dns else [], - 'mask': interface.ipv4_mask}, - 'ipv6': {'address': interface.ipv6_address, - 'gateway': interface.ipv6_gateway, - 'DNS': interface.ipv6_dns.split(',') if interface.ipv6_dns else [], - 'prefix': interface.ipv6_prefix}, - 'ports': {'filtered': interface.ports_filtered, - 'opened': interface.ports_opened, - 'closed': interface.ports_closed}, - 'metadata': { - 'update_time': interface.update_time, - 'update_user': interface.update_user, - 'update_action': interface.update_action, - 'creator': interface.creator, - 'create_time': interface.create_time, - 'update_controller_action': interface.update_controller_action, - 'owner': interface.owner, - 'command_id': interface.command_id - }, - 'host_id': interface.host_id} - } diff --git a/server/dao/note.py b/server/dao/note.py deleted file mode 100644 index ba35a62006e..00000000000 --- a/server/dao/note.py +++ /dev/null @@ -1,76 +0,0 @@ -# Faraday Penetration Test IDE -# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) -# See the file 'doc/LICENSE' for the license information - -from sqlalchemy.sql import func -from sqlalchemy.orm.query import Bundle - -from server.dao.base import FaradayDAO -from server.models import Note, EntityMetadata -from server.utils.database import apply_search_filter - -class NoteDAO(FaradayDAO): - MAPPED_ENTITY = Note - COLUMNS_MAP = { - 'couchid': [EntityMetadata.couchdb_id], - 'name': [Note.name], - 'text': [Note.text], - 'description': [Note.description], - } - STRICT_FILTERING = ["couchid"] - - def list(self, search=None, note_filter={}): - results = self.__query_database(search, note_filter) - - rows = [ self.__get_note_data(result.note) for result in results ] - - result = { - 'rows': rows - } - - return result - - def __query_database(self, search=None, note_filter={}): - note_bundle = Bundle('note', Note.id, Note.name, Note.text, Note.description, Note.owned, EntityMetadata.couchdb_id,\ - EntityMetadata.revision, EntityMetadata.update_time, EntityMetadata.update_user,\ - EntityMetadata.update_action, EntityMetadata.creator, EntityMetadata.create_time,\ - EntityMetadata.update_controller_action, EntityMetadata.owner, EntityMetadata.command_id) - - query = self._session.query(note_bundle)\ - .outerjoin(EntityMetadata, EntityMetadata.id == Note.entity_metadata_id) - - # Apply filtering options to the query - query = apply_search_filter(query, self.COLUMNS_MAP, search, note_filter, self.STRICT_FILTERING) - - results = query.all() - - return results - - def __get_note_data(self, note): - return { - 'id': note.couchdb_id, - 'key': note.couchdb_id, - '_id': note.id, - 'value': { - '_id': note.couchdb_id, - 'name': note.name, - 'text': note.text, - 'description': note.description, - 'owned': note.owned, - 'owner': note.owner, - 'metadata': { - 'update_time': note.update_time, - 'update_user': note.update_user, - 'update_action': note.update_action, - 'creator': note.creator, - 'create_time': note.create_time, - 'update_controller_action': note.update_controller_action, - 'owner': note.owner, - 'command_id': note.command_id - }, - 'couchid': note.couchdb_id }} - - def count(self): - total_count = self._session.query(func.count(Note.id)).scalar() - return { 'total_count': total_count } - diff --git a/server/dao/service.py b/server/dao/service.py index 489927f851d..c98fc991538 100644 --- a/server/dao/service.py +++ b/server/dao/service.py @@ -9,7 +9,6 @@ from server.dao.base import FaradayDAO from server.models import ( Host, - Interface, Service, EntityMetadata, Vulnerability, @@ -21,7 +20,6 @@ class ServiceDAO(FaradayDAO): MAPPED_ENTITY = Service COLUMNS_MAP = { - "interface": [Service.interface_id], "couchid": [EntityMetadata.couchdb_id], "command_id": [EntityMetadata.command_id], 'id': [Service.id], @@ -34,13 +32,12 @@ class ServiceDAO(FaradayDAO): "hostIdCouchdb": [] } - STRICT_FILTERING = ["couchid", "interface", 'id', 'hostid'] + STRICT_FILTERING = ["couchid", 'id', 'hostid'] def list(self, service_filter={}): service_bundle = Bundle('service', Service.id, Service.name, Service.description, Service.protocol, - Service.status, Service.ports, Service.version, Service.owned, - Service.interface_id, + Service.status, Service.port, Service.version, Service.owned, func.count(distinct(Vulnerability.id)).label('vuln_count'), EntityMetadata.couchdb_id,\ EntityMetadata.revision, EntityMetadata.update_time, EntityMetadata.update_user,\ EntityMetadata.update_action, EntityMetadata.creator, EntityMetadata.create_time,\ @@ -51,9 +48,7 @@ def list(self, service_filter={}): group_by(Service.id).\ outerjoin(EntityMetadata, EntityMetadata.id == Service.entity_metadata_id).\ outerjoin(Vulnerability, Service.id == Vulnerability.service_id).group_by(Service.id).\ - outerjoin(Interface, Interface.id == Service.interface_id).\ - outerjoin(Credential, (Credential.service_id == Service.id) and (Credential.host_id == None)).\ - outerjoin(Host, Host.id == Interface.host_id) + outerjoin(Credential, (Credential.service_id == Service.id) and (Credential.host_id == None)) query = query.filter(Service.workspace == self.workspace) query = apply_search_filter(query, self.COLUMNS_MAP, None, service_filter, self.STRICT_FILTERING) @@ -90,7 +85,7 @@ def __get_service_data(self, service): }, 'protocol': service.protocol, 'status': service.status, - 'ports': [int(i) for i in service.ports.split(',') if service.ports], + 'port': [int(i) for i in service.port.split(',') if service.port], 'version': service.version, 'owned': service.owned, 'owner': service.owner, diff --git a/server/dao/vuln.py b/server/dao/vuln.py index 5f512fbae51..8d12e2a70b6 100644 --- a/server/dao/vuln.py +++ b/server/dao/vuln.py @@ -17,7 +17,6 @@ from sqlalchemy.orm.query import Bundle from server.models import ( Host, - Interface, Service, Vulnerability, EntityMetadata @@ -33,40 +32,27 @@ class VulnerabilityDAO(FaradayDAO): "confirmed": [Vulnerability.confirmed], "name": [Vulnerability.name], "severity": [Vulnerability.severity], - "service": [Service.ports, Service.protocol, Service.name], - "target": [Host.name], + "service": [Service.port, Service.protocol, Service.name], + "target": [Host.ip], "desc": [Vulnerability.description], "resolution": [Vulnerability.resolution], "data": [Vulnerability.data], "owner": [EntityMetadata.owner], - "owned": [Vulnerability.owned], - "easeofresolution": [Vulnerability.easeofresolution], + "ease_of_resolution": [Vulnerability.ease_of_resolution], "type": [EntityMetadata.document_type], "status": [Vulnerability.status], - "website": [Vulnerability.website], - "path": [Vulnerability.path], - "request": [Vulnerability.request], - "refs": [Vulnerability.refs], "tags": [], "evidence": [], - "hostnames": [Interface.hostnames], "impact": [], - "method": [Vulnerability.method], - "params": [Vulnerability.params], - "pname": [Vulnerability.pname], - "query": [Vulnerability.query], - "response": [Vulnerability.response], "hostid": [Host.id], "serviceid": [Service.id], - "interfaceid": [Interface.id], "web": [], "issuetracker": [], "creator": [EntityMetadata.creator], - "command_id": [EntityMetadata.command_id], - "policyviolations": [Vulnerability.policyviolations] + "command_id": [EntityMetadata.command_id] } - STRICT_FILTERING = ["type", "service", "couchid", "hostid", "serviceid", 'interfaceid', 'id', 'status', 'command_id'] + STRICT_FILTERING = ["type", "service", "couchid", "hostid", "serviceid", 'id', 'status', 'command_id'] def list(self, search=None, page=0, page_size=0, order_by=None, order_dir=None, vuln_filter={}): results, count = self.__query_database(search, page, page_size, order_by, order_dir, vuln_filter) @@ -85,25 +71,22 @@ def __query_database(self, search=None, page=0, page_size=0, order_by=None, orde # times from large workspaces almost 2x. vuln_bundle = Bundle('vuln', Vulnerability.id.label('server_id'), Vulnerability.name.label('v_name'),\ Vulnerability.confirmed, Vulnerability.data,\ - Vulnerability.description, Vulnerability.easeofresolution, Vulnerability.impact_accountability,\ + Vulnerability.description, Vulnerability.ease_of_resolution, Vulnerability.impact_accountability,\ Vulnerability.impact_availability, Vulnerability.impact_confidentiality, Vulnerability.impact_integrity,\ - Vulnerability.refs, Vulnerability.resolution, Vulnerability.severity, Vulnerability.owned, Vulnerability.status,\ - Vulnerability.website, Vulnerability.path, Vulnerability.request, Vulnerability.response,\ - Vulnerability.method, Vulnerability.params, Vulnerability.pname, Vulnerability.query,\ + Vulnerability.resolution, Vulnerability.severity, Vulnerability.status,\ EntityMetadata.couchdb_id, EntityMetadata.revision, EntityMetadata.create_time, EntityMetadata.creator,\ EntityMetadata.owner, EntityMetadata.update_action, EntityMetadata.update_controller_action,\ EntityMetadata.update_time, EntityMetadata.update_user, EntityMetadata.document_type, EntityMetadata.command_id, \ - Vulnerability.attachments, Vulnerability.policyviolations) - service_bundle = Bundle('service', Service.name.label('s_name'), Service.ports, Service.protocol, Service.id) - host_bundle = Bundle('host', Host.name) + Vulnerability.attachments) + service_bundle = Bundle('service', Service.name.label('s_name'), Service.port, Service.protocol, Service.id) + host_bundle = Bundle('host', Host.ip) # IMPORTANT: OUTER JOINS on those tables is IMPERATIVE. Changing them could result in loss of # data. For example, on vulnerabilities not associated with any service and instead to its host # directly. query = self._session.query(vuln_bundle, service_bundle, - host_bundle, - func.group_concat(Interface.hostnames))\ + host_bundle)\ .group_by(Vulnerability.id)\ .outerjoin(EntityMetadata, EntityMetadata.id == Vulnerability.entity_metadata_id)\ .outerjoin(Service, Service.id == Vulnerability.service_id)\ @@ -162,7 +145,7 @@ def get_parent_id(couchdb_id): 'data': vuln.data, 'desc': vuln.description, 'description': vuln.description, - 'easeofresolution': vuln.easeofresolution, + 'ease_of_resolution': vuln.ease_of_resolution, 'impact': { 'accountability': vuln.impact_accountability, 'availability': vuln.impact_availability, @@ -186,24 +169,14 @@ def get_parent_id(couchdb_id): 'owned': vuln.owned, 'owner': vuln.owner, 'parent': get_parent_id(vuln.couchdb_id), - 'policyviolations': json.loads(vuln.policyviolations), - 'refs': json.loads(vuln.refs), 'status': vuln.status, - 'website': vuln.website, - 'path': vuln.path, - 'request': vuln.request, - 'response': vuln.response, - 'method': vuln.method, - 'params': vuln.params, - 'pname': vuln.pname, - 'query': vuln.query, 'resolution': vuln.resolution, 'severity': vuln.severity, 'tags': [], 'type': vuln.document_type, - 'target': host.name, + 'target': Host.ip, 'hostnames': hostnames.split(',') if hostnames else '', - 'service': "(%s/%s) %s" % (service.ports, service.protocol, service.s_name) if service.ports else '' + 'service': "(%s/%s) %s" % (service.port, service.protocol, service.s_name) if service.port else '' }} def count(self, group_by=None, search=None, vuln_filter={}): diff --git a/server/dao/workspace.py b/server/dao/workspace.py index 52a6bd7fa74..d373e736fd2 100644 --- a/server/dao/workspace.py +++ b/server/dao/workspace.py @@ -19,22 +19,16 @@ class WorkspaceDAO(object): MAPPED_ENTITY = Workspace COLUMNS_MAP = { - "create_date": [Workspace.create_date], - "creator": [Workspace.creator], + "active": [Workspace.active], "customer": [Workspace.customer], "description": [Workspace.description], - "disabled": [Workspace.disabled], "end_date": [Workspace.end_date], "name": [Workspace.name], "public": [Workspace.public], "scope": [Workspace.scope], - "start_date": [Workspace.start_date], - "update_date": [Workspace.update_date] + "start_date": [Workspace.start_date] } - STRICT_FILTERING = ["name", "creator", "customer", "disabled", - "public", "update_date"] - def __init__(self): self._logger = logger.get_logger(self) self._session = db.session @@ -72,18 +66,18 @@ def __query_database(self, search=None, page=0, page_size=0, order_by=None, order_dir=None, workspace_filter={}): workspace_bundle = Bundle('workspace', - Workspace.create_date, - Workspace.creator, + Workspace.active, Workspace.customer, Workspace.description, - Workspace.disabled, Workspace.end_date, Workspace.name, Workspace.public, Workspace.scope, Workspace.start_date, - Workspace.update_date) + ) + STRICT_FILTERING = ["name", "customer", "disabled", + "public", "update_date"] query = self._session.query(workspace_bundle) query = apply_search_filter(query, self.COLUMNS_MAP, search, @@ -100,15 +94,12 @@ def __query_database(self, search=None, page=0, page_size=0, def __get_workspace_data(self, workspace): return { - "create_date": workspace.create_date, - "creator": workspace.creator, + "active": workspace.active, "customer": workspace.customer, "description": workspace.description, - "disabled": workspace.disabled, "end_date": workspace.end_date, "name": workspace.name, "public": workspace.public, "scope": workspace.scope, - "start_date": workspace.start_date, - "update_date": workspace.update_date + "start_date": workspace.start_date } diff --git a/server/importer.py b/server/importer.py index 587552863ad..56c95909c5c 100644 --- a/server/importer.py +++ b/server/importer.py @@ -16,9 +16,7 @@ db, EntityMetadata, Host, - Interface, Service, - Note, Command, Workspace, Vulnerability @@ -104,43 +102,6 @@ def __get_default_gateway(self, document): return u'', u'' -class InterfaceImporter(object): - DOC_TYPE = 'Interface' - - @classmethod - def update_from_document(cls, document): - interface = Interface() - interface.name = document.get('name') - interface.description = document.get('description') - interface.mac = document.get('mac') - interface.owned = document.get('owned', False) - interface.hostnames = u','.join(document.get('hostnames') or []) - interface.network_segment = document.get('network_segment') - interface.ipv4_address = document.get('ipv4').get('address') - interface.ipv4_gateway = document.get('ipv4').get('gateway') - interface.ipv4_dns = u','.join(document.get('ipv4').get('DNS')) - interface.ipv4_mask = document.get('ipv4').get('mask') - interface.ipv6_address = document.get('ipv6').get('address') - interface.ipv6_gateway = document.get('ipv6').get('gateway') - interface.ipv6_dns = u','.join(document.get('ipv6').get('DNS')) - interface.ipv6_prefix = str(document.get('ipv6').get('prefix')) - interface.ports_filtered = document.get('ports', {}).get('filtered') - interface.ports_opened = document.get('ports', {}).get('opened') - interface.ports_closed = document.get('ports', {}).get('closed') - return interface - - def add_relationships_from_dict(self, entity, entities): - host_id = '.'.join(entity.entity_metadata.couchdb_id.split('.')[:-1]) - if host_id not in entities: - raise EntityNotFound(host_id) - entity.host = entities[host_id] - - def add_relationships_from_db(self, entity, session): - host_id = '.'.join(entity.entity_metadata.couchdb_id.split('.')[:-1]) - query = session.query(Host).join(EntityMetadata).filter(EntityMetadata.couchdb_id == host_id) - entity.host = query.one() - - class ServiceImporter(object): DOC_TYPE = 'Service' @@ -169,21 +130,12 @@ def add_relationships_from_dict(self, entity, entities): raise EntityNotFound(host_id) entity.host = entities[host_id] - interface_id = '.'.join(couchdb_id.split('.')[:-1]) - if interface_id not in entities: - raise EntityNotFound(interface_id) - entity.interface = entities[interface_id] - def add_relationships_from_db(self, entity, session): couchdb_id = entity.entity_metadata.couchdb_id host_id = couchdb_id.split('.')[0] query = session.query(Host).join(EntityMetadata).filter(EntityMetadata.couchdb_id == host_id) entity.host = query.one() - interface_id = '.'.join(couchdb_id.split('.')[:-1]) - query = session.query(Interface).join(EntityMetadata).filter(EntityMetadata.couchdb_id == interface_id) - entity.interface = query.one() - class VulnerabilityImporter(object): DOC_TYPE = ['Vulnerability', 'VulnerabilityWeb'] @@ -346,7 +298,7 @@ def add_relationships_from_dict(self, entity, entities): class FaradayEntityImporter(object): - # Document Types: [u'Service', u'Communication', u'Vulnerability', u'CommandRunInformation', u'Reports', u'Host', u'Workspace', u'Interface'] + # Document Types: [u'Service', u'Communication', u'Vulnerability', u'CommandRunInformation', u'Reports', u'Host', u'Workspace'] @classmethod def parse(cls, document): """Get an instance of a DAO object given a document""" @@ -367,7 +319,6 @@ def get_importer_from_document(cls, document): importer_class_mapper = { 'EntityMetadata': EntityMetadataImporter, 'Host': HostImporter, - 'Interface': InterfaceImporter, 'Service': ServiceImporter, 'Note': NoteImporter, 'CommandRunInformation': CommandImporter, @@ -375,8 +326,9 @@ def get_importer_from_document(cls, document): 'Vulnerability': VulnerabilityImporter, 'VulnerabilityWeb': VulnerabilityImporter, } + # TODO: remove this! if doc_type in ('Communication', 'Cred', 'Reports', - 'Task', 'TaskGroup'): + 'Task', 'TaskGroup', 'Interface', 'Note'): return importer_cls = importer_class_mapper.get(doc_type, None) if not importer_cls: diff --git a/server/models.py b/server/models.py index 37e9bf586ce..f477642e092 100644 --- a/server/models.py +++ b/server/models.py @@ -61,8 +61,8 @@ class SourceCode(db.Model): id = Column(Integer, primary_key=True) filename = Column(Text, nullable=False) - workspace = relationship('Workspace', backref='source_codes') workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace = relationship('Workspace', backref='source_codes') class Host(db.Model): @@ -81,11 +81,21 @@ class Host(db.Model): mac = Column(Text, nullable=True) net_segment = Column(Text, nullable=True) - entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) + entity_metadata = relationship( + EntityMetadata, + uselist=False, + cascade="all, delete-orphan", + single_parent=True, + foreign_keys=[entity_metadata_id] + ) - workspace = relationship('Workspace', backref='hosts') workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace = relationship( + 'Workspace', + backref='hosts', + foreign_keys=[workspace_id] + ) class Hostname(db.Model): @@ -94,8 +104,8 @@ class Hostname(db.Model): id = Column(Integer, primary_key=True) name = Column(Text, nullable=False) - host = relationship('Host', backref='hostnames') host_id = Column(Integer, ForeignKey('host.id'), index=True) + host = relationship('Host', backref='hostnames') class Service(db.Model): @@ -118,14 +128,24 @@ class Service(db.Model): banner = Column(Text, nullable=True) - entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) + entity_metadata = relationship( + EntityMetadata, + uselist=False, + cascade="all, delete-orphan", + single_parent=True, + foreign_keys=[entity_metadata_id] + ) - host = relationship('Host', backref='services') host_id = Column(Integer, ForeignKey('host.id'), index=True) + host = relationship('Host', backref='services', foreign_keys=[host_id]) - workspace = relationship('Workspace', backref='services') workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace = relationship( + 'Workspace', + backref='services', + foreign_keys=[workspace_id] + ) class Reference(db.Model): @@ -133,19 +153,27 @@ class Reference(db.Model): id = Column(Integer, primary_key=True) name = Column(Text, nullable=False) - workspace = relationship('Workspace', backref='references') workspace_id = Column( Integer, ForeignKey('workspace.id'), index=True ) + workspace = relationship( + 'Workspace', + backref='references', + foreign_keys=[workspace_id], + ) - vulnerability = relationship('Vulnerability', backref='references') vulnerability_id = Column( Integer, - ForeignKey('vulnerbility.id'), + ForeignKey('vulnerability.id'), index=True ) + vulnerability = relationship( + 'Vulnerability', + backref='references', + foreign_keys=[vulnerability_id], + ) class PolicyViolation(db.Model): @@ -153,19 +181,27 @@ class PolicyViolation(db.Model): id = Column(Integer, primary_key=True) name = Column(Text, nullable=False) - workspace = relationship('Workspace', backref='policy_violations') workspace_id = Column( Integer, ForeignKey('workspace.id'), index=True ) + workspace = relationship( + 'Workspace', + backref='policy_violations', + foreign_keys=[workspace_id], + ) - vulnerability = relationship('Vulnerability', backref='policy_violations') vulnerability_id = Column( Integer, - ForeignKey('vulnerbility.id'), + ForeignKey('vulnerability.id'), index=True ) + vulnerability = relationship( + 'Vulnerability', + backref='policy_violations', + foreign_keys=[vulnerability_id] + ) class VulnerabilityABC(db.Model): @@ -218,12 +254,12 @@ class VulnerabilityGeneric(VulnerabilityABC): status = Column(Enum(*STATUSES), nullable=False, default="open") type = Column(Enum(*VULN_TYPES), nullable=False) - workspace = relationship('Workspace', backref='vulnerabilities') workspace_id = Column( Integer, ForeignKey('workspace.id'), index=True, ) + workspace = relationship('Workspace', backref='vulnerabilities') __mapper_args__ = { 'polymorphic_on': type @@ -231,11 +267,19 @@ class VulnerabilityGeneric(VulnerabilityABC): class Vulnerability(VulnerabilityGeneric): - host = relationship('Host', backref='vulnerabilities') host_id = Column(Integer, ForeignKey(Host.id), index=True) + host = relationship( + 'Host', + backref='vulnerabilities', + foreign_keys=[host_id], + ) - service = relationship('Service', backref='vulnerabilities') service_id = Column(Integer, ForeignKey(Service.id), index=True) + service = relationship( + 'Service', + backref='vulnerabilities', + foreign_keys=[service_id], + ) __table_args__ = { 'extend_existing': True @@ -256,8 +300,12 @@ class VulnerabilityWeb(VulnerabilityGeneric): response = Column(Text(), nullable=True) website = Column(String(250), nullable=True) - service = relationship('Service', backref='vulnerabilities_web') - service_id = Column(Integer, ForeignKey(Service.id), index=True) + service_id_lala = Column(Integer, ForeignKey(Service.id), index=True) + service_lala = relationship( + 'Service', + backref='vulnerabilities_web', + foreign_keys=[service_id_lala], + ) __table_args__ = { 'extend_existing': True @@ -271,8 +319,12 @@ class VulnerabilityWeb(VulnerabilityGeneric): class VulnerabilityCode(VulnerabilityGeneric): line = Column(Integer, nullable=True) - source_code = relationship('SourceCode', backref='vulnerabilities') - source_code_id = Column(Integer, ForeignKey('SourceCode.id'), index=True) + source_code_id = Column(Integer, ForeignKey(SourceCode.id), index=True) + source_code = relationship( + SourceCode, + backref='vulnerabilities', + foreign_keys=[source_code_id] + ) __table_args__ = { 'extend_existing': True @@ -293,17 +345,31 @@ class Credential(db.Model): description = Column(Text(), nullable=True) name = Column(String(250), nullable=True) - entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) + entity_metadata = relationship( + EntityMetadata, + uselist=False, + cascade="all, delete-orphan", + single_parent=True, + foreign_keys=[entity_metadata_id], + ) - host = relationship('Host', backref='credentials') host_id = Column(Integer, ForeignKey(Host.id), index=True, nullable=True) + host = relationship('Host', backref='credentials', foreign_keys=[host_id]) - service = relationship('Service', backref='credentials') service_id = Column(Integer, ForeignKey(Service.id), index=True, nullable=True) + service = relationship( + 'Service', + backref='credentials', + foreign_keys=[service_id], + ) - workspace = relationship('Workspace', backref='credentials') workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) + workspace = relationship( + 'Workspace', + backref='credentials', + foreign_keys=[workspace_id], + ) class Command(db.Model): @@ -316,12 +382,23 @@ class Command(db.Model): hostname = Column(String(250), nullable=False) # where the command was executed params = Column(String(250), nullable=True) user = Column(String(250), nullable=True) # where the command was executed - workspace = relationship('Workspace') + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace = relationship('Workspace', foreign_keys=[workspace_id]) # TODO: add Tool relationship and report_attachment - entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) - entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) + entity_metadata_id = Column( + Integer, + ForeignKey(EntityMetadata.id), + index=True + ) + entity_metadata = relationship( + EntityMetadata, + uselist=False, + cascade="all, delete-orphan", + single_parent=True, + foreign_keys=[entity_metadata_id] + ) class Workspace(db.Model): @@ -415,8 +492,18 @@ class Methodology(db.Model): id = Column(Integer, primary_key=True) name = Column(Text, nullable=False) - entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) - entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) + entity_metadata_id = Column( + Integer, + ForeignKey(EntityMetadata.id), + index=True + ) + entity_metadata = relationship( + EntityMetadata, + uselist=False, + cascade="all, delete-orphan", + single_parent=True, + foreign_keys=[entity_metadata_id] + ) template = relationship('MethodologyTemplate', backref='methodologies') template_id = Column( @@ -534,11 +621,15 @@ class Comment(db.Model): text = Column(Text, nullable=False) - reply_to = relationship('Comment', backref='replies') reply_to_id = Column(Integer, ForeignKey('comment.id')) + reply_to = relationship( + 'Comment', + remote_side=[id], + foreign_keys=[reply_to_id] + ) object_id = Column(Integer, nullable=False) object_type = Column(Text, nullable=False) - workspace = relationship('Workspace') workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace = relationship('Workspace', foreign_keys=[workspace_id]) From f4094920979c53802c7627ae6fc59a162092a3ce Mon Sep 17 00:00:00 2001 From: micabot Date: Fri, 25 Aug 2017 19:25:14 -0300 Subject: [PATCH 0118/1506] Fix Policy Violations and References inheritance --- server/models.py | 112 +++++++++++++++++++++++------------------------ 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/server/models.py b/server/models.py index f477642e092..1c424ac5b32 100644 --- a/server/models.py +++ b/server/models.py @@ -148,62 +148,6 @@ class Service(db.Model): ) -class Reference(db.Model): - __tablename__ = 'reference' - id = Column(Integer, primary_key=True) - name = Column(Text, nullable=False) - - workspace_id = Column( - Integer, - ForeignKey('workspace.id'), - index=True - ) - workspace = relationship( - 'Workspace', - backref='references', - foreign_keys=[workspace_id], - ) - - vulnerability_id = Column( - Integer, - ForeignKey('vulnerability.id'), - index=True - ) - vulnerability = relationship( - 'Vulnerability', - backref='references', - foreign_keys=[vulnerability_id], - ) - - -class PolicyViolation(db.Model): - __tablename__ = 'policy_violation' - id = Column(Integer, primary_key=True) - name = Column(Text, nullable=False) - - workspace_id = Column( - Integer, - ForeignKey('workspace.id'), - index=True - ) - workspace = relationship( - 'Workspace', - backref='policy_violations', - foreign_keys=[workspace_id], - ) - - vulnerability_id = Column( - Integer, - ForeignKey('vulnerability.id'), - index=True - ) - vulnerability = relationship( - 'Vulnerability', - backref='policy_violations', - foreign_keys=[vulnerability_id] - ) - - class VulnerabilityABC(db.Model): # TODO: add unique constraint to -> name, description, severity, parent, method, pname, path, website, workspace # revisar plugin nexpose, netspark para terminar de definir uniques. asegurar que se carguen bien @@ -335,6 +279,62 @@ class VulnerabilityCode(VulnerabilityGeneric): } +class Reference(db.Model): + __tablename__ = 'reference' + id = Column(Integer, primary_key=True) + name = Column(Text, nullable=False) + + workspace_id = Column( + Integer, + ForeignKey('workspace.id'), + index=True + ) + workspace = relationship( + 'Workspace', + backref='references', + foreign_keys=[workspace_id], + ) + + vulnerability_id = Column( + Integer, + ForeignKey(VulnerabilityGeneric.id), + index=True + ) + vulnerability = relationship( + 'VulnerabilityGeneric', + backref='references', + foreign_keys=[vulnerability_id], + ) + + +class PolicyViolation(db.Model): + __tablename__ = 'policy_violation' + id = Column(Integer, primary_key=True) + name = Column(Text, nullable=False) + + workspace_id = Column( + Integer, + ForeignKey('workspace.id'), + index=True + ) + workspace = relationship( + 'Workspace', + backref='policy_violations', + foreign_keys=[workspace_id], + ) + + vulnerability_id = Column( + Integer, + ForeignKey(VulnerabilityGeneric.id), + index=True + ) + vulnerability = relationship( + 'VulnerabilityGeneric', + backref='policy_violations', + foreign_keys=[vulnerability_id] + ) + + class Credential(db.Model): # TODO: add unique constraint -> username, host o service y workspace # TODO: add constraint host y service, uno o el otro From d4bff910620d02a12984caecbc496bd17d9527fb Mon Sep 17 00:00:00 2001 From: micabot Date: Fri, 25 Aug 2017 19:35:06 -0300 Subject: [PATCH 0119/1506] Add Policy Violation and Reference template models These templates tables are added to provide PV and Ref for the Vuln Templates. --- server/models.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/server/models.py b/server/models.py index 1c424ac5b32..8c4c9850720 100644 --- a/server/models.py +++ b/server/models.py @@ -279,6 +279,23 @@ class VulnerabilityCode(VulnerabilityGeneric): } +class ReferenceTemplate(db.Model): + __tablename__ = 'reference_template' + id = Column(Integer, primary_key=True) + name = Column(Text, nullable=False) + + vulnerability_id = Column( + Integer, + ForeignKey(VulnerabilityTemplate.id), + index=True + ) + vulnerability = relationship( + 'VulnerabilityTemplate', + backref='references', + foreign_keys=[vulnerability_id], + ) + + class Reference(db.Model): __tablename__ = 'reference' id = Column(Integer, primary_key=True) @@ -307,6 +324,23 @@ class Reference(db.Model): ) +class PolicyViolationTemplate(db.Model): + __tablename__ = 'policy_violation_template' + id = Column(Integer, primary_key=True) + name = Column(Text, nullable=False) + + vulnerability_id = Column( + Integer, + ForeignKey(VulnerabilityTemplate.id), + index=True + ) + vulnerability = relationship( + 'VulnerabilityTemplate', + backref='policy_violations', + foreign_keys=[vulnerability_id] + ) + + class PolicyViolation(db.Model): __tablename__ = 'policy_violation' id = Column(Integer, primary_key=True) From 404471ae4696b199196642ec1cd0be37df736747 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 28 Aug 2017 14:24:14 -0300 Subject: [PATCH 0120/1506] Disable object importer on faraday-server startup --- faraday-server.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/faraday-server.py b/faraday-server.py index 1278a8a3394..7cfac7ff035 100755 --- a/faraday-server.py +++ b/faraday-server.py @@ -52,11 +52,6 @@ def setup_environment(check_deps=False): server.couchdb.push_reports() -def import_workspaces(): - import server.importer - server.importer.import_workspaces() - - def stop_server(): if not daemonize.stop_server(): # Exists with an error if it couldn't close the server @@ -118,7 +113,6 @@ def main(): if not args.no_setup: setup_environment(not args.nodeps) - import_workspaces() if args.start: # Starts a new process on background with --ignore-setup From 7e6d506c896f1e0594cfa410d372151c97e4b521 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 28 Aug 2017 14:37:29 -0300 Subject: [PATCH 0121/1506] Fix service relationship in vuln and web vuln --- server/models.py | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/server/models.py b/server/models.py index 8c4c9850720..06854c8b96a 100644 --- a/server/models.py +++ b/server/models.py @@ -14,6 +14,7 @@ UniqueConstraint, ) from sqlalchemy.orm import relationship, backref +from sqlalchemy.ext.declarative import declared_attr from flask_sqlalchemy import SQLAlchemy from flask_security import ( RoleMixin, @@ -218,12 +219,21 @@ class Vulnerability(VulnerabilityGeneric): foreign_keys=[host_id], ) - service_id = Column(Integer, ForeignKey(Service.id), index=True) + @declared_attr + def service_id(cls): + return VulnerabilityGeneric.__table__.c.get( + 'service_id', + Column( + Integer, + ForeignKey(Service.id), + index=True + ) + ) + service = relationship( - 'Service', - backref='vulnerabilities', - foreign_keys=[service_id], - ) + 'Service', + backref='vulnerabilities', + ) __table_args__ = { 'extend_existing': True @@ -244,12 +254,21 @@ class VulnerabilityWeb(VulnerabilityGeneric): response = Column(Text(), nullable=True) website = Column(String(250), nullable=True) - service_id_lala = Column(Integer, ForeignKey(Service.id), index=True) - service_lala = relationship( - 'Service', - backref='vulnerabilities_web', - foreign_keys=[service_id_lala], - ) + @declared_attr + def service_id(cls): + return VulnerabilityGeneric.__table__.c.get( + 'service_id', + Column( + Integer, + ForeignKey(Service.id), + index=True, + ) + ) + + service = relationship( + 'Service', + backref='vulnerabilities_web', + ) __table_args__ = { 'extend_existing': True From 88ea4d93471bfb3e0b723c6d8c3e668b3aa6d6ff Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 28 Aug 2017 16:03:35 -0300 Subject: [PATCH 0122/1506] Update importer to use levels --- server/importer.py | 312 ++++++++++++++++++--------------------------- 1 file changed, 121 insertions(+), 191 deletions(-) diff --git a/server/importer.py b/server/importer.py index 587552863ad..390bbf23650 100644 --- a/server/importer.py +++ b/server/importer.py @@ -6,15 +6,22 @@ import json import os +import requests +from tqdm import tqdm +from flask_script import Command as FlaskScriptCommand +from restkit.errors import RequestError, Unauthorized + import server.app import server.utils.logger import server.couchdb import server.database import server.models +import server.config from server.utils.database import get_or_create from server.models import ( db, EntityMetadata, + Credential, Host, Interface, Service, @@ -23,9 +30,7 @@ Workspace, Vulnerability ) -from restkit.errors import RequestError, Unauthorized -from tqdm import tqdm logger = server.utils.logger.get_logger(__name__) session = db.session @@ -73,9 +78,9 @@ class HostImporter(object): DOC_TYPE = 'Host' @classmethod - def update_from_document(cls, document): + def update_from_document(cls, document, workspace): # Ticket #3387: if the 'os' field is None, we default to 'unknown' - host = Host() + host, created = get_or_create(session, Host, name=document.get('name')) if not document.get('os'): document['os'] = 'unknown' @@ -87,29 +92,19 @@ def update_from_document(cls, document): host.default_gateway_ip = default_gateway and default_gateway[0] or '' host.default_gateway_mac = default_gateway and default_gateway[1] or '' host.owned = document.get('owned', False) + host.workspace = workspace return host - def add_relationships_from_dict(self, host, entities): - assert len(filter(lambda entity: isinstance(entity, Workspace), entities)) <= 1 - for couch_id, entity in entities.items(): - if isinstance(entity, Workspace): - host.workspace = entity - - - def __get_default_gateway(self, document): - default_gateway = document.get('default_gateway', None) - if default_gateway: - return default_gateway - else: - return u'', u'' + def set_parent(self, host, parent): + raise NotImplementedError class InterfaceImporter(object): DOC_TYPE = 'Interface' @classmethod - def update_from_document(cls, document): - interface = Interface() + def update_from_document(cls, document, workspace): + interface, created = get_or_create(session, Interface, name=document.get('name')) interface.name = document.get('name') interface.description = document.get('description') interface.mac = document.get('mac') @@ -127,32 +122,28 @@ def update_from_document(cls, document): interface.ports_filtered = document.get('ports', {}).get('filtered') interface.ports_opened = document.get('ports', {}).get('opened') interface.ports_closed = document.get('ports', {}).get('closed') + interface.workspace = workspace return interface - def add_relationships_from_dict(self, entity, entities): - host_id = '.'.join(entity.entity_metadata.couchdb_id.split('.')[:-1]) - if host_id not in entities: - raise EntityNotFound(host_id) - entity.host = entities[host_id] + @classmethod + def set_parent(cls, interface, parent_relation_db_id, level): + interface.host = session.query(Host).filter_by(id=parent_relation_db_id).first() - def add_relationships_from_db(self, entity, session): - host_id = '.'.join(entity.entity_metadata.couchdb_id.split('.')[:-1]) - query = session.query(Host).join(EntityMetadata).filter(EntityMetadata.couchdb_id == host_id) - entity.host = query.one() class ServiceImporter(object): DOC_TYPE = 'Service' @classmethod - def update_from_document(cls, document): - service = Service() + def update_from_document(cls, document, workspace): + service, created = get_or_create(session, Service, name=document.get('name')) service.name = document.get('name') service.description = document.get('description') service.owned = document.get('owned', False) service.protocol = document.get('protocol') service.status = document.get('status') service.version = document.get('version') + service.workspace = workspace # We found workspaces where ports are defined as an integer if isinstance(document.get('ports', None), (int, long)): @@ -161,38 +152,16 @@ def update_from_document(cls, document): service.ports = u','.join(map(str, document.get('ports'))) return service - def add_relationships_from_dict(self, entity, entities): - couchdb_id = entity.entity_metadata.couchdb_id - - host_id = couchdb_id.split('.')[0] - if host_id not in entities: - raise EntityNotFound(host_id) - entity.host = entities[host_id] - - interface_id = '.'.join(couchdb_id.split('.')[:-1]) - if interface_id not in entities: - raise EntityNotFound(interface_id) - entity.interface = entities[interface_id] - - def add_relationships_from_db(self, entity, session): - couchdb_id = entity.entity_metadata.couchdb_id - host_id = couchdb_id.split('.')[0] - query = session.query(Host).join(EntityMetadata).filter(EntityMetadata.couchdb_id == host_id) - entity.host = query.one() - - interface_id = '.'.join(couchdb_id.split('.')[:-1]) - query = session.query(Interface).join(EntityMetadata).filter(EntityMetadata.couchdb_id == interface_id) - entity.interface = query.one() + def set_parent(self, service, parent_id): + raise NotImplementedError class VulnerabilityImporter(object): DOC_TYPE = ['Vulnerability', 'VulnerabilityWeb'] @classmethod - def update_from_document(cls, document): - vulnerability = Vulnerability() - vulnerability.name = document.get('name') - vulnerability.description = document.get('desc') + def update_from_document(cls, document, workspace): + vulnerability, created = get_or_create(session, Vulnerability, name=document.get('name'), description= document.get('desc')) vulnerability.confirmed = document.get('confirmed') vulnerability.vuln_type = document.get('type') vulnerability.data = document.get('data') @@ -215,6 +184,7 @@ def update_from_document(cls, document): vulnerability.response = document.get('response') vulnerability.website = document.get('website') vulnerability.status = document.get('status', 'opened') + vulnerability.workspace = workspace params = document.get('params', u'') if isinstance(params, (list, tuple)): @@ -224,36 +194,19 @@ def update_from_document(cls, document): return vulnerability - def add_relationships_from_dict(self, entity, entities): - couchdb_id = entity.entity_metadata.couchdb_id - host_id = couchdb_id.split('.')[0] - if host_id not in entities: - raise EntityNotFound(host_id) - entity.host = entities[host_id] - - parent_id = '.'.join(couchdb_id.split('.')[:-1]) - if parent_id != host_id: - if parent_id not in entities: - raise EntityNotFound(parent_id) - entity.service = entities[parent_id] - - def add_relationships_from_db(self, entity, session): - couchdb_id = entity.entity_metadata.couchdb_id - host_id = couchdb_id.split('.')[0] - query = session.query(Host).join(EntityMetadata).filter(EntityMetadata.couchdb_id == host_id) - entity.host = query.one() - - parent_id = '.'.join(couchdb_id.split('.')[:-1]) - if parent_id != host_id: - query = session.query(Service).join(EntityMetadata).filter(EntityMetadata.couchdb_id == parent_id) - entity.service = query.one() - + @classmethod + def set_parent(self, vulnerability, parent_id, level=2): + logger.debug('Set parent for vulnerabiity level {0}'.format(level)) + if level == 2: + vulnerability.host = session.query(Host).filter_by(id=parent_id).first() + if level == 3: + vulnerability.service = session.query(Service).filter_by(id=parent_id).first() class CommandImporter(object): DOC_TYPE = 'CommandRunInformation' @classmethod - def update_from_document(cls, document): + def update_from_document(cls, document, workspace): command, instance = get_or_create(session, Command, command=document.get('command', None)) command.command = document.get('command', None) command.duration = document.get('duration', None) @@ -262,18 +215,12 @@ def update_from_document(cls, document): command.hostname = document.get('hostname', None) command.params = document.get('params', None) command.user = document.get('user', None) - - workspace_name = document.get('workspace', None) - if workspace_name: - workspace, instance = get_or_create(session, Workspace, name=document.get('workspace', None)) - command.workspace = workspace + command.workspace = workspace return command - def add_relationships_from_dict(self, entity, entities): - # this method is not required since update_from_document uses - # workspace name to create the relation - pass + def set_parent(self, command, parent_id): + raise NotImplementedError class NoteImporter(object): @@ -298,38 +245,18 @@ class CredentialImporter(object): DOC_TYPE = 'Cred' @classmethod - def update_from_document(cls, document): - credential = Credential() + def update_from_document(cls, document, workspace): + credential, created = get_or_create(session, Credential, name=document.get('username')) credential.username = document.get('username') credential.password = document.get('password', '') credential.owned = document.get('owned', False) credential.description = document.get('description', '') credential.name = document.get('name', '') + credential.workspace = workspace return credential - def add_relationships_from_dict(self, entity, entities): - couchdb_id = entity.entity_metadata.couchdb_id - host_id = couchdb_id.split('.')[0] - if host_id not in entities: - raise EntityNotFound(host_id) - entity.host = entities[host_id] - - parent_id = '.'.join(couchdb_id.split('.')[:-1]) - if parent_id != host_id: - if parent_id not in entities: - raise EntityNotFound(parent_id) - entity.service = entities[parent_id] - - def add_relationships_from_db(self, entity, session): - couchdb_id = entity.entity_metadata.couchdb_id - host_id = couchdb_id.split('.')[0] - query = session.query(Host).join(EntityMetadata).filter(EntityMetadata.couchdb_id == host_id) - entity.host = query.one() - - parent_id = '.'.join(couchdb_id.split('.')[:-1]) - if parent_id != host_id: - query = session.query(Service).join(EntityMetadata).filter(EntityMetadata.couchdb_id == parent_id) - entity.service = query.one() + def set_parent(self, credential, parent_id): + raise NotImplementedError class WorkspaceImporter(object): @@ -360,10 +287,8 @@ def parse(cls, document): return None, None @classmethod - def get_importer_from_document(cls, document): - doc_type = document.get('type') - if not doc_type: - return + def get_importer_from_document(cls, doc_type): + logger.info('Getting class importer for {0}'.format(doc_type)) importer_class_mapper = { 'EntityMetadata': EntityMetadataImporter, 'Host': HostImporter, @@ -388,87 +313,92 @@ def update_from_document(self, document): raise Exception('MUST IMPLEMENT') -def import_workspaces(): - """ - Main entry point for couchdb import - """ - app = server.app.create_app() - app.app_context().push() - db.create_all() +class ImportCouchDB(FlaskScriptCommand): - couchdb_server_conn, workspaces_list = _open_couchdb_conn() + def _open_couchdb_conn(self): + try: + couchdb_server_conn = server.couchdb.CouchDBServer() + workspaces_list = couchdb_server_conn.list_workspaces() - for workspace_name in workspaces_list: - logger.info(u'Setting up workspace {}'.format(workspace_name)) + except RequestError: + logger.error(u"CouchDB is not running at {}. Check faraday-server's"\ + " configuration and make sure CouchDB is running".format( + server.couchdb.get_couchdb_url())) + sys.exit(1) - if not server.couchdb.server_has_access_to(workspace_name): + except Unauthorized: logger.error(u"Unauthorized access to CouchDB. Make sure faraday-server's"\ - " configuration file has CouchDB admin's credentials set") + " configuration file has CouchDB admin's credentials set") sys.exit(1) - import_workspace_into_database(workspace_name, couchdb_server_conn=couchdb_server_conn) + return couchdb_server_conn, workspaces_list + def run(self): + """ + Main entry point for couchdb import + """ + couchdb_server_conn, workspaces_list = self._open_couchdb_conn() -def _open_couchdb_conn(): - try: - couchdb_server_conn = server.couchdb.CouchDBServer() - workspaces_list = couchdb_server_conn.list_workspaces() + for workspace_name in workspaces_list: + logger.info(u'Setting up workspace {}'.format(workspace_name)) - except RequestError: - logger.error(u"CouchDB is not running at {}. Check faraday-server's"\ - " configuration and make sure CouchDB is running".format( - server.couchdb.get_couchdb_url())) - sys.exit(1) + if not server.couchdb.server_has_access_to(workspace_name): + logger.error(u"Unauthorized access to CouchDB. Make sure faraday-server's"\ + " configuration file has CouchDB admin's credentials set") + sys.exit(1) - except Unauthorized: - logger.error(u"Unauthorized access to CouchDB. Make sure faraday-server's"\ - " configuration file has CouchDB admin's credentials set") - sys.exit(1) + self.import_workspace_into_database(workspace_name) - return couchdb_server_conn, workspaces_list - - -def import_workspace_into_database(workspace_name, couchdb_server_conn): - - workspace, created = get_or_create(session, server.models.Workspace, name=workspace_name) - - _import_from_couchdb(workspace, couchdb_server_conn) - session.commit() - - return created - - -def _import_from_couchdb(workspace, couchdb_conn): - if 'FARADAY_DONT_IMPORT' in os.environ: - return - couchdb_workspace = server.couchdb.CouchDBWorkspace(workspace.name, couchdb_server_conn=couchdb_conn) - total_amount = couchdb_workspace.get_total_amount_of_documents() - processed_docs, progress = 0, 0 - should_flush_changes = False - host_entities = {} - - def flush_changes(): - host_entities.clear() - session.commit() - session.expunge_all() - - for doc in tqdm(couchdb_workspace.get_documents(per_request=1000), total=total_amount): - processed_docs = processed_docs + 1 - entity_data = doc.get('doc') - importer, entity = FaradayEntityImporter.parse(entity_data) - if entity is not None: - if isinstance(entity, server.models.Host) and should_flush_changes: - flush_changes() - should_flush_changes = False - - try: - importer.add_relationships_from_dict(entity, host_entities) - except EntityNotFound: - logger.warning(u"Ignoring {} entity ({}) because its parent wasn't found".format( - entity.entity_metadata.document_type, entity.entity_metadata.couchdb_id)) - else: - host_entities[doc.get('key')] = entity - session.add(entity) + def get_objs(self, host, obj_type, level): + data = { + "map": "function(doc) { if(doc.type == '%s' && doc._id.split('.').length == %d) emit(null, doc); }" % (obj_type, level) + } - logger.info(u'{} importation done!'.format(workspace.name)) - flush_changes() + r = requests.post(host, json=data) + + return r.json() + + def import_workspace_into_database(self, workspace_name): + + faraday_importer = FaradayEntityImporter() + workspace, created = get_or_create(session, server.models.Workspace, name=workspace_name) + + couch_url = "http://{username}:{password}@{hostname}:{port}/{workspace_name}/_temp_view?include_docs=true".format( + username=server.config.couchdb.user, + password=server.config.couchdb.password, + hostname=server.config.couchdb.host, + port=server.config.couchdb.port, + workspace_name=workspace_name + ) + + obj_types = [ + (1, 'Host'), + (1, 'EntityMetadata'), + (1, 'Note'), + (1, 'CommandRunInformation'), + (2, 'Interface'), + (2, 'Service'), + (2, 'Vulnerability'), + (2, 'VulnerabilityWeb'), + (3, 'Vulnerability'), + (3, 'VulnerabilityWeb'), + ] + couchdb_relational_map = {} + + for level, obj_type in obj_types: + obj_importer = faraday_importer.get_importer_from_document(obj_type) + objs_dict = self.get_objs(couch_url, obj_type, level) + for raw_obj in tqdm(objs_dict.get('rows', [])): + raw_obj = raw_obj['value'] + couchdb_id = raw_obj['_id'] + new_obj = obj_importer.update_from_document(raw_obj, workspace) + if raw_obj.get('parent', None): + obj_importer.set_parent( + new_obj, + couchdb_relational_map[raw_obj['parent']], + level + ) + session.commit() + couchdb_relational_map[couchdb_id] = new_obj.id + + return created From 0d1926ce6f575b7a2884f48dff05d58a601ada28 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 28 Aug 2017 16:49:23 -0300 Subject: [PATCH 0123/1506] Add Exception when Postgres is not running --- server/commands/__init__.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/server/commands/__init__.py b/server/commands/__init__.py index 1455f61686e..2776353d754 100644 --- a/server/commands/__init__.py +++ b/server/commands/__init__.py @@ -5,15 +5,17 @@ from subprocess import Popen, PIPE try: + # py2.7 from configparser import ConfigParser, NoSectionError, NoOptionError except ImportError: + # py3 from ConfigParser import ConfigParser, NoSectionError, NoOptionError from flask import current_app from flask_script import Command from colorama import init from colorama import Fore, Back, Style - +from sqlalchemy.exc import OperationalError from config.globals import CONST_FARADAY_HOME_PATH from server.config import LOCAL_CONFIG_FILE @@ -27,7 +29,7 @@ def _check_current_config(self, config): config.get('database', 'connection_string') reconfigure = None while not reconfigure: - reconfigure = raw_input('Database section {red} already found.{white} Do you want to reconfigure database? (yes/no) '.format(red=Fore.RED, white=Fore.WHITE)) + reconfigure = raw_input('Database section {yellow} already found.{white} Do you want to reconfigure database? (yes/no) '.format(yellow=Fore.YELLOW, white=Fore.WHITE)) if reconfigure.lower() == 'no': return False elif reconfigure.lower() == 'yes': @@ -77,18 +79,18 @@ def run(self): def _check_psql_output(self, psql_log_output): if 'unknown user: postgres' in psql_log_output: - raise UserWarning('postgres user not found. did you installed postgresql?') + raise UserWarning('postgres user not found. did you install postgresql?') def _configure_postgres(self, psql_log_file): """ This step will create the role on the database. we return username and password and those values will be saved in the config file. """ - username = raw_input('Please enter the {red} database user {white} (press enter to use "faraday"): '.format(red=Fore.RED, white=Fore.WHITE)) or 'faraday' + username = raw_input('Please enter the {blue} database user {white} (press enter to use "faraday"): '.format(blue=Fore.BLUE, white=Fore.WHITE)) or 'faraday' postgres_command = ['sudo', '-u', 'postgres'] password = None while not password: - password = getpass.getpass(prompt='Please enter the {red} password for the postgreSQL username {white}: '.format(red=Fore.RED, white=Fore.WHITE)) + password = getpass.getpass(prompt='Please enter the {blue} password for the postgreSQL username {white}: '.format(blue=Fore.BLUE, white=Fore.WHITE)) if not password: print('Please type a valid password') command = postgres_command + ['psql', '-c', 'CREATE ROLE {0} WITH LOGIN PASSWORD \'{1}\';'.format(username, password)] @@ -101,19 +103,19 @@ def _create_database(self, username, psql_log_file): This step uses the createdb command to add a new database. """ postgres_command = ['sudo', '-u', 'postgres'] - database_name = raw_input('Please enter the {red} database name {white} (press enter to use "faraday"): '.format(red=Fore.RED, white=Fore.WHITE)) or 'faraday' + database_name = raw_input('Please enter the {blue} database name {white} (press enter to use "faraday"): '.format(blue=Fore.BLUE, white=Fore.WHITE)) or 'faraday' print('Creating database {0}'.format(database_name)) command = postgres_command + ['createdb', '-O', username, database_name] p = Popen(command, stderr=psql_log_file, stdout=psql_log_file) p.wait() - return database_name + return database_name, p.returncode def _save_config(self, config, username, password, database_name): """ This step saves database configuration to server.ini """ db_server = 'localhost' - print('{red}Saving {white} database credentials file in {0}'.format(LOCAL_CONFIG_FILE, red=Fore.RED, white=Fore.WHITE)) + print('Saving database credentials file in {0}'.format(LOCAL_CONFIG_FILE)) conn_string = 'postgresql+psycopg2://{username}:{password}@{server}/{database_name}'.format( username=username, @@ -132,7 +134,16 @@ def _create_tables(self, conn_string): current_app.config['SQLALCHEMY_DATABASE_URI'] = conn_string try: db.create_all() + except OperationalError as ex: + if 'could not connect to server' in ex.message: + print('[ERROR] {red}PostgreSQL service{white} is not running. Please verify that it is running in port 5432 before executing setup script.'.format(red=Fore.RED, white=Fore.WHITE)) + sys.exit(1) + else: + raise except ImportError as ex: if 'psycopg2' in ex: - print('Missing python depency {red}psycopg2{white}. Please install it with {green}pip install psycopg2'.format(red=Fore.RED, white=Fore.WHITE, green=Fore.GREEN)) + print( + '[ERROR] Missing python depency {red}psycopg2{white}. Please install it with {blue}pip install psycopg2'.format(red=Fore.RED, white=Fore.WHITE, blue=Fore.BLUE)) sys.exit(1) + else: + raise From ace54b3f6879b78a438bbb8485c154c8b20dde50 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 28 Aug 2017 17:29:26 -0300 Subject: [PATCH 0124/1506] add manage script --- manage.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100755 manage.py diff --git a/manage.py b/manage.py new file mode 100755 index 00000000000..0eb516eab14 --- /dev/null +++ b/manage.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python + + +from flask_script import Manager + +from server.web import app +from server.importer import ImportCouchDB + +manager = Manager(app) + + +if __name__ == "__main__": + manager.add_command('import-from-couchdb', ImportCouchDB()) + manager.run() From ca26d5cd23039af9fc97aea70ffff0c6b7a15757 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 28 Aug 2017 17:47:21 -0300 Subject: [PATCH 0125/1506] Add warning for specific errors on DB initialization Also changes the name for the default DB user --- server/commands/__init__.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/server/commands/__init__.py b/server/commands/__init__.py index 2776353d754..0f81fe896ca 100644 --- a/server/commands/__init__.py +++ b/server/commands/__init__.py @@ -29,7 +29,7 @@ def _check_current_config(self, config): config.get('database', 'connection_string') reconfigure = None while not reconfigure: - reconfigure = raw_input('Database section {yellow} already found.{white} Do you want to reconfigure database? (yes/no) '.format(yellow=Fore.YELLOW, white=Fore.WHITE)) + reconfigure = raw_input('Database section {yellow} already found{white}. Do you want to reconfigure database? (yes/no) '.format(yellow=Fore.YELLOW, white=Fore.WHITE)) if reconfigure.lower() == 'no': return False elif reconfigure.lower() == 'yes': @@ -60,15 +60,15 @@ def run(self): # current_psql_output is for checking psql command already known errors for each execution. psql_log_filename = os.path.join(faraday_path_conf, 'logs', 'psql_log.log') with open(psql_log_filename, 'a+') as psql_log_file: - username, password = self._configure_postgres(current_psql_output) + username, password, process_status = self._configure_postgres(current_psql_output) current_psql_output.seek(0) psql_output = current_psql_output.read() psql_log_file.write(psql_output) current_psql_output.seek(0) psql_output = current_psql_output.read() - self._check_psql_output(psql_output) - database_name = self._create_database(username, current_psql_output) - self._check_psql_output(psql_output) + self._check_psql_output(psql_output, process_status) + database_name, process_status = self._create_database(username, current_psql_output) + self._check_psql_output(psql_output, process_status) current_psql_output.close() conn_string = self._save_config(config, username, password, database_name) self._create_tables(conn_string) @@ -77,16 +77,22 @@ def run(self): print('User cancelled.') sys.exit(1) - def _check_psql_output(self, psql_log_output): + def _check_psql_output(self, psql_log_output, process_status): if 'unknown user: postgres' in psql_log_output: - raise UserWarning('postgres user not found. did you install postgresql?') + print('ERROR: Postgres user not found. Did you install package {blue}postgresql{white}?'.format(blue=Fore.BLUE, white=Fore.WHITE)) + elif 'could not connect to server' in psql_log_output: + print('ERROR: {red}PostgreSQL service{white} is not running. Please verify that it is running in port 5432 before executing setup script.'.format(red=Fore.RED, white=Fore.WHITE)) + elif process_status > 0: + print('ERROR: ' + psql_log_output) + sys.exit(1) def _configure_postgres(self, psql_log_file): """ This step will create the role on the database. we return username and password and those values will be saved in the config file. """ - username = raw_input('Please enter the {blue} database user {white} (press enter to use "faraday"): '.format(blue=Fore.BLUE, white=Fore.WHITE)) or 'faraday' + username_default = 'faraday_db_admin' + username = raw_input('Please enter the {blue} database user {white} (press enter to use "{0}"): '.format(username_default, blue=Fore.BLUE, white=Fore.WHITE)) or username_default postgres_command = ['sudo', '-u', 'postgres'] password = None while not password: @@ -96,7 +102,7 @@ def _configure_postgres(self, psql_log_file): command = postgres_command + ['psql', '-c', 'CREATE ROLE {0} WITH LOGIN PASSWORD \'{1}\';'.format(username, password)] p = Popen(command, stderr=psql_log_file, stdout=psql_log_file) p.wait() - return username, password + return username, password, p.returncode def _create_database(self, username, psql_log_file): """ @@ -136,14 +142,14 @@ def _create_tables(self, conn_string): db.create_all() except OperationalError as ex: if 'could not connect to server' in ex.message: - print('[ERROR] {red}PostgreSQL service{white} is not running. Please verify that it is running in port 5432 before executing setup script.'.format(red=Fore.RED, white=Fore.WHITE)) + print('ERROR: {red}PostgreSQL service{white} is not running. Please verify that it is running in port 5432 before executing setup script.'.format(red=Fore.RED, white=Fore.WHITE)) sys.exit(1) else: raise except ImportError as ex: if 'psycopg2' in ex: print( - '[ERROR] Missing python depency {red}psycopg2{white}. Please install it with {blue}pip install psycopg2'.format(red=Fore.RED, white=Fore.WHITE, blue=Fore.BLUE)) + 'ERROR: Missing python depency {red}psycopg2{white}. Please install it with {blue}pip install psycopg2'.format(red=Fore.RED, white=Fore.WHITE, blue=Fore.BLUE)) sys.exit(1) else: raise From afba2442a9e62d60a22f6f324573a4141e8b4338 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 28 Aug 2017 18:49:27 -0300 Subject: [PATCH 0126/1506] normalize manage command --- manage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manage.py b/manage.py index 44b8bad5fc0..498d5d07bfb 100755 --- a/manage.py +++ b/manage.py @@ -12,7 +12,7 @@ if __name__ == "__main__": - manager.add_command('import-from-couchdb', ImportCouchDB()) + manager.add_command('import_from_couchdb', ImportCouchDB()) manager.add_command('generate_database_schemas', DatabaseSchema()) manager.add_command('initdb', InitDB()) manager.add_command('faraday_schema_display', DatabaseSchema()) From fc91cffaa1bfc477bc9b66070043730f034147c9 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 28 Aug 2017 19:08:23 -0300 Subject: [PATCH 0127/1506] Fix bug introduced in InitDB When checking the log for the Init DB script the program always ended. --- server/commands/initdb.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index 0f81fe896ca..b32905ee758 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -84,7 +84,9 @@ def _check_psql_output(self, psql_log_output, process_status): print('ERROR: {red}PostgreSQL service{white} is not running. Please verify that it is running in port 5432 before executing setup script.'.format(red=Fore.RED, white=Fore.WHITE)) elif process_status > 0: print('ERROR: ' + psql_log_output) - sys.exit(1) + + if process_status is not 0: + sys.exit(process_status) def _configure_postgres(self, psql_log_file): """ From 489d4dd7cb23f3826b2048e3ca7371779d88762e Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 28 Aug 2017 19:14:59 -0300 Subject: [PATCH 0128/1506] Fix Enum error on DB creation PostgreSQL requires all Enum to be named. --- server/models.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/models.py b/server/models.py index 06854c8b96a..bcef172b576 100644 --- a/server/models.py +++ b/server/models.py @@ -124,7 +124,7 @@ class Service(db.Model): owned = Column(Boolean, nullable=False, default=False) protocol = Column(Text, nullable=False) - status = Column(Enum(*STATUSES), nullable=True) + status = Column(Enum(*STATUSES, name='service_statuses'), nullable=True) version = Column(Text, nullable=True) banner = Column(Text, nullable=True) @@ -165,7 +165,7 @@ class VulnerabilityABC(db.Model): data = Column(Text, nullable=True) description = Column(Text, nullable=False) - ease_of_resolution = Column(Enum(*EASE_OF_RESOLUTIONS), nullable=True) + ease_of_resolution = Column(Enum(*EASE_OF_RESOLUTIONS, name='vulnerability_ease_of_resolution'), nullable=True) name = Column(Text, nullable=False) resolution = Column(Text, nullable=True) severity = Column(String(50), nullable=False) @@ -196,8 +196,8 @@ class VulnerabilityGeneric(VulnerabilityABC): __tablename__ = 'vulnerability' confirmed = Column(Boolean, nullable=False, default=False) - status = Column(Enum(*STATUSES), nullable=False, default="open") - type = Column(Enum(*VULN_TYPES), nullable=False) + status = Column(Enum(*STATUSES, name='vulnerbaility_statuses'), nullable=False, default="open") + type = Column(Enum(*VULN_TYPES, name='vulnerability_types'), nullable=False) workspace_id = Column( Integer, @@ -607,7 +607,7 @@ class Task(TaskABC): id = Column(Integer, primary_key=True) due_date = Column(DateTime, nullable=True) - status = Column(Enum(*STATUSES), nullable=True) + status = Column(Enum(*STATUSES, name='task_statuses'), nullable=True) __mapper_args__ = { 'concrete': True From 6a910117393bbb13621fb6c20143c34aecb41e49 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 28 Aug 2017 19:37:27 -0300 Subject: [PATCH 0129/1506] Fix typo --- server/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index bcef172b576..79d85fa69c9 100644 --- a/server/models.py +++ b/server/models.py @@ -196,7 +196,7 @@ class VulnerabilityGeneric(VulnerabilityABC): __tablename__ = 'vulnerability' confirmed = Column(Boolean, nullable=False, default=False) - status = Column(Enum(*STATUSES, name='vulnerbaility_statuses'), nullable=False, default="open") + status = Column(Enum(*STATUSES, name='vulnerability_statuses'), nullable=False, default="open") type = Column(Enum(*VULN_TYPES, name='vulnerability_types'), nullable=False) workspace_id = Column( From f70862e87bcdf3f5b8023a061ad95bbfbb1ed73e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 26 Jun 2017 11:48:51 -0300 Subject: [PATCH 0130/1506] ADD postgresql support --- server/dao/host.py | 5 +++-- server/dao/service.py | 2 +- server/utils/database.py | 27 +++++++++++++++++++++++---- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/server/dao/host.py b/server/dao/host.py index 75f48a776d9..b90e55ef08d 100644 --- a/server/dao/host.py +++ b/server/dao/host.py @@ -69,8 +69,9 @@ def __query_database(self, search=None, page=0, page_size=0, order_by=None, orde .outerjoin(EntityMetadata, EntityMetadata.id == Host.entity_metadata_id)\ .outerjoin(Vulnerability, Host.id == Vulnerability.host_id)\ .outerjoin(Service, (Host.id == Service.host_id) & (Service.status.in_(("open", "running", "opened"))))\ - .outerjoin(Credential, (Credential.host_id == Host.id) & Credential.service_id == None)\ - .group_by(Host.id) + .outerjoin(Credential, (Credential.host_id == Host.id) ) \ + .filter(Credential.service_id == None) \ + .group_by(Host.id, EntityMetadata.id) query = query.filter(Host.workspace == self.workspace) # Apply pagination, sorting and filtering options to the query diff --git a/server/dao/service.py b/server/dao/service.py index c98fc991538..f39cd32b2fc 100644 --- a/server/dao/service.py +++ b/server/dao/service.py @@ -45,7 +45,7 @@ def list(self, service_filter={}): func.count(distinct(Credential.id)).label("credentials_count")) query = self._session.query(service_bundle).\ - group_by(Service.id).\ + group_by(Service.id, EntityMetadata.id).\ outerjoin(EntityMetadata, EntityMetadata.id == Service.entity_metadata_id).\ outerjoin(Vulnerability, Service.id == Vulnerability.service_id).group_by(Service.id).\ outerjoin(Credential, (Credential.service_id == Service.id) and (Credential.host_id == None)) diff --git a/server/utils/database.py b/server/utils/database.py index 7da669d1220..bb506440cca 100644 --- a/server/utils/database.py +++ b/server/utils/database.py @@ -2,11 +2,11 @@ # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information -import server.utils.logger - from sqlalchemy import distinct, Boolean from sqlalchemy.sql import func, asc, desc from sqlalchemy.sql.expression import ClauseElement +from sqlalchemy.sql import expression +from sqlalchemy.ext import compiler class ORDER_DIRECTIONS: @@ -18,7 +18,7 @@ def paginate(query, page, page_size): """ Limit results from a query based on pagination parameters """ - if not (page >= 0 and page_size >=0): + if not (page >= 0 and page_size >= 0): raise Exception("invalid values for pagination (page: %d, page_size: %d)" % (page, page_size)) return query.limit(page_size).offset(page * page_size) @@ -153,7 +153,8 @@ def get_count(query, count_col=None): else: count_filter = [func.count(distinct(count_col))] - count_q = query.statement.with_only_columns(count_filter).order_by(None).group_by(None) + count_q = query.statement.with_only_columns(count_filter).\ + order_by(None).group_by(None) count = query.session.execute(count_q).scalar() return count @@ -169,3 +170,21 @@ def get_or_create(session, model, defaults=None, **kwargs): instance = model(**params) session.add(instance) return instance, True + + +class GroupConcat(expression.FunctionElement): + name = "group_concat" + + +@compiler.compiles(GroupConcat, 'postgresql') +def _group_concat_postgresql(element, compiler, **kw): + if len(element.clauses) == 2: + separator = compiler.process(element.clauses.clauses[1]) + else: + separator = ',' + + res = 'array_to_string(array_agg({0}), \'{1}\')'.format( + compiler.process(element.clauses.clauses[0]), + separator, + ) + return res From a6b793842b90e5b3120068750957b82c923cdf6e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 10 Jul 2017 16:13:20 -0300 Subject: [PATCH 0131/1506] Fix more code to work in postgresql --- server/dao/vuln.py | 12 +++++++----- server/database.py | 2 +- server/importer.py | 2 +- server/utils/database.py | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/server/dao/vuln.py b/server/dao/vuln.py index 8d12e2a70b6..1a181424544 100644 --- a/server/dao/vuln.py +++ b/server/dao/vuln.py @@ -6,6 +6,7 @@ from server.dao.base import FaradayDAO from server.utils.database import ( + GroupConcat, paginate, sort_results, apply_search_filter, @@ -23,6 +24,7 @@ ) + class VulnerabilityDAO(FaradayDAO): MAPPED_ENTITY = Vulnerability COLUMNS_MAP = { @@ -86,8 +88,9 @@ def __query_database(self, search=None, page=0, page_size=0, order_by=None, orde # directly. query = self._session.query(vuln_bundle, service_bundle, - host_bundle)\ - .group_by(Vulnerability.id)\ + host_bundle, + GroupConcat(Interface.hostnames))\ + .group_by(Vulnerability.id, EntityMetadata.id, Service.id, Host.id)\ .outerjoin(EntityMetadata, EntityMetadata.id == Vulnerability.entity_metadata_id)\ .outerjoin(Service, Service.id == Vulnerability.service_id)\ .outerjoin(Host, Host.id == Vulnerability.host_id)\ @@ -201,10 +204,9 @@ def count(self, group_by=None, search=None, vuln_filter={}): return None col = VulnerabilityDAO.COLUMNS_MAP.get(group_by)[0] - vuln_bundle = Bundle('vuln', Vulnerability.id, col) + vuln_bundle = Bundle('vuln', col) query = self._session.query(vuln_bundle, func.count())\ - .group_by(col)\ - .outerjoin(EntityMetadata, EntityMetadata.id == Vulnerability.entity_metadata_id) + .group_by(col) query = apply_search_filter(query, self.COLUMNS_MAP, search, vuln_filter, self.STRICT_FILTERING) result = query.all() diff --git a/server/database.py b/server/database.py index 01f53001547..343b79b27cf 100644 --- a/server/database.py +++ b/server/database.py @@ -25,7 +25,7 @@ def add_entity_from_doc(self, document): include into the database (ie: these documents are added on future changes) """ - entity = server.models.FaradayEntity.parse(document) + entity = server.models.FaradayEntity.parse(self.__db_conn.session, document) if entity is None: return False diff --git a/server/importer.py b/server/importer.py index 56c95909c5c..67ace178276 100644 --- a/server/importer.py +++ b/server/importer.py @@ -340,6 +340,7 @@ def update_from_document(self, document): raise Exception('MUST IMPLEMENT') + def import_workspaces(): """ Main entry point for couchdb import @@ -386,7 +387,6 @@ def import_workspace_into_database(workspace_name, couchdb_server_conn): _import_from_couchdb(workspace, couchdb_server_conn) session.commit() - return created diff --git a/server/utils/database.py b/server/utils/database.py index bb506440cca..e13800ae65c 100644 --- a/server/utils/database.py +++ b/server/utils/database.py @@ -94,7 +94,7 @@ def apply_search_filter(query, field_to_col_map, free_text_search=None, field_fi # ignore this list since its purpose is clearly to # match anything it can find. if is_direct_filter_search and attribute in strict_filter: - search_term = column.is_(field_filter.get(attribute)) + search_term = column.op('=')(field_filter.get(attribute)) else: search_term = column.like(like_str) From 5226640210369c40b6b21e56d37dd9da904cd48f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 29 Aug 2017 16:07:16 -0300 Subject: [PATCH 0132/1506] update code to use the new model --- server/importer.py | 106 +++++++++++++++++++++++++++++++-------------- server/models.py | 6 +-- 2 files changed, 77 insertions(+), 35 deletions(-) diff --git a/server/importer.py b/server/importer.py index a0c9f8bd821..2e1d0868db6 100644 --- a/server/importer.py +++ b/server/importer.py @@ -4,7 +4,7 @@ import sys import json -import os +import datetime import requests from tqdm import tqdm @@ -26,6 +26,7 @@ Service, Command, Workspace, + Hostname, Vulnerability ) @@ -78,7 +79,7 @@ class HostImporter(object): @classmethod def update_from_document(cls, document, workspace): # Ticket #3387: if the 'os' field is None, we default to 'unknown' - host, created = get_or_create(session, Host, name=document.get('name')) + host, created = get_or_create(session, Host, ip=document.get('name')) if not document.get('os'): document['os'] = 'unknown' @@ -98,34 +99,60 @@ def set_parent(self, host, parent): class InterfaceImporter(object): + """ + Class interface was removed in the new model. + We will merge the interface data with the host. + For ports we will create new services for open ports + if it was not previously created. + """ DOC_TYPE = 'Interface' @classmethod def update_from_document(cls, document, workspace): - interface, created = get_or_create(session, Interface, name=document.get('name')) - interface.name = document.get('name') - interface.description = document.get('description') - interface.mac = document.get('mac') - interface.owned = document.get('owned', False) - interface.hostnames = u','.join(document.get('hostnames') or []) - interface.network_segment = document.get('network_segment') - interface.ipv4_address = document.get('ipv4').get('address') - interface.ipv4_gateway = document.get('ipv4').get('gateway') - interface.ipv4_dns = u','.join(document.get('ipv4').get('DNS')) - interface.ipv4_mask = document.get('ipv4').get('mask') - interface.ipv6_address = document.get('ipv6').get('address') - interface.ipv6_gateway = document.get('ipv6').get('gateway') - interface.ipv6_dns = u','.join(document.get('ipv6').get('DNS')) - interface.ipv6_prefix = str(document.get('ipv6').get('prefix')) - interface.ports_filtered = document.get('ports', {}).get('filtered') - interface.ports_opened = document.get('ports', {}).get('opened') - interface.ports_closed = document.get('ports', {}).get('closed') - interface.workspace = workspace + + interface = {} + interface['name'] = document.get('name') + interface['description'] = document.get('description') + interface['mac'] = document.get('mac') + interface['owned'] = document.get('owned', False) + interface['hostnames'] = document.get('hostnames') + interface['network_segment'] = document.get('network_segment') + interface['ipv4_address'] = document.get('ipv4').get('address') + interface['ipv4_gateway'] = document.get('ipv4').get('gateway') + interface['ipv4_dns'] = document.get('ipv4').get('DNS') + interface['ipv4_mask'] = document.get('ipv4').get('mask') + interface['ipv6_address'] = document.get('ipv6').get('address') + interface['ipv6_gateway'] = document.get('ipv6').get('gateway') + interface['ipv6_dns'] = document.get('ipv6').get('DNS') + interface['ipv6_prefix'] = str(document.get('ipv6').get('prefix')) + # ports_* are integers with counts + interface['ports_filtered'] = document.get('ports', {}).get('filtered') + interface['ports_opened'] = document.get('ports', {}).get('opened') + interface['ports_closed'] = document.get('ports', {}).get('closed') + interface['workspace'] = workspace return interface @classmethod def set_parent(cls, interface, parent_relation_db_id, level): - interface.host = session.query(Host).filter_by(id=parent_relation_db_id).first() + host = session.query(Host).filter_by(id=parent_relation_db_id).first() + assert host.workspace == interface['workspace'] + if interface['mac']: + host.mac = interface['mac'] + if interface['owned']: + host.owned = interface['owned'] + if interface['ipv4_address'] or interface['ipv6_address']: + host.ip = interface['ipv4_address'] or interface['ipv6_address'] + if interface['ipv4_gateway'] or interface['ipv6_gateway']: + host.default_gateway_ip = interface['ipv4_gateway'] or interface['ipv6_gateway'] + #host.default_gateway_mac + if interface['network_segment']: + host.net_segment = interface['network_segment'] + if interface['description']: + host.description += '\n {0}'.format(interface['description']) + + for hostname in interface['hostnames']: + hostname, created = get_or_create(session, Hostname, name=hostname, host=host) + host.owned = host.owned or interface['owned'] class ServiceImporter(object): @@ -180,7 +207,12 @@ def update_from_document(cls, document, workspace): vulnerability.request = document.get('request') vulnerability.response = document.get('response') vulnerability.website = document.get('website') - vulnerability.status = document.get('status', 'opened') + status_map = { + 'opened': 'open', + 'closed': 'closed', + } + status = status_map[document.get('status', 'opened')] + vulnerability.status = status vulnerability.workspace = workspace params = document.get('params', u'') @@ -204,10 +236,16 @@ class CommandImporter(object): @classmethod def update_from_document(cls, document, workspace): - command, instance = get_or_create(session, Command, command=document.get('command', None)) + start_date = datetime.datetime.fromtimestamp(document.get('itime')) + end_date = start_date + datetime.timedelta(seconds=document.get('duration')) + command, instance = get_or_create( + session, + Command, + command=document.get('command', None), + start_date=start_date, + end_date=end_date + ) command.command = document.get('command', None) - command.duration = document.get('duration', None) - command.itime = document.get('itime', None) command.ip = document.get('ip', None) command.hostname = document.get('hostname', None) command.params = document.get('params', None) @@ -291,15 +329,12 @@ def get_importer_from_document(cls, doc_type): 'Host': HostImporter, 'Service': ServiceImporter, 'Note': NoteImporter, + 'Interface': InterfaceImporter, 'CommandRunInformation': CommandImporter, 'Workspace': WorkspaceImporter, 'Vulnerability': VulnerabilityImporter, 'VulnerabilityWeb': VulnerabilityImporter, } - # TODO: remove this! - if doc_type in ('Communication', 'Cred', 'Reports', - 'Task', 'TaskGroup', 'Interface', 'Note'): - return importer_cls = importer_class_mapper.get(doc_type, None) if not importer_cls: raise NotImplementedError('Class importer for {0} not implemented'.format(doc_type)) @@ -368,6 +403,9 @@ def import_workspace_into_database(self, workspace_name): workspace_name=workspace_name ) + # obj_types are tuples. the first value is the level on the tree + # for the desired obj. + # the idea is to improt by level from couchdb data. obj_types = [ (1, 'Host'), (1, 'EntityMetadata'), @@ -381,13 +419,16 @@ def import_workspace_into_database(self, workspace_name): (3, 'VulnerabilityWeb'), ] couchdb_relational_map = {} - + removed_objs = ['Interface'] for level, obj_type in obj_types: obj_importer = faraday_importer.get_importer_from_document(obj_type) objs_dict = self.get_objs(couch_url, obj_type, level) for raw_obj in tqdm(objs_dict.get('rows', [])): raw_obj = raw_obj['value'] couchdb_id = raw_obj['_id'] + if obj_importer is None: + import ipdb + ipdb.set_trace() new_obj = obj_importer.update_from_document(raw_obj, workspace) if raw_obj.get('parent', None): obj_importer.set_parent( @@ -396,6 +437,7 @@ def import_workspace_into_database(self, workspace_name): level ) session.commit() - couchdb_relational_map[couchdb_id] = new_obj.id + if obj_type not in removed_objs: + couchdb_relational_map[couchdb_id] = new_obj.id return created diff --git a/server/models.py b/server/models.py index bcef172b576..a5c78d7f8e6 100644 --- a/server/models.py +++ b/server/models.py @@ -428,13 +428,13 @@ class Credential(db.Model): class Command(db.Model): __tablename__ = 'command' id = Column(Integer, primary_key=True) - command = Column(String(250), nullable=False) + command = Column(Text(), nullable=False) start_date = Column(DateTime, nullable=False) end_date = Column(DateTime, nullable=False) ip = Column(String(250), nullable=False) # where the command was executed hostname = Column(String(250), nullable=False) # where the command was executed - params = Column(String(250), nullable=True) - user = Column(String(250), nullable=True) # where the command was executed + params = Column(Text(), nullable=True) + user = Column(String(250), nullable=True) # os username where the command was executed workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) workspace = relationship('Workspace', foreign_keys=[workspace_id]) From a28fe7efae4989d8ba1e913868b1143a654088d0 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 29 Aug 2017 16:23:14 -0300 Subject: [PATCH 0133/1506] Temporary fix for service_id prop Vuln and VulnWeb --- server/models.py | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/server/models.py b/server/models.py index bcef172b576..5fadc2ecf06 100644 --- a/server/models.py +++ b/server/models.py @@ -219,17 +219,7 @@ class Vulnerability(VulnerabilityGeneric): foreign_keys=[host_id], ) - @declared_attr - def service_id(cls): - return VulnerabilityGeneric.__table__.c.get( - 'service_id', - Column( - Integer, - ForeignKey(Service.id), - index=True - ) - ) - + service_id = Column(Integer, ForeignKey(Service.id)) service = relationship( 'Service', backref='vulnerabilities', @@ -254,17 +244,7 @@ class VulnerabilityWeb(VulnerabilityGeneric): response = Column(Text(), nullable=True) website = Column(String(250), nullable=True) - @declared_attr - def service_id(cls): - return VulnerabilityGeneric.__table__.c.get( - 'service_id', - Column( - Integer, - ForeignKey(Service.id), - index=True, - ) - ) - + service_id = Column(Integer, ForeignKey(Service.id)) service = relationship( 'Service', backref='vulnerabilities_web', From c06dcb084ae4283e0afbd2f2ac7b1109e37a8f77 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 29 Aug 2017 18:08:52 -0300 Subject: [PATCH 0134/1506] Add model for Executive Report --- server/models.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/server/models.py b/server/models.py index d24303c7b6b..408fe7dc128 100644 --- a/server/models.py +++ b/server/models.py @@ -666,3 +666,26 @@ class Comment(db.Model): workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) workspace = relationship('Workspace', foreign_keys=[workspace_id]) + +class ExecutiveReport(db.Model): + STATUSES = [ + 'created' + ] + __tablename__ = 'executive_report' + id = Column(Integer, primary_key=True) + + grouped = Column(Boolean, nullable=False, default=False) + name = Column(Text, nullable=False, index=True) + status = Column(Enum(*STATUSES, name='executive_report_statuses'), nullable=True) + template_name = Column(Text, nullable=False) + + conclusions = Column(Text, nullable=True) + enterprise = Column(Text, nullable=True) + objectives = Column(Text, nullable=True) + recommendations = Column(Text, nullable=True) + scope = Column(Text, nullable=True) + summary = Column(Text, nullable=True) + title = Column(Text, nullable=True) + + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace = relationship('Workspace', foreign_keys=[workspace_id]) From c470240070a278fb3f807d41838d6a29ac658d02 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 29 Aug 2017 18:32:18 -0300 Subject: [PATCH 0135/1506] Add Statuses for Executive Reports --- server/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index 408fe7dc128..82d557bb69a 100644 --- a/server/models.py +++ b/server/models.py @@ -669,7 +669,9 @@ class Comment(db.Model): class ExecutiveReport(db.Model): STATUSES = [ - 'created' + 'created', + 'error', + 'processing', ] __tablename__ = 'executive_report' id = Column(Integer, primary_key=True) From fa7a8ccdaf343efd3dbb732925f82b05d2d47a08 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 29 Aug 2017 20:04:06 -0300 Subject: [PATCH 0136/1506] Importer updates. add credential support and import one service per port --- server/importer.py | 128 +++++++++++++++++++++++++++------------------ 1 file changed, 78 insertions(+), 50 deletions(-) diff --git a/server/importer.py b/server/importer.py index 2e1d0868db6..8246d4cd815 100644 --- a/server/importer.py +++ b/server/importer.py @@ -43,7 +43,7 @@ def __init__(self, entity_id): class EntityMetadataImporter(object): @classmethod - def update_from_document(cls, document): + def update_from_document(cls, document, workspace, level=None, couchdb_relational_map=None): entity, created = get_or_create(session, EntityMetadata, couchdb_id=document.get('_id')) metadata = document.get('metadata', dict()) entity.update_time = metadata.get('update_time', None) @@ -60,7 +60,7 @@ def update_from_document(cls, document): if entity.create_time is not None: entity.create_time = cls.__truncate_to_epoch_in_seconds(entity.create_time) - return entity + yield entity @classmethod def __truncate_to_epoch_in_seconds(self, timestamp): @@ -77,7 +77,7 @@ class HostImporter(object): DOC_TYPE = 'Host' @classmethod - def update_from_document(cls, document, workspace): + def update_from_document(cls, document, workspace, level=None, couchdb_relational_map=None): # Ticket #3387: if the 'os' field is None, we default to 'unknown' host, created = get_or_create(session, Host, ip=document.get('name')) if not document.get('os'): @@ -92,7 +92,7 @@ def update_from_document(cls, document, workspace): host.default_gateway_mac = default_gateway and default_gateway[1] or '' host.owned = document.get('owned', False) host.workspace = workspace - return host + yield host def set_parent(self, host, parent): raise NotImplementedError @@ -108,7 +108,7 @@ class InterfaceImporter(object): DOC_TYPE = 'Interface' @classmethod - def update_from_document(cls, document, workspace): + def update_from_document(cls, document, workspace, level=None, couchdb_relational_map=None): interface = {} interface['name'] = document.get('name') @@ -130,7 +130,7 @@ def update_from_document(cls, document, workspace): interface['ports_opened'] = document.get('ports', {}).get('opened') interface['ports_closed'] = document.get('ports', {}).get('closed') interface['workspace'] = workspace - return interface + yield interface @classmethod def set_parent(cls, interface, parent_relation_db_id, level): @@ -159,32 +159,44 @@ class ServiceImporter(object): DOC_TYPE = 'Service' @classmethod - def update_from_document(cls, document, workspace): - service, created = get_or_create(session, Service, name=document.get('name')) - service.name = document.get('name') - service.description = document.get('description') - service.owned = document.get('owned', False) - service.protocol = document.get('protocol') - service.status = document.get('status') - service.version = document.get('version') - service.workspace = workspace - - # We found workspaces where ports are defined as an integer - if isinstance(document.get('ports', None), (int, long)): - service.ports = str(document.get('ports')) - else: - service.ports = u','.join(map(str, document.get('ports'))) - return service + def update_from_document(cls, document, workspace, level=None, couchdb_relational_map=None): + #service was always above interface, not it's above host. + try: + parent_id = document['parent'].split('.')[0] + except KeyError: + # some services are missing the parent key + parent_id = document['_id'].split('.')[0] + host, created = get_or_create(session, Host, id=couchdb_relational_map[parent_id]) + for port in document.get('ports'): + service, created = get_or_create(session, Service, name=document.get('name'), port=port) + service.name = document.get('name') + service.description = document.get('description') + service.owned = document.get('owned', False) + service.protocol = document.get('protocol') + if not document.get('status'): + logger.warning('Service {0} with empty status. Using open as status'.format(document['_id'])) + document['status'] = 'open' + status_mapper = { + 'open': 'open', + 'closed': 'closed', + 'filtered': 'filtered' + } + service.status = status_mapper[document.get('status')] + service.version = document.get('version') + service.workspace = workspace + + yield service - def set_parent(self, service, parent_id): - raise NotImplementedError + @classmethod + def set_parent(self, service, parent_id, level): + pass class VulnerabilityImporter(object): DOC_TYPE = ['Vulnerability', 'VulnerabilityWeb'] @classmethod - def update_from_document(cls, document, workspace): + def update_from_document(cls, document, workspace, level=None, couchdb_relational_map=None): vulnerability, created = get_or_create(session, Vulnerability, name=document.get('name'), description= document.get('desc')) vulnerability.confirmed = document.get('confirmed') vulnerability.vuln_type = document.get('type') @@ -221,7 +233,7 @@ def update_from_document(cls, document, workspace): else: vulnerability.params = params if params is not None else u'' - return vulnerability + yield vulnerability @classmethod def set_parent(self, vulnerability, parent_id, level=2): @@ -235,7 +247,7 @@ class CommandImporter(object): DOC_TYPE = 'CommandRunInformation' @classmethod - def update_from_document(cls, document, workspace): + def update_from_document(cls, document, workspace, level=None, couchdb_relational_map=None): start_date = datetime.datetime.fromtimestamp(document.get('itime')) end_date = start_date + datetime.timedelta(seconds=document.get('duration')) command, instance = get_or_create( @@ -252,7 +264,7 @@ def update_from_document(cls, document, workspace): command.user = document.get('user', None) command.workspace = workspace - return command + yield command def set_parent(self, command, parent_id): raise NotImplementedError @@ -262,13 +274,13 @@ class NoteImporter(object): DOC_TYPE = 'Note' @classmethod - def update_from_document(cls, document): + def update_from_document(cls, document, workspace, level=None, couchdb_relational_map=None): note = Note() note.name = document.get('name') note.text = document.get('text', None) note.description = document.get('description', None) note.owned = document.get('owned', False) - return note + yield note def add_relationships_from_dict(self, entity, entities): # this method is not required since update_from_document uses @@ -280,18 +292,25 @@ class CredentialImporter(object): DOC_TYPE = 'Cred' @classmethod - def update_from_document(cls, document, workspace): - credential, created = get_or_create(session, Credential, name=document.get('username')) + def update_from_document(cls, document, workspace, level=None, couchdb_relational_map=None): + host = None + service = None + if level == 2: + parent_id = couchdb_relational_map[document['_id'].split('.')[0]] + host = session.query(Host).filter_by(id=parent_id).first() + if level == 4: + parent_id = couchdb_relational_map['.'.join(document['_id'].split('.')[:3])] + service = session.query(Service).filter_by(id=parent_id).first() + if not host and not service: + raise Exception('Missing host or service for credential {0}'.format(document['_id'])) + credential, created = get_or_create(session, Credential, name=document.get('username'), host=host, service=service) credential.username = document.get('username') credential.password = document.get('password', '') credential.owned = document.get('owned', False) credential.description = document.get('description', '') credential.name = document.get('name', '') credential.workspace = workspace - return credential - - def set_parent(self, credential, parent_id): - raise NotImplementedError + yield credential class WorkspaceImporter(object): @@ -300,7 +319,7 @@ class WorkspaceImporter(object): @classmethod def update_from_document(cls, document): workspace, created = get_or_create(session, server.models.Workspace, name=document.get('name', None)) - return workspace + yield workspace def add_relationships_from_dict(self, entity, entities): for couch_id, child_entity in entities.items(): @@ -329,6 +348,7 @@ def get_importer_from_document(cls, doc_type): 'Host': HostImporter, 'Service': ServiceImporter, 'Note': NoteImporter, + 'Credential': CredentialImporter, 'Interface': InterfaceImporter, 'CommandRunInformation': CommandImporter, 'Workspace': WorkspaceImporter, @@ -382,6 +402,8 @@ def run(self): self.import_workspace_into_database(workspace_name) def get_objs(self, host, obj_type, level): + if obj_type == 'Credential': + obj_type = 'Cred' data = { "map": "function(doc) { if(doc.type == '%s' && doc._id.split('.').length == %d) emit(null, doc); }" % (obj_type, level) } @@ -413,12 +435,16 @@ def import_workspace_into_database(self, workspace_name): (1, 'CommandRunInformation'), (2, 'Interface'), (2, 'Service'), + (2, 'Credential'), (2, 'Vulnerability'), (2, 'VulnerabilityWeb'), (3, 'Vulnerability'), (3, 'VulnerabilityWeb'), + (3, 'Service'), + (4, 'Credential'), # Level 4 is for interface ] couchdb_relational_map = {} + couchdb_removed_objs = set() removed_objs = ['Interface'] for level, obj_type in obj_types: obj_importer = faraday_importer.get_importer_from_document(obj_type) @@ -426,18 +452,20 @@ def import_workspace_into_database(self, workspace_name): for raw_obj in tqdm(objs_dict.get('rows', [])): raw_obj = raw_obj['value'] couchdb_id = raw_obj['_id'] - if obj_importer is None: - import ipdb - ipdb.set_trace() - new_obj = obj_importer.update_from_document(raw_obj, workspace) - if raw_obj.get('parent', None): - obj_importer.set_parent( - new_obj, - couchdb_relational_map[raw_obj['parent']], - level - ) - session.commit() - if obj_type not in removed_objs: - couchdb_relational_map[couchdb_id] = new_obj.id + for new_obj in obj_importer.update_from_document(raw_obj, workspace, level, couchdb_relational_map): + if raw_obj.get('parent', None): + parent_id = raw_obj['parent'] + if parent_id in couchdb_removed_objs: + parent_id = '.'.join(parent_id.split('.')[:-1]) + obj_importer.set_parent( + new_obj, + couchdb_relational_map[parent_id], + level + ) + session.commit() + if obj_type not in removed_objs: + couchdb_relational_map[couchdb_id] = new_obj.id + else: + couchdb_removed_objs.add(couchdb_id) return created From 42d1d7c05ac07e895ce86c6cf3d7f79142c3e0de Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 30 Aug 2017 11:53:38 -0300 Subject: [PATCH 0137/1506] Code clean up --- server/importer.py | 99 ++++++++++++++++------------------------------ 1 file changed, 35 insertions(+), 64 deletions(-) diff --git a/server/importer.py b/server/importer.py index 8246d4cd815..bc0d15608fe 100644 --- a/server/importer.py +++ b/server/importer.py @@ -42,8 +42,7 @@ def __init__(self, entity_id): class EntityMetadataImporter(object): - @classmethod - def update_from_document(cls, document, workspace, level=None, couchdb_relational_map=None): + def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): entity, created = get_or_create(session, EntityMetadata, couchdb_id=document.get('_id')) metadata = document.get('metadata', dict()) entity.update_time = metadata.get('update_time', None) @@ -58,11 +57,11 @@ def update_from_document(cls, document, workspace, level=None, couchdb_relationa entity.command_id = metadata.get('command_id', None) if entity.create_time is not None: - entity.create_time = cls.__truncate_to_epoch_in_seconds(entity.create_time) + entity.create_time = self.__truncate_to_epoch_in_seconds(entity.create_time) yield entity - @classmethod + def __truncate_to_epoch_in_seconds(self, timestamp): """ In a not so elegant fashion, identifies and truncate epoch timestamps expressed in milliseconds to seconds""" @@ -76,8 +75,7 @@ def __truncate_to_epoch_in_seconds(self, timestamp): class HostImporter(object): DOC_TYPE = 'Host' - @classmethod - def update_from_document(cls, document, workspace, level=None, couchdb_relational_map=None): + def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): # Ticket #3387: if the 'os' field is None, we default to 'unknown' host, created = get_or_create(session, Host, ip=document.get('name')) if not document.get('os'): @@ -94,9 +92,6 @@ def update_from_document(cls, document, workspace, level=None, couchdb_relationa host.workspace = workspace yield host - def set_parent(self, host, parent): - raise NotImplementedError - class InterfaceImporter(object): """ @@ -107,8 +102,7 @@ class InterfaceImporter(object): """ DOC_TYPE = 'Interface' - @classmethod - def update_from_document(cls, document, workspace, level=None, couchdb_relational_map=None): + def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): interface = {} interface['name'] = document.get('name') @@ -130,10 +124,14 @@ def update_from_document(cls, document, workspace, level=None, couchdb_relationa interface['ports_opened'] = document.get('ports', {}).get('opened') interface['ports_closed'] = document.get('ports', {}).get('closed') interface['workspace'] = workspace + couch_parent_id = document.get('parent', None) + if not couch_parent_id: + couch_parent_id = '.'.join(document['_id'].split('.')[:-1]) + parent_id = couchdb_relational_map[couch_parent_id] + self.merge_with_host(interface, parent_id) yield interface - @classmethod - def set_parent(cls, interface, parent_relation_db_id, level): + def merge_with_host(self, interface, parent_relation_db_id): host = session.query(Host).filter_by(id=parent_relation_db_id).first() assert host.workspace == interface['workspace'] if interface['mac']: @@ -158,8 +156,8 @@ def set_parent(cls, interface, parent_relation_db_id, level): class ServiceImporter(object): DOC_TYPE = 'Service' - @classmethod - def update_from_document(cls, document, workspace, level=None, couchdb_relational_map=None): + + def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): #service was always above interface, not it's above host. try: parent_id = document['parent'].split('.')[0] @@ -187,16 +185,12 @@ def update_from_document(cls, document, workspace, level=None, couchdb_relationa yield service - @classmethod - def set_parent(self, service, parent_id, level): - pass - class VulnerabilityImporter(object): DOC_TYPE = ['Vulnerability', 'VulnerabilityWeb'] - @classmethod - def update_from_document(cls, document, workspace, level=None, couchdb_relational_map=None): + + def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): vulnerability, created = get_or_create(session, Vulnerability, name=document.get('name'), description= document.get('desc')) vulnerability.confirmed = document.get('confirmed') vulnerability.vuln_type = document.get('type') @@ -233,9 +227,13 @@ def update_from_document(cls, document, workspace, level=None, couchdb_relationa else: vulnerability.params = params if params is not None else u'' + couch_parent_id = document.get('parent', None) + if not couch_parent_id: + couch_parent_id = '.'.join(document['_id'].split('.')[:-1]) + parent_id = couchdb_relational_map[couch_parent_id] + self.set_parent(vulnerability, parent_id, level) yield vulnerability - @classmethod def set_parent(self, vulnerability, parent_id, level=2): logger.debug('Set parent for vulnerabiity level {0}'.format(level)) if level == 2: @@ -243,11 +241,12 @@ def set_parent(self, vulnerability, parent_id, level=2): if level == 3: vulnerability.service = session.query(Service).filter_by(id=parent_id).first() + class CommandImporter(object): DOC_TYPE = 'CommandRunInformation' - @classmethod - def update_from_document(cls, document, workspace, level=None, couchdb_relational_map=None): + + def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): start_date = datetime.datetime.fromtimestamp(document.get('itime')) end_date = start_date + datetime.timedelta(seconds=document.get('duration')) command, instance = get_or_create( @@ -266,15 +265,12 @@ def update_from_document(cls, document, workspace, level=None, couchdb_relationa yield command - def set_parent(self, command, parent_id): - raise NotImplementedError - class NoteImporter(object): DOC_TYPE = 'Note' - @classmethod - def update_from_document(cls, document, workspace, level=None, couchdb_relational_map=None): + + def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): note = Note() note.name = document.get('name') note.text = document.get('text', None) @@ -282,17 +278,11 @@ def update_from_document(cls, document, workspace, level=None, couchdb_relationa note.owned = document.get('owned', False) yield note - def add_relationships_from_dict(self, entity, entities): - # this method is not required since update_from_document uses - # workspace name to create the relation - pass - class CredentialImporter(object): DOC_TYPE = 'Cred' - @classmethod - def update_from_document(cls, document, workspace, level=None, couchdb_relational_map=None): + def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): host = None service = None if level == 2: @@ -316,22 +306,17 @@ def update_from_document(cls, document, workspace, level=None, couchdb_relationa class WorkspaceImporter(object): DOC_TYPE = 'Workspace' - @classmethod - def update_from_document(cls, document): + def update_from_document(self, document): workspace, created = get_or_create(session, server.models.Workspace, name=document.get('name', None)) yield workspace - def add_relationships_from_dict(self, entity, entities): - for couch_id, child_entity in entities.items(): - child_entity.workspace = entity - class FaradayEntityImporter(object): # Document Types: [u'Service', u'Communication', u'Vulnerability', u'CommandRunInformation', u'Reports', u'Host', u'Workspace'] - @classmethod - def parse(cls, document): + + def parse(self, document): """Get an instance of a DAO object given a document""" - importer_class = cls.get_importer_from_document(document) + importer_class = self.get_importer_from_document(document) if importer_class is not None: importer = importer_class() entity = importer.update_from_document(document) @@ -340,8 +325,7 @@ def parse(cls, document): return importer, entity return None, None - @classmethod - def get_importer_from_document(cls, doc_type): + def get_importer_from_document(self, doc_type): logger.info('Getting class importer for {0}'.format(doc_type)) importer_class_mapper = { 'EntityMetadata': EntityMetadataImporter, @@ -355,14 +339,10 @@ def get_importer_from_document(cls, doc_type): 'Vulnerability': VulnerabilityImporter, 'VulnerabilityWeb': VulnerabilityImporter, } - importer_cls = importer_class_mapper.get(doc_type, None) - if not importer_cls: + importer_self = importer_class_mapper.get(doc_type, None) + if not importer_self: raise NotImplementedError('Class importer for {0} not implemented'.format(doc_type)) - return importer_cls - - @classmethod - def update_from_document(self, document): - raise Exception('MUST IMPLEMENT') + return importer_self class ImportCouchDB(FlaskScriptCommand): @@ -447,21 +427,12 @@ def import_workspace_into_database(self, workspace_name): couchdb_removed_objs = set() removed_objs = ['Interface'] for level, obj_type in obj_types: - obj_importer = faraday_importer.get_importer_from_document(obj_type) + obj_importer = faraday_importer.get_importer_from_document(obj_type)() objs_dict = self.get_objs(couch_url, obj_type, level) for raw_obj in tqdm(objs_dict.get('rows', [])): raw_obj = raw_obj['value'] couchdb_id = raw_obj['_id'] for new_obj in obj_importer.update_from_document(raw_obj, workspace, level, couchdb_relational_map): - if raw_obj.get('parent', None): - parent_id = raw_obj['parent'] - if parent_id in couchdb_removed_objs: - parent_id = '.'.join(parent_id.split('.')[:-1]) - obj_importer.set_parent( - new_obj, - couchdb_relational_map[parent_id], - level - ) session.commit() if obj_type not in removed_objs: couchdb_relational_map[couchdb_id] = new_obj.id From 416299eb7f2bd0b11ecd0471f87df41cff799ba5 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 30 Aug 2017 11:58:28 -0300 Subject: [PATCH 0138/1506] Fix duplicate credentials --- server/importer.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/server/importer.py b/server/importer.py index bc0d15608fe..46bf73e5be3 100644 --- a/server/importer.py +++ b/server/importer.py @@ -293,12 +293,11 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation service = session.query(Service).filter_by(id=parent_id).first() if not host and not service: raise Exception('Missing host or service for credential {0}'.format(document['_id'])) - credential, created = get_or_create(session, Credential, name=document.get('username'), host=host, service=service) - credential.username = document.get('username') - credential.password = document.get('password', '') + credential, created = get_or_create(session, Credential, username=document.get('username'), host=host, service=service) + credential.password = document.get('password', None) credential.owned = document.get('owned', False) - credential.description = document.get('description', '') - credential.name = document.get('name', '') + credential.description = document.get('description', None) + credential.name = document.get('name', None) credential.workspace = workspace yield credential From 23cfc650bf78657e200f22feafee2902f1a5a2c8 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 30 Aug 2017 12:59:59 -0300 Subject: [PATCH 0139/1506] add import verification and log diff if necessary --- server/importer.py | 43 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/server/importer.py b/server/importer.py index 46bf73e5be3..b6f701e0a2b 100644 --- a/server/importer.py +++ b/server/importer.py @@ -1,7 +1,7 @@ # Faraday Penetration Test IDE # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information - +import os import sys import json import datetime @@ -10,6 +10,7 @@ from tqdm import tqdm from flask_script import Command as FlaskScriptCommand from restkit.errors import RequestError, Unauthorized +from IPy import IP import server.app import server.utils.logger @@ -61,7 +62,6 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation yield entity - def __truncate_to_epoch_in_seconds(self, timestamp): """ In a not so elegant fashion, identifies and truncate epoch timestamps expressed in milliseconds to seconds""" @@ -156,9 +156,8 @@ def merge_with_host(self, interface, parent_relation_db_id): class ServiceImporter(object): DOC_TYPE = 'Service' - def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): - #service was always above interface, not it's above host. + # service was always above interface, not it's above host. try: parent_id = document['parent'].split('.')[0] except KeyError: @@ -189,7 +188,6 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation class VulnerabilityImporter(object): DOC_TYPE = ['Vulnerability', 'VulnerabilityWeb'] - def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): vulnerability, created = get_or_create(session, Vulnerability, name=document.get('name'), description= document.get('desc')) vulnerability.confirmed = document.get('confirmed') @@ -245,7 +243,6 @@ def set_parent(self, vulnerability, parent_id, level=2): class CommandImporter(object): DOC_TYPE = 'CommandRunInformation' - def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): start_date = datetime.datetime.fromtimestamp(document.get('itime')) end_date = start_date + datetime.timedelta(seconds=document.get('duration')) @@ -269,7 +266,6 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation class NoteImporter(object): DOC_TYPE = 'Note' - def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): note = Note() note.name = document.get('name') @@ -391,6 +387,35 @@ def get_objs(self, host, obj_type, level): return r.json() + def verify_import_data(self, couchdb_relational_map, couchdb_removed_objs, workspace): + all_docs_url = "http://{username}:{password}@{hostname}:{port}/{workspace_name}/_all_docs?include_docs=true".format( + username=server.config.couchdb.user, + password=server.config.couchdb.password, + hostname=server.config.couchdb.host, + port=server.config.couchdb.port, + workspace_name=workspace.name + ) + all_ids = map(lambda x: x['doc']['_id'], requests.get(all_docs_url).json()['rows']) + if len(all_ids) != len(couchdb_relational_map.keys()) + len(couchdb_removed_objs): + missing_objs_filename = os.path.join(os.path.expanduser('~/.faraday'), 'logs', 'import_missing_objects_{0}.json'.format(workspace.name)) + logger.warning('Not all objects were imported. Saving difference to file {0}'.format(missing_objs_filename)) + missing_ids = set(all_ids) - set(couchdb_relational_map.keys()).union(couchdb_removed_objs) + objs_diff = [] + logger.info('Downloading missing couchdb docs') + for missing_id in tqdm(missing_ids): + doc_url = 'http://{username}:{password}@{hostname}:{port}/{workspace_name}/{doc_id}'.format( + username=server.config.couchdb.user, + password=server.config.couchdb.password, + hostname=server.config.couchdb.host, + port=server.config.couchdb.port, + workspace_name=workspace.name, + doc_id=missing_id + ) + objs_diff.append(requests.get(doc_url).json()) + + with open(missing_objs_filename, 'w') as missing_objs_file: + missing_objs_file.write(json.dumps(objs_diff)) + def import_workspace_into_database(self, workspace_name): faraday_importer = FaradayEntityImporter() @@ -420,7 +445,7 @@ def import_workspace_into_database(self, workspace_name): (3, 'Vulnerability'), (3, 'VulnerabilityWeb'), (3, 'Service'), - (4, 'Credential'), # Level 4 is for interface + (4, 'Credential'), # Level 4 is for interface ] couchdb_relational_map = {} couchdb_removed_objs = set() @@ -437,5 +462,5 @@ def import_workspace_into_database(self, workspace_name): couchdb_relational_map[couchdb_id] = new_obj.id else: couchdb_removed_objs.add(couchdb_id) - + self.verify_import_data(couchdb_relational_map, couchdb_removed_objs, workspace) return created From 356a69cf2144ff800348d346410a8093babcf01c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 30 Aug 2017 15:21:46 -0300 Subject: [PATCH 0140/1506] Merge user import from couchdb --- import_users_from_couch.py | 95 -------------------------------------- server/importer.py | 94 ++++++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 96 deletions(-) delete mode 100644 import_users_from_couch.py diff --git a/import_users_from_couch.py b/import_users_from_couch.py deleted file mode 100644 index 5939a94d20c..00000000000 --- a/import_users_from_couch.py +++ /dev/null @@ -1,95 +0,0 @@ -from __future__ import print_function -import argparse -import requests -from urlparse import urljoin - -from binascii import unhexlify -from passlib.utils.binary import ab64_encode -from passlib.hash import pbkdf2_sha1 - -from flask_script import Manager -from server.web import app -from server.models import db - - -COUCHDB_USER_PREFIX = 'org.couchdb.user:' -COUCHDB_PASSWORD_PREXFIX = '-pbkdf2-' - - -def modular_crypt_pbkdf2_sha1(checksum, salt, iterations=1000): - return '$pbkdf2${iterations}${salt}${checksum}'.format( - iterations=iterations, - salt=ab64_encode(salt), - checksum=ab64_encode(unhexlify(checksum)), - ) - - -def convert_couchdb_hash(original_hash): - if not original_hash.startswith(COUCHDB_PASSWORD_PREXFIX): - # Should be a plaintext password - return original_hash - checksum, salt, iterations = original_hash[ - len(COUCHDB_PASSWORD_PREXFIX):].split(',') - iterations = int(iterations) - return modular_crypt_pbkdf2_sha1(checksum, salt, iterations) - - -def get_hash_from_document(doc): - scheme = doc.get('password_scheme', 'unset') - if scheme != 'pbkdf2': - raise ValueError('Unknown password scheme: %s' % scheme) - return modular_crypt_pbkdf2_sha1(doc['derived_key'], doc['salt'], - doc['iterations']) - - -def parse_all_docs(doc): - return [row['doc'] for row in doc['rows']] - - -def import_users(admins, all_users): - - manager = Manager(app) - with app.app_context(): - - # Import admin users - for (username, password) in admins.items(): - print("Creating user", username) - app.user_datastore.create_user( - username=username, - email=username + '@test.com', - password=convert_couchdb_hash(password), - is_ldap=False - ) - - # Import non admin users - for user in all_users: - if not user['_id'].startswith(COUCHDB_USER_PREFIX): - # It can be a view or something other than a user - continue - if user['name'] in admins.keys(): - # This is an already imported admin user, skip - continue - print("Importing", user['name']) - app.user_datastore.create_user( - username=user['name'], - email=user['name'] + '@test.com', - password=get_hash_from_document(user), - is_ldap=False - ) - db.session.commit() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('--couch-url', default='http://localhost:5984') - parser.add_argument('username') - parser.add_argument('password') - args = parser.parse_args() - - auth = (args.username, args.password) - admins_url = urljoin(args.couch_url, - '/_config/admins') - users_url = urljoin(args.couch_url, - '/_users/_all_docs?include_docs=true') - import_users(requests.get(admins_url, auth=auth).json(), - parse_all_docs(requests.get(users_url, auth=auth).json())) diff --git a/server/importer.py b/server/importer.py index b6f701e0a2b..39aeee9e20e 100644 --- a/server/importer.py +++ b/server/importer.py @@ -5,14 +5,17 @@ import sys import json import datetime +from binascii import unhexlify import requests from tqdm import tqdm from flask_script import Command as FlaskScriptCommand from restkit.errors import RequestError, Unauthorized from IPy import IP +from passlib.utils.binary import ab64_encode +from passlib.hash import pbkdf2_sha1 -import server.app +from server.web import app import server.utils.logger import server.couchdb import server.database @@ -31,6 +34,9 @@ Vulnerability ) +COUCHDB_USER_PREFIX = 'org.couchdb.user:' +COUCHDB_PASSWORD_PREXFIX = '-pbkdf2-' + logger = server.utils.logger.get_logger(__name__) session = db.session @@ -340,6 +346,90 @@ def get_importer_from_document(self, doc_type): return importer_self +class ImportCouchDBUsers(FlaskScriptCommand): + + def modular_crypt_pbkdf2_sha1(self, checksum, salt, iterations=1000): + return '$pbkdf2${iterations}${salt}${checksum}'.format( + iterations=iterations, + salt=ab64_encode(salt), + checksum=ab64_encode(unhexlify(checksum)), + ) + + def convert_couchdb_hash(self, original_hash): + if not original_hash.startswith(COUCHDB_PASSWORD_PREXFIX): + # Should be a plaintext password + return original_hash + checksum, salt, iterations = original_hash[ + len(COUCHDB_PASSWORD_PREXFIX):].split(',') + iterations = int(iterations) + return self.modular_crypt_pbkdf2_sha1(checksum, salt, iterations) + + def get_hash_from_document(self, doc): + scheme = doc.get('password_scheme', 'unset') + if scheme != 'pbkdf2': + raise ValueError('Unknown password scheme: %s' % scheme) + return self.modular_crypt_pbkdf2_sha1(doc['derived_key'], doc['salt'], + doc['iterations']) + + def parse_all_docs(self, doc): + return [row['doc'] for row in doc['rows']] + + def get_users_and_admins(self): + admins_url = "http://{username}:{password}@{hostname}:{port}/{path}".format( + username=server.config.couchdb.user, + password=server.config.couchdb.password, + hostname=server.config.couchdb.host, + port=server.config.couchdb.port, + path='_config/admins' + ) + + users_url = "http://{username}:{password}@{hostname}:{port}/{path}".format( + username=server.config.couchdb.user, + password=server.config.couchdb.password, + hostname=server.config.couchdb.host, + port=server.config.couchdb.port, + path='_users/_all_docs?include_docs=true' + ) + admins = requests.get(admins_url).json() + users = requests.get(users_url).json() + return users, admins + + def import_admins(self, admins): + # Import admin users + for (username, password) in admins.items(): + logger.info('Creating user {0}'.format(username)) + app.user_datastore.create_user( + username=username, + email=username + '@test.com', + password=self.convert_couchdb_hash(password), + is_ldap=False + ) + + def import_users(self, all_users, admins): + # Import non admin users + for user in all_users['rows']: + user = user['doc'] + if not user['_id'].startswith(COUCHDB_USER_PREFIX): + # It can be a view or something other than a user + continue + if user['name'] in admins.keys(): + # This is an already imported admin user, skip + continue + logger.info('Importing {0}'.format(user['name'])) + app.user_datastore.create_user( + username=user['name'], + email=user['name'] + '@test.com', + password=self.get_hash_from_document(user), + is_ldap=False + ) + + def run(self): + all_users, admins = self.get_users_and_admins() + self.import_users(all_users, admins) + self.import_admins(admins) + db.session.commit() + + class ImportCouchDB(FlaskScriptCommand): def _open_couchdb_conn(self): @@ -364,6 +454,8 @@ def run(self): """ Main entry point for couchdb import """ + users_import = ImportCouchDBUsers() + users_import.run() couchdb_server_conn, workspaces_list = self._open_couchdb_conn() for workspace_name in workspaces_list: From 5d404486cec9eab64fcf6b91dbc8c80f9266a2b2 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 30 Aug 2017 15:34:22 -0300 Subject: [PATCH 0141/1506] Skip user creation if already exists --- server/importer.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/server/importer.py b/server/importer.py index 39aeee9e20e..06cf495f97f 100644 --- a/server/importer.py +++ b/server/importer.py @@ -31,7 +31,8 @@ Command, Workspace, Hostname, - Vulnerability + Vulnerability, + User, ) COUCHDB_USER_PREFIX = 'org.couchdb.user:' @@ -398,12 +399,13 @@ def import_admins(self, admins): # Import admin users for (username, password) in admins.items(): logger.info('Creating user {0}'.format(username)) - app.user_datastore.create_user( - username=username, - email=username + '@test.com', - password=self.convert_couchdb_hash(password), - is_ldap=False - ) + if not db.session.query(User).filter_by(username=username).first(): + app.user_datastore.create_user( + username=username, + email=username + '@test.com', + password=self.convert_couchdb_hash(password), + is_ldap=False + ) def import_users(self, all_users, admins): # Import non admin users @@ -416,12 +418,13 @@ def import_users(self, all_users, admins): # This is an already imported admin user, skip continue logger.info('Importing {0}'.format(user['name'])) - app.user_datastore.create_user( - username=user['name'], - email=user['name'] + '@test.com', - password=self.get_hash_from_document(user), - is_ldap=False - ) + if not db.session.query(User).filter_by(username=user['name']).first(): + app.user_datastore.create_user( + username=user['name'], + email=user['name'] + '@test.com', + password=self.get_hash_from_document(user), + is_ldap=False + ) def run(self): all_users, admins = self.get_users_and_admins() From afeffd35e65105cb0f75fbcb3fa7b1810ee0a041 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 30 Aug 2017 15:51:21 -0300 Subject: [PATCH 0142/1506] Import vuln level 4 instead level 3 --- server/importer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/importer.py b/server/importer.py index 06cf495f97f..16a03364858 100644 --- a/server/importer.py +++ b/server/importer.py @@ -243,7 +243,7 @@ def set_parent(self, vulnerability, parent_id, level=2): logger.debug('Set parent for vulnerabiity level {0}'.format(level)) if level == 2: vulnerability.host = session.query(Host).filter_by(id=parent_id).first() - if level == 3: + if level == 4: vulnerability.service = session.query(Service).filter_by(id=parent_id).first() @@ -537,10 +537,10 @@ def import_workspace_into_database(self, workspace_name): (2, 'Credential'), (2, 'Vulnerability'), (2, 'VulnerabilityWeb'), - (3, 'Vulnerability'), - (3, 'VulnerabilityWeb'), (3, 'Service'), (4, 'Credential'), # Level 4 is for interface + (4, 'Vulnerability'), + (4, 'VulnerabilityWeb'), ] couchdb_relational_map = {} couchdb_removed_objs = set() From a132aa0508d8ff3a3d30c7545cdf24409f8cc936 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 31 Aug 2017 12:04:57 -0300 Subject: [PATCH 0143/1506] Add reference import --- server/importer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index 16a03364858..0c238d1aa6b 100644 --- a/server/importer.py +++ b/server/importer.py @@ -28,6 +28,7 @@ Credential, Host, Service, + Reference, Command, Workspace, Hostname, @@ -201,7 +202,6 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation vulnerability.vuln_type = document.get('type') vulnerability.data = document.get('data') vulnerability.easeofresolution = document.get('easeofresolution') - vulnerability.refs = json.dumps(document.get('refs', [])) vulnerability.resolution = document.get('resolution') vulnerability.severity = document.get('severity') vulnerability.owned = document.get('owned', False) @@ -237,8 +237,14 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation couch_parent_id = '.'.join(document['_id'].split('.')[:-1]) parent_id = couchdb_relational_map[couch_parent_id] self.set_parent(vulnerability, parent_id, level) + self.add_references(document, vulnerability, workspace) yield vulnerability + def add_references(self, document, vulnerability, workspace): + for ref in document.get('refs', []): + new_ref, created = get_or_create(session, Reference, name=ref, workspace=workspace, vulnerability=vulnerability) + + def set_parent(self, vulnerability, parent_id, level=2): logger.debug('Set parent for vulnerabiity level {0}'.format(level)) if level == 2: From 08de51dbf1f8936012216c389f52bd2f9edc229c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 31 Aug 2017 15:36:52 -0300 Subject: [PATCH 0144/1506] Add Generic view, readonly mixins and readonly host API --- requirements_server.txt | 2 + server/api/base.py | 98 +++++++++++++++++++++++++++++++++++++ server/api/modules/hosts.py | 18 +++++++ 3 files changed, 118 insertions(+) create mode 100644 server/api/base.py diff --git a/requirements_server.txt b/requirements_server.txt index 4e95cbcb3e0..56279c73f37 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -12,3 +12,5 @@ Flask-Security==3.0.0 bcrypt Flask-SQLAlchemy==2.2 Flask-Script==2.0.5 +flask-classful +marshmallow diff --git a/server/api/base.py b/server/api/base.py new file mode 100644 index 00000000000..f51bae783f6 --- /dev/null +++ b/server/api/base.py @@ -0,0 +1,98 @@ +import flask +import json + +from flask_classful import FlaskView +from sqlalchemy.orm.exc import NoResultFound +from werkzeug.routing import parse_rule +from server.models import Workspace + + +def output_json(data, code, headers=None): + content_type = 'application/json' + dumped = json.dumps(data) + if headers: + headers.update({'Content-Type': content_type}) + else: + headers = {'Content-Type': content_type} + response = flask.make_response(dumped, code, headers) + return response + + +# TODO: Require @view decorator to enable custom routes +class GenericWorkspacedView(FlaskView): + """Abstract class for a view that depends on the workspace, that is + passed in the URL""" + + # Must-implement attributes + model_class = None + schema_class = None + + # Default attributes + route_prefix = '/v2//' + base_args = ['workspace_name'] # Required to prevent double usage of + representations = {'application/json': output_json} + lookup_field = 'id' + + @classmethod + def get_route_base(cls): + """Fix issue with base_args overriding + + See https://github.com/teracyhq/flask-classful/issues/50 for + more information""" + + if cls.route_base is not None: + route_base = cls.route_base + base_rule = parse_rule(route_base) + cls.base_args += [r[2] for r in base_rule] + else: + route_base = cls.default_route_base() + + return route_base.strip("/") + + def _get_schema_class(self): + assert self.schema_class is not None, "You must define schema_class" + return self.schema_class + + def _get_lookup_field(self): + return getattr(self.model_class, self.lookup_field) + + def _get_base_query(self, workspace_name): + try: + ws = Workspace.query.filter_by(name=workspace_name).one() + except NoResultFound: + flask.abort(404, "No such workspace: %s" % workspace_name) + return self.model_class.query.join(Workspace) \ + .filter(Workspace.id==ws.id) + + def _get_object(self, workspace_name, object_id): + try: + obj = self._get_base_query(workspace_name).filter( + self._get_lookup_field() == object_id).one() + except NoResultFound: + flask.abort(404, 'Object with id "%s" not found' % object_id) + return obj + + def _dump(self, obj, **kwargs): + return self._get_schema_class()(**kwargs).dump(obj) + + +class ListWorkspacedMixin(object): + """Add GET // route""" + + def index(self, workspace_name): + return self._dump(self._get_base_query(workspace_name).all(), + many=True) + + +class RetrieveWorkspacedMixin(object): + """Add GET /// route""" + + def get(self, workspace_name, object_id): + return self._dump(self._get_object(workspace_name, object_id)) + + +class ReadOnlyWorkspacedView(GenericWorkspacedView, + ListWorkspacedMixin, + RetrieveWorkspacedMixin): + """A generic view with list and retrieve endpoints""" + pass diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index ea13cf8d90c..42c9d1bbb2d 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -4,15 +4,33 @@ import flask from flask import Blueprint +from marshmallow import Schema, fields from server.utils.logger import get_logger from server.utils.web import gzipped, validate_workspace,\ get_integer_parameter, filter_request_args from server.dao.host import HostDAO +from server.api.base import ReadOnlyWorkspacedView +from server.models import Host host_api = Blueprint('host_api', __name__) +class HostSchema(Schema): + id = fields.String(required=True, dump_only=True) + ip = fields.String(required=True) + description = fields.String(required=True) + os = fields.String() + + +class HostsView(ReadOnlyWorkspacedView): + route_base = 'hosts' + model_class = Host + schema_class = HostSchema + +HostsView.register(host_api) + + @gzipped @host_api.route('/ws//hosts', methods=['GET']) def list_hosts(workspace=None): From c17624912fc3deb5b692d97efe7b6a535e569ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 1 Sep 2017 15:12:41 -0300 Subject: [PATCH 0145/1506] Fix tests and factories Fix integration with pytest_factoryboy (register fixtures in conftest), remove interface factory, don't use id field in factories, add workspacefactory abstract class --- test_cases/conftest.py | 10 ++++++++++ test_cases/factories.py | 43 +++++++++++++---------------------------- 2 files changed, 23 insertions(+), 30 deletions(-) diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 4e2737c9145..7945acfc558 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -1,10 +1,20 @@ import os import sys import pytest +from pytest_factoryboy import register sys.path.append(os.path.abspath(os.getcwd())) from server.app import create_app from server.models import db +from test_cases import factories + + + +register(factories.WorkspaceFactory) +register(factories.HostFactory) +register(factories.ServiceFactory) +register(factories.VulnerabilityFactory) +register(factories.CredentialFactory) @pytest.fixture(scope='session') diff --git a/test_cases/factories.py b/test_cases/factories.py index 195ba120cc4..e6001554460 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -3,13 +3,11 @@ FuzzyText, FuzzyChoice ) -from pytest_factoryboy import register from server.models import ( db, Host, Command, Service, - Interface, Workspace, Credential, Vulnerability, @@ -19,7 +17,8 @@ class FaradayFactory(factory.alchemy.SQLAlchemyModelFactory): - id = factory.Sequence(lambda n: n) + # id = factory.Sequence(lambda n: n) + pass class WorkspaceFactory(FaradayFactory): @@ -31,8 +30,12 @@ class Meta: sqlalchemy_session = db.session -class HostFactory(FaradayFactory): - name = FuzzyText() +class WorkspaceObjectFactory(FaradayFactory): + workspace = factory.SubFactory(WorkspaceFactory) + + +class HostFactory(WorkspaceObjectFactory): + ip = factory.Faker('ipv4') description = FuzzyText() os = FuzzyChoice(['Linux', 'Windows', 'OSX', 'Android', 'iOS']) @@ -41,7 +44,7 @@ class Meta: sqlalchemy_session = db.session -class EntityMetadataFactory(FaradayFactory): +class EntityMetadataFactory(WorkspaceObjectFactory): couchdb_id = factory.Sequence(lambda n: '{0}.1.2'.format(n)) class Meta: @@ -49,22 +52,10 @@ class Meta: sqlalchemy_session = db.session -class InterfaceFactory(FaradayFactory): - name = FuzzyText() - description = FuzzyText() - mac = FuzzyText() - host = factory.SubFactory(HostFactory) - - class Meta: - model = Interface - sqlalchemy_session = db.session - - -class ServiceFactory(FaradayFactory): +class ServiceFactory(WorkspaceObjectFactory): name = FuzzyText() description = FuzzyText() ports = FuzzyChoice(['443', '80', '22']) - interface = factory.SubFactory(InterfaceFactory) host = factory.SubFactory(HostFactory) class Meta: @@ -72,7 +63,7 @@ class Meta: sqlalchemy_session = db.session -class VulnerabilityFactory(FaradayFactory): +class VulnerabilityFactory(WorkspaceObjectFactory): name = FuzzyText() description = FuzzyText() @@ -90,7 +81,7 @@ class Meta: sqlalchemy_session = db.session -class CredentialFactory(FaradayFactory): +class CredentialFactory(WorkspaceObjectFactory): username = FuzzyText() password = FuzzyText() @@ -99,17 +90,9 @@ class Meta: sqlalchemy_session = db.session -class CommandFactory(FaradayFactory): +class CommandFactory(WorkspaceObjectFactory): command = FuzzyText() class Meta: model = Command sqlalchemy_session = db.session - - -register(WorkspaceFactory) -register(HostFactory) -register(ServiceFactory) -register(InterfaceFactory) -register(VulnerabilityFactory) -register(CredentialFactory) From d369221ed5df2298f5a9bb8b3e2a1598edb5d5a5 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 1 Sep 2017 16:23:07 -0300 Subject: [PATCH 0146/1506] Update importer to use the new model --- server/importer.py | 136 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 120 insertions(+), 16 deletions(-) diff --git a/server/importer.py b/server/importer.py index 0c238d1aa6b..53c168112cb 100644 --- a/server/importer.py +++ b/server/importer.py @@ -33,7 +33,14 @@ Workspace, Hostname, Vulnerability, + VulnerabilityWeb, User, + PolicyViolation, + Task, + TaskTemplate, + Methodology, + MethodologyTemplate, + ExecutiveReport ) COUCHDB_USER_PREFIX = 'org.couchdb.user:' @@ -197,27 +204,34 @@ class VulnerabilityImporter(object): DOC_TYPE = ['Vulnerability', 'VulnerabilityWeb'] def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): - vulnerability, created = get_or_create(session, Vulnerability, name=document.get('name'), description= document.get('desc')) + vuln_class = Vulnerability + if document['type'] == 'VulnerabilityWeb': + vuln_class = VulnerabilityWeb + vulnerability, created = get_or_create( + session, + vuln_class, + name=document.get('name'), + description=document.get('desc') + ) vulnerability.confirmed = document.get('confirmed') - vulnerability.vuln_type = document.get('type') vulnerability.data = document.get('data') vulnerability.easeofresolution = document.get('easeofresolution') vulnerability.resolution = document.get('resolution') vulnerability.severity = document.get('severity') vulnerability.owned = document.get('owned', False) - vulnerability.attachments = json.dumps(document.get('_attachments', {})) - vulnerability.policyviolations = json.dumps(document.get('policyviolations', [])) + #vulnerability.attachments = json.dumps(document.get('_attachments', {})) vulnerability.impact_accountability = document.get('impact', {}).get('accountability') vulnerability.impact_availability = document.get('impact', {}).get('availability') vulnerability.impact_confidentiality = document.get('impact', {}).get('confidentiality') vulnerability.impact_integrity = document.get('impact', {}).get('integrity') - vulnerability.method = document.get('method') - vulnerability.path = document.get('path') - vulnerability.pname = document.get('pname') - vulnerability.query = document.get('query') - vulnerability.request = document.get('request') - vulnerability.response = document.get('response') - vulnerability.website = document.get('website') + if document['type'] == 'VulnerabilityWeb': + vulnerability.method = document.get('method') + vulnerability.path = document.get('path') + vulnerability.pname = document.get('pname') + vulnerability.query = document.get('query') + vulnerability.request = document.get('request') + vulnerability.response = document.get('response') + vulnerability.website = document.get('website') status_map = { 'opened': 'open', 'closed': 'closed', @@ -238,19 +252,35 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation parent_id = couchdb_relational_map[couch_parent_id] self.set_parent(vulnerability, parent_id, level) self.add_references(document, vulnerability, workspace) + self.add_policy_violations(document, vulnerability, workspace) yield vulnerability + def add_policy_violations(self, document, vulnerability, workspace): + for policy_violation in document.get('policyviolations', []): + get_or_create( + session, + PolicyViolation, + name=policy_violation, + workspace=workspace, + vulnerability=vulnerability + ) + def add_references(self, document, vulnerability, workspace): for ref in document.get('refs', []): - new_ref, created = get_or_create(session, Reference, name=ref, workspace=workspace, vulnerability=vulnerability) - + get_or_create( + session, + Reference, + name=ref, + workspace=workspace, + vulnerability=vulnerability + ) def set_parent(self, vulnerability, parent_id, level=2): logger.debug('Set parent for vulnerabiity level {0}'.format(level)) if level == 2: - vulnerability.host = session.query(Host).filter_by(id=parent_id).first() + vulnerability.host_id = session.query(Host).filter_by(id=parent_id).first().id if level == 4: - vulnerability.service = session.query(Service).filter_by(id=parent_id).first() + vulnerability.service_id = session.query(Service).filter_by(id=parent_id).first().id class CommandImporter(object): @@ -314,11 +344,77 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation class WorkspaceImporter(object): DOC_TYPE = 'Workspace' - def update_from_document(self, document): + def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): workspace, created = get_or_create(session, server.models.Workspace, name=document.get('name', None)) + workspace.description = document.get('description') + workspace.start_date = datetime.datetime.fromtimestamp(document.get('duration')['start']/1000) + workspace.end_date = datetime.datetime.fromtimestamp(document.get('duration')['end']/1000) + workspace.scope = document.get('scope') yield workspace +class MethodologyImporter(object): + def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): + if document.get('group_type') == 'template': + methodology, created = get_or_create(session, MethodologyTemplate, name=document.get('name')) + + yield methodology + + if document.get('group_type') == 'instance': + methodology, created = get_or_create(session, Methodology, name=document.get('name')) + methodology.workspace = workspace + yield methodology +# methodology. + + +class TaskImporter(object): + + def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): + methodology_id = couchdb_relational_map[document.get('group_id')] + methodology = session.query(Methodology).filter_by(id=methodology_id).first() + task_class = Task + if not methodology: + methodology = session.query(MethodologyTemplate).filter_by(id=methodology_id).first() + task_class = TaskTemplate + task, created = get_or_create(session, task_class, name=document.get('name')) + if task_class == TaskTemplate: + task.template = methodology + else: + task.methodology = methodology + task.description = document.get('description') + task.assigned_to = session.query(User).filter_by(username=document.get('username')).first() + mapped_status = { + 'New': 'new', + 'In Progress': 'in progress', + 'Review': 'review', + 'Completed': 'completed' + } + task.status = mapped_status[document.get('status')] + #tags + #task.due_date = datetime.datetime.fromtimestamp(document.get('due_date')) + yield task + + +class ReportsImporter(object): + + def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): + report, created = get_or_create(session, ExecutiveReport, name=document.get('name')) + report.template_name = document.get('name') + report.title = document.get('title') + report.status = document.get('status') + # TODO: add tags + report.conclusions = document.get('conclusions') + report.summary = document.get('summary') + report.recommendations = document.get('recommendations') + report.enterprise = document.get('enterprise') + report.summary = document.get('summary') + report.scope = document.get('scope') + report.objectives = document.get('objectives') + report.grouped = document.get('grouped') + report.workspace = workspace + yield report + + class FaradayEntityImporter(object): # Document Types: [u'Service', u'Communication', u'Vulnerability', u'CommandRunInformation', u'Reports', u'Host', u'Workspace'] @@ -346,6 +442,9 @@ def get_importer_from_document(self, doc_type): 'Workspace': WorkspaceImporter, 'Vulnerability': VulnerabilityImporter, 'VulnerabilityWeb': VulnerabilityImporter, + 'TaskGroup': MethodologyImporter, + 'Task': TaskImporter, + 'Reports': ReportsImporter, } importer_self = importer_class_mapper.get(doc_type, None) if not importer_self: @@ -538,6 +637,10 @@ def import_workspace_into_database(self, workspace_name): (1, 'EntityMetadata'), (1, 'Note'), (1, 'CommandRunInformation'), + (1, 'TaskGroup'), + (1, 'Task'), + (1, 'Workspace'), + (1, 'Reports'), (2, 'Interface'), (2, 'Service'), (2, 'Credential'), @@ -557,6 +660,7 @@ def import_workspace_into_database(self, workspace_name): for raw_obj in tqdm(objs_dict.get('rows', [])): raw_obj = raw_obj['value'] couchdb_id = raw_obj['_id'] + for new_obj in obj_importer.update_from_document(raw_obj, workspace, level, couchdb_relational_map): session.commit() if obj_type not in removed_objs: From ef6c52a190903e5f488ca07c36d4f6c40909cd9c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 1 Sep 2017 16:31:40 -0300 Subject: [PATCH 0147/1506] If report status was error don't save grouped. Add check when timestamps were empty --- server/importer.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/server/importer.py b/server/importer.py index 53c168112cb..55072637d33 100644 --- a/server/importer.py +++ b/server/importer.py @@ -347,8 +347,10 @@ class WorkspaceImporter(object): def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): workspace, created = get_or_create(session, server.models.Workspace, name=document.get('name', None)) workspace.description = document.get('description') - workspace.start_date = datetime.datetime.fromtimestamp(document.get('duration')['start']/1000) - workspace.end_date = datetime.datetime.fromtimestamp(document.get('duration')['end']/1000) + if document.get('duration')['start']: + workspace.start_date = datetime.datetime.fromtimestamp(float(document.get('duration')['start'])/1000) + if document.get('duration')['end']: + workspace.end_date = datetime.datetime.fromtimestamp(float(document.get('duration')['end'])/1000) workspace.scope = document.get('scope') yield workspace @@ -410,7 +412,8 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation report.summary = document.get('summary') report.scope = document.get('scope') report.objectives = document.get('objectives') - report.grouped = document.get('grouped') + if report.status != 'error': + report.grouped = document.get('grouped') report.workspace = workspace yield report From f5a1b72277de812e4cf747959bf3a6f96a466121 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 1 Sep 2017 16:43:34 -0300 Subject: [PATCH 0148/1506] Filter some couchdb documents --- server/importer.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/importer.py b/server/importer.py index 55072637d33..d3ffac5be9c 100644 --- a/server/importer.py +++ b/server/importer.py @@ -601,7 +601,6 @@ def verify_import_data(self, couchdb_relational_map, couchdb_removed_objs, works all_ids = map(lambda x: x['doc']['_id'], requests.get(all_docs_url).json()['rows']) if len(all_ids) != len(couchdb_relational_map.keys()) + len(couchdb_removed_objs): missing_objs_filename = os.path.join(os.path.expanduser('~/.faraday'), 'logs', 'import_missing_objects_{0}.json'.format(workspace.name)) - logger.warning('Not all objects were imported. Saving difference to file {0}'.format(missing_objs_filename)) missing_ids = set(all_ids) - set(couchdb_relational_map.keys()).union(couchdb_removed_objs) objs_diff = [] logger.info('Downloading missing couchdb docs') @@ -614,7 +613,13 @@ def verify_import_data(self, couchdb_relational_map, couchdb_removed_objs, works workspace_name=workspace.name, doc_id=missing_id ) - objs_diff.append(requests.get(doc_url).json()) + not_imported_obj = requests.get(doc_url).json() + filter_keys = ['views', 'validate_doc_update'] + if not any(map(lambda x: x not in filter_keys,not_imported_obj.keys())): + # we filter custom views, validation funcs, etc + logger.warning( + 'Not all objects were imported. Saving difference to file {0}'.format(missing_objs_filename)) + objs_diff.append() with open(missing_objs_filename, 'w') as missing_objs_file: missing_objs_file.write(json.dumps(objs_diff)) From d89354622ce426993bd0ec7dab256566a71850a5 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 1 Sep 2017 18:42:38 -0300 Subject: [PATCH 0149/1506] Minor fixes to support import old couchdb documents --- server/importer.py | 28 ++++++++++++++++------------ server/models.py | 2 +- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/server/importer.py b/server/importer.py index d3ffac5be9c..a00f3c85dd1 100644 --- a/server/importer.py +++ b/server/importer.py @@ -191,7 +191,8 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation status_mapper = { 'open': 'open', 'closed': 'closed', - 'filtered': 'filtered' + 'filtered': 'filtered', + 'open|filtered': 'filtered' } service.status = status_mapper[document.get('status')] service.version = document.get('version') @@ -213,7 +214,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation name=document.get('name'), description=document.get('desc') ) - vulnerability.confirmed = document.get('confirmed') + vulnerability.confirmed = document.get('confirmed', False) vulnerability.data = document.get('data') vulnerability.easeofresolution = document.get('easeofresolution') vulnerability.resolution = document.get('resolution') @@ -288,14 +289,16 @@ class CommandImporter(object): def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): start_date = datetime.datetime.fromtimestamp(document.get('itime')) - end_date = start_date + datetime.timedelta(seconds=document.get('duration')) + command, instance = get_or_create( session, Command, command=document.get('command', None), start_date=start_date, - end_date=end_date ) + if document.get('duration'): + command.end_date = start_date + datetime.timedelta(seconds=document.get('duration')) + command.command = document.get('command', None) command.ip = document.get('ip', None) command.hostname = document.get('hostname', None) @@ -401,7 +404,7 @@ class ReportsImporter(object): def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): report, created = get_or_create(session, ExecutiveReport, name=document.get('name')) - report.template_name = document.get('name') + report.template_name = document.get('template_name', 'generic_default.docx') report.title = document.get('title') report.status = document.get('status') # TODO: add tags @@ -412,8 +415,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation report.summary = document.get('summary') report.scope = document.get('scope') report.objectives = document.get('objectives') - if report.status != 'error': - report.grouped = document.get('grouped') + report.grouped = document.get('grouped', False) report.workspace = workspace yield report @@ -476,7 +478,9 @@ def convert_couchdb_hash(self, original_hash): def get_hash_from_document(self, doc): scheme = doc.get('password_scheme', 'unset') if scheme != 'pbkdf2': - raise ValueError('Unknown password scheme: %s' % scheme) + # Flask Security will encrypt the password next time the user logs in. + logger.warning('Found user {0} without password.'.format(doc.get('name'))) + return 'changeme' return self.modular_crypt_pbkdf2_sha1(doc['derived_key'], doc['salt'], doc['iterations']) @@ -615,14 +619,14 @@ def verify_import_data(self, couchdb_relational_map, couchdb_removed_objs, works ) not_imported_obj = requests.get(doc_url).json() filter_keys = ['views', 'validate_doc_update'] - if not any(map(lambda x: x not in filter_keys,not_imported_obj.keys())): + if not any(map(lambda x: x not in filter_keys, not_imported_obj.keys())): # we filter custom views, validation funcs, etc logger.warning( 'Not all objects were imported. Saving difference to file {0}'.format(missing_objs_filename)) - objs_diff.append() + objs_diff.append(not_imported_obj) - with open(missing_objs_filename, 'w') as missing_objs_file: - missing_objs_file.write(json.dumps(objs_diff)) + with open(missing_objs_filename, 'w') as missing_objs_file: + missing_objs_file.write(json.dumps(objs_diff)) def import_workspace_into_database(self, workspace_name): diff --git a/server/models.py b/server/models.py index 82745c148fa..4d9f231fdb1 100644 --- a/server/models.py +++ b/server/models.py @@ -410,7 +410,7 @@ class Command(db.Model): id = Column(Integer, primary_key=True) command = Column(Text(), nullable=False) start_date = Column(DateTime, nullable=False) - end_date = Column(DateTime, nullable=False) + end_date = Column(DateTime, nullable=True) ip = Column(String(250), nullable=False) # where the command was executed hostname = Column(String(250), nullable=False) # where the command was executed params = Column(Text(), nullable=True) From 53b3705e9ac73a4c84998bee4db6fa5c5594704d Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 4 Sep 2017 13:48:25 -0300 Subject: [PATCH 0150/1506] Import fixes to support old dbs --- server/importer.py | 19 +++++++++++++------ server/models.py | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/server/importer.py b/server/importer.py index a00f3c85dd1..88284f52d52 100644 --- a/server/importer.py +++ b/server/importer.py @@ -214,7 +214,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation name=document.get('name'), description=document.get('desc') ) - vulnerability.confirmed = document.get('confirmed', False) + vulnerability.confirmed = document.get('confirmed', False) or False vulnerability.data = document.get('data') vulnerability.easeofresolution = document.get('easeofresolution') vulnerability.resolution = document.get('resolution') @@ -350,9 +350,9 @@ class WorkspaceImporter(object): def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): workspace, created = get_or_create(session, server.models.Workspace, name=document.get('name', None)) workspace.description = document.get('description') - if document.get('duration')['start']: + if document.get('duration') and document.get('duration')['start']: workspace.start_date = datetime.datetime.fromtimestamp(float(document.get('duration')['start'])/1000) - if document.get('duration')['end']: + if document.get('duration') and document.get('duration')['end']: workspace.end_date = datetime.datetime.fromtimestamp(float(document.get('duration')['end'])/1000) workspace.scope = document.get('scope') yield workspace @@ -362,7 +362,7 @@ class MethodologyImporter(object): def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): if document.get('group_type') == 'template': methodology, created = get_or_create(session, MethodologyTemplate, name=document.get('name')) - + methodology.workspace = workspace yield methodology if document.get('group_type') == 'instance': @@ -375,7 +375,11 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation class TaskImporter(object): def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): - methodology_id = couchdb_relational_map[document.get('group_id')] + try: + methodology_id = couchdb_relational_map[document.get('group_id')] + except KeyError: + logger.warn('Could not found methodology with id {0}'.format(document.get('group_id'))) + return [] methodology = session.query(Methodology).filter_by(id=methodology_id).first() task_class = Task if not methodology: @@ -395,9 +399,10 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation 'Completed': 'completed' } task.status = mapped_status[document.get('status')] + task.workspace = workspace #tags #task.due_date = datetime.datetime.fromtimestamp(document.get('due_date')) - yield task + return [task] class ReportsImporter(object): @@ -674,6 +679,8 @@ def import_workspace_into_database(self, workspace_name): couchdb_id = raw_obj['_id'] for new_obj in obj_importer.update_from_document(raw_obj, workspace, level, couchdb_relational_map): + if not new_obj: + continue session.commit() if obj_type not in removed_objs: couchdb_relational_map[couchdb_id] = new_obj.id diff --git a/server/models.py b/server/models.py index 4d9f231fdb1..a5173c7e5a3 100644 --- a/server/models.py +++ b/server/models.py @@ -616,7 +616,7 @@ class Task(TaskABC): ) workspace = relationship('Workspace', backref='tasks') - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) class License(db.Model): From fe51a9d68c80d8bfa61d32b3903943b648c5964f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 4 Sep 2017 14:24:05 -0300 Subject: [PATCH 0151/1506] Fix ugly pytest stuff breaking the tests --- server/api/base.py | 7 ++++-- test_cases/conftest.py | 35 +++++++++++++++++++----------- test_cases/test_server_api.py | 40 +++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 14 deletions(-) create mode 100644 test_cases/test_server_api.py diff --git a/server/api/base.py b/server/api/base.py index f51bae783f6..d8bd2aa41fe 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -56,13 +56,16 @@ def _get_schema_class(self): def _get_lookup_field(self): return getattr(self.model_class, self.lookup_field) - def _get_base_query(self, workspace_name): + def _get_workspace(self, workspace_name): try: ws = Workspace.query.filter_by(name=workspace_name).one() except NoResultFound: flask.abort(404, "No such workspace: %s" % workspace_name) + return ws + + def _get_base_query(self, workspace_name): return self.model_class.query.join(Workspace) \ - .filter(Workspace.id==ws.id) + .filter(Workspace.id==self._get_workspace(workspace_name).id) def _get_object(self, workspace_name, object_id): try: diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 7945acfc558..6834d4ae747 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -10,11 +10,15 @@ -register(factories.WorkspaceFactory) -register(factories.HostFactory) -register(factories.ServiceFactory) -register(factories.VulnerabilityFactory) -register(factories.CredentialFactory) +enabled_factories = [ + factories.WorkspaceFactory, + factories.HostFactory, + factories.ServiceFactory, + factories.VulnerabilityFactory, + factories.CredentialFactory, +] +for factory in enabled_factories: + register(factory) @pytest.fixture(scope='session') @@ -50,14 +54,20 @@ def teardown(): @pytest.fixture(scope='function') def session(database, request): - connection = db.engine.connect() + connection = database.engine.connect() transaction = connection.begin() options = {"bind": connection, 'binds': {}} session = db.create_scoped_session(options=options) + database.session = session db.session = session + for factory in enabled_factories: + factory._meta.sqlalchemy_session = session + + + def teardown(): transaction.rollback() connection.close() @@ -71,24 +81,25 @@ def test_client(app): return app.test_client() -def create_user(app, username, email, password, **kwargs): +def create_user(app, session, username, email, password, **kwargs): user = app.user_datastore.create_user(username=username, email=email, password=password, **kwargs) - db.session.add(user) - db.session.commit() + session.add(user) + session.commit() return user @pytest.fixture -def user(app, session): - return create_user(app, 'test', 'user@test.com', 'password', is_ldap=False) +def user(app, database, session): + # print 'user', id(session), session + return create_user(app, session, 'test', 'user@test.com', 'password', is_ldap=False) @pytest.fixture def ldap_user(app, session): - return create_user(app, 'ldap', 'ldap@test.com', 'password', is_ldap=True) + return create_user(app, session, 'ldap', 'ldap@test.com', 'password', is_ldap=True) def login_as(test_client, user): diff --git a/test_cases/test_server_api.py b/test_cases/test_server_api.py new file mode 100644 index 00000000000..14fed270395 --- /dev/null +++ b/test_cases/test_server_api.py @@ -0,0 +1,40 @@ +import pytest +from json import loads as decode_json +from test_cases import factories +from server.models import db, Workspace + +PREFIX = '/v2/' +HOSTS_COUNT = 5 + +@pytest.mark.usefixtures('database', 'logged_user') +class TestHostAPI: + + @pytest.fixture(autouse=True) + def load_workspace_with_hosts(self, request, database, session, workspace, host_factory): + host_factory.create_batch(HOSTS_COUNT, workspace=workspace) + database.session.commit() + assert workspace.id is not None + assert workspace.hosts[0].id is not None + self.workspace = workspace + return workspace + + def url(self, host=None, workspace=None): + workspace = workspace or self.workspace + url = PREFIX + workspace.name + '/hosts/' + if host is not None: + url += str(host.id) + return url + + def test_list_retrieves_all_items(self, test_client): + res = test_client.get(self.url()) + assert res.status_code == 200 + assert len(decode_json(res.data)) == HOSTS_COUNT + + def test_retrieve_one_host(self, test_client, database): + # self.workspace = Workspace.query.first() + host = self.workspace.hosts[0] + assert host.id is not None + res = test_client.get(self.url(host)) + assert res.status_code == 200 + assert decode_json(res.data)['ip'] == host.ip + From ddadf91f6940dd1e329a1aaba65a14fd2f42829d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 4 Sep 2017 14:58:14 -0300 Subject: [PATCH 0152/1506] Add custom flask client test class To easily use JSON requests. Also add test for retrieving a host from other workspace --- test_cases/conftest.py | 13 +++++++++++++ test_cases/test_server_api.py | 16 ++++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 6834d4ae747..4decffbf2e8 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -1,6 +1,8 @@ import os import sys +import json import pytest +from flask.testing import FlaskClient from pytest_factoryboy import register sys.path.append(os.path.abspath(os.getcwd())) @@ -21,11 +23,22 @@ register(factory) +class CustomClient(FlaskClient): + def open(self, *args, **kwargs): + ret = super(CustomClient, self).open(*args, **kwargs) + try: + ret.json = json.loads(ret.data) + except ValueError: + ret.json = None + return ret + + @pytest.fixture(scope='session') def app(request): # we use sqlite memory for tests test_conn_string = 'sqlite://' app = create_app(db_connection_string=test_conn_string, testing=True) + app.test_client_class = CustomClient # Establish an application context before running the tests. ctx = app.app_context() diff --git a/test_cases/test_server_api.py b/test_cases/test_server_api.py index 14fed270395..86cd14622a5 100644 --- a/test_cases/test_server_api.py +++ b/test_cases/test_server_api.py @@ -1,5 +1,4 @@ import pytest -from json import loads as decode_json from test_cases import factories from server.models import db, Workspace @@ -10,7 +9,7 @@ class TestHostAPI: @pytest.fixture(autouse=True) - def load_workspace_with_hosts(self, request, database, session, workspace, host_factory): + def load_workspace_with_hosts(self, database, session, workspace, host_factory): host_factory.create_batch(HOSTS_COUNT, workspace=workspace) database.session.commit() assert workspace.id is not None @@ -28,7 +27,7 @@ def url(self, host=None, workspace=None): def test_list_retrieves_all_items(self, test_client): res = test_client.get(self.url()) assert res.status_code == 200 - assert len(decode_json(res.data)) == HOSTS_COUNT + assert len(res.json) == HOSTS_COUNT def test_retrieve_one_host(self, test_client, database): # self.workspace = Workspace.query.first() @@ -36,5 +35,14 @@ def test_retrieve_one_host(self, test_client, database): assert host.id is not None res = test_client.get(self.url(host)) assert res.status_code == 200 - assert decode_json(res.data)['ip'] == host.ip + assert res.json['ip'] == host.ip + + def test_retrieve_fails_with_host_of_another_workspace(self, + test_client, + session, + workspace_factory): + new = workspace_factory.create() + session.commit() + res = test_client.get(self.url(self.workspace.hosts[0], new)) + assert res.status_code == 404 From f9ed41c052d44015c2bb80b9dbcb9d202ee49d8a Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 4 Sep 2017 15:44:25 -0300 Subject: [PATCH 0153/1506] Add vulnerability template importer and some fixes and add nullalbe=False in some relations. minor fixes on importer to proper data import. --- server/commands/faraday_schema_display.py | 4 ++ server/importer.py | 80 ++++++++++++++++++----- server/models.py | 23 ++++--- 3 files changed, 82 insertions(+), 25 deletions(-) diff --git a/server/commands/faraday_schema_display.py b/server/commands/faraday_schema_display.py index 0b134acf51f..5bebf7ef529 100644 --- a/server/commands/faraday_schema_display.py +++ b/server/commands/faraday_schema_display.py @@ -19,6 +19,10 @@ def run(self): self._draw_entity_diagrama() self._draw_uml_class_diagram() + @property + def description(self): + return 'Generates an entity diagram and uml class diagram from the implemented model' + def _draw_entity_diagrama(self): # create the pydot graph object by autoloading all tables via a bound metadata object graph = create_schema_graph( diff --git a/server/importer.py b/server/importer.py index 88284f52d52..6852170c932 100644 --- a/server/importer.py +++ b/server/importer.py @@ -9,6 +9,7 @@ import requests from tqdm import tqdm +from IPy import IP from flask_script import Command as FlaskScriptCommand from restkit.errors import RequestError, Unauthorized from IPy import IP @@ -40,7 +41,9 @@ TaskTemplate, Methodology, MethodologyTemplate, - ExecutiveReport + ExecutiveReport, + VulnerabilityTemplate, + ReferenceTemplate, ) COUCHDB_USER_PREFIX = 'org.couchdb.user:' @@ -92,7 +95,12 @@ class HostImporter(object): def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): # Ticket #3387: if the 'os' field is None, we default to 'unknown' - host, created = get_or_create(session, Host, ip=document.get('name')) + try: + IP(document.get('name')) # this will raise ValueError on valid IPs + host, created = get_or_create(session, Host, ip=document.get('name')) + except ValueError: + host, created = get_or_create(session, Host, ip=document.get('ip')) + if not document.get('os'): document['os'] = 'unknown' @@ -101,8 +109,8 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation host.name = document.get('name') host.description = document.get('description') host.os = document.get('os') - host.default_gateway_ip = default_gateway and default_gateway[0] or '' - host.default_gateway_mac = default_gateway and default_gateway[1] or '' + host.default_gateway_ip = default_gateway and default_gateway[0] + host.default_gateway_mac = default_gateway and default_gateway[1] host.owned = document.get('owned', False) host.workspace = workspace yield host @@ -118,7 +126,7 @@ class InterfaceImporter(object): DOC_TYPE = 'Interface' def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): - + # interface dict is used to merge interface with host and connect it the services to the host interface = {} interface['name'] = document.get('name') interface['description'] = document.get('description') @@ -143,7 +151,8 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation if not couch_parent_id: couch_parent_id = '.'.join(document['_id'].split('.')[:-1]) parent_id = couchdb_relational_map[couch_parent_id] - self.merge_with_host(interface, parent_id) + interface['parent_id'] = parent_id + host = self.merge_with_host(interface, parent_id) yield interface def merge_with_host(self, interface, parent_relation_db_id): @@ -161,11 +170,12 @@ def merge_with_host(self, interface, parent_relation_db_id): if interface['network_segment']: host.net_segment = interface['network_segment'] if interface['description']: - host.description += '\n {0}'.format(interface['description']) + host.description += '\n Interface data: {0}'.format(interface['description']) for hostname in interface['hostnames']: hostname, created = get_or_create(session, Hostname, name=hostname, host=host) host.owned = host.owned or interface['owned'] + return host class ServiceImporter(object): @@ -180,10 +190,14 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation parent_id = document['_id'].split('.')[0] host, created = get_or_create(session, Host, id=couchdb_relational_map[parent_id]) for port in document.get('ports'): - service, created = get_or_create(session, Service, name=document.get('name'), port=port) - service.name = document.get('name') + service, created = get_or_create(session, + Service, + name=document.get('name'), + port=port, + host=host) service.description = document.get('description') service.owned = document.get('owned', False) + service.banner = document.get('banner') service.protocol = document.get('protocol') if not document.get('status'): logger.warning('Service {0} with empty status. Using open as status'.format(document['_id'])) @@ -233,6 +247,11 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation vulnerability.request = document.get('request') vulnerability.response = document.get('response') vulnerability.website = document.get('website') + params = document.get('params', u'') + if isinstance(params, (list, tuple)): + vulnerability.parameters = (u' '.join(params)).strip() + else: + vulnerability.parameters = params if params is not None else u'' status_map = { 'opened': 'open', 'closed': 'closed', @@ -241,11 +260,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation vulnerability.status = status vulnerability.workspace = workspace - params = document.get('params', u'') - if isinstance(params, (list, tuple)): - vulnerability.params = (u' '.join(params)).strip() - else: - vulnerability.params = params if params is not None else u'' + couch_parent_id = document.get('parent', None) if not couch_parent_id: @@ -390,6 +405,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation task.template = methodology else: task.methodology = methodology + task.workspace = workspace task.description = document.get('description') task.assigned_to = session.query(User).filter_by(username=document.get('username')).first() mapped_status = { @@ -399,7 +415,6 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation 'Completed': 'completed' } task.status = mapped_status[document.get('status')] - task.workspace = workspace #tags #task.due_date = datetime.datetime.fromtimestamp(document.get('due_date')) return [task] @@ -550,6 +565,38 @@ def run(self): db.session.commit() + +class ImportVulnerabilityTemplates(FlaskScriptCommand): + + def run(self): + cwe_url = "http://{username}:{password}@{hostname}:{port}/{path}".format( + username=server.config.couchdb.user, + password=server.config.couchdb.password, + hostname=server.config.couchdb.host, + port=server.config.couchdb.port, + path='cwe/_all_docs?include_docs=true' + ) + cwes = requests.get(cwe_url).json()['rows'] + for cwe in cwes: + document = cwe['doc'] + vuln_template, created = get_or_create(session, + VulnerabilityTemplate, + name=document.get('name'), + severity=document.get('exploitation'), + description=document.get('description')) + vuln_template.resolution = document.get('resolution') + for ref_doc in document['references']: + get_or_create(session, + ReferenceTemplate, + vulnerability=vuln_template, + name=ref_doc) + + + + + + + class ImportCouchDB(FlaskScriptCommand): def _open_couchdb_conn(self): @@ -574,6 +621,8 @@ def run(self): """ Main entry point for couchdb import """ + vuln_templates_import = ImportVulnerabilityTemplates() + vuln_templates_import.run() users_import = ImportCouchDBUsers() users_import.run() couchdb_server_conn, workspaces_list = self._open_couchdb_conn() @@ -685,6 +734,7 @@ def import_workspace_into_database(self, workspace_name): if obj_type not in removed_objs: couchdb_relational_map[couchdb_id] = new_obj.id else: + couchdb_relational_map[couchdb_id] = new_obj['parent_id'] couchdb_removed_objs.add(couchdb_id) self.verify_import_data(couchdb_relational_map, couchdb_removed_objs, workspace) return created diff --git a/server/models.py b/server/models.py index a5173c7e5a3..a46f3983287 100644 --- a/server/models.py +++ b/server/models.py @@ -62,7 +62,7 @@ class SourceCode(db.Model): id = Column(Integer, primary_key=True) filename = Column(Text, nullable=False) - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship('Workspace', backref='source_codes') @@ -91,7 +91,7 @@ class Host(db.Model): foreign_keys=[entity_metadata_id] ) - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship( 'Workspace', backref='hosts', @@ -138,10 +138,10 @@ class Service(db.Model): foreign_keys=[entity_metadata_id] ) - host_id = Column(Integer, ForeignKey('host.id'), index=True) + host_id = Column(Integer, ForeignKey('host.id'), index=True, nullable=False) host = relationship('Host', backref='services', foreign_keys=[host_id]) - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship( 'Workspace', backref='services', @@ -203,6 +203,7 @@ class VulnerabilityGeneric(VulnerabilityABC): Integer, ForeignKey('workspace.id'), index=True, + nullable = False ) workspace = relationship('Workspace', backref='vulnerabilities') @@ -303,7 +304,8 @@ class Reference(db.Model): workspace_id = Column( Integer, ForeignKey('workspace.id'), - index=True + index=True, + nullable=False ) workspace = relationship( 'Workspace', @@ -348,7 +350,8 @@ class PolicyViolation(db.Model): workspace_id = Column( Integer, ForeignKey('workspace.id'), - index=True + index=True, + nullable=False ) workspace = relationship( 'Workspace', @@ -416,7 +419,7 @@ class Command(db.Model): params = Column(Text(), nullable=True) user = Column(String(250), nullable=True) # os username where the command was executed - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship('Workspace', foreign_keys=[workspace_id]) # TODO: add Tool relationship and report_attachment @@ -547,7 +550,7 @@ class Methodology(db.Model): ) workspace = relationship('Workspace', backref='methodologies') - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) class TaskABC(db.Model): @@ -664,7 +667,7 @@ class Comment(db.Model): object_id = Column(Integer, nullable=False) object_type = Column(Text, nullable=False) - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship('Workspace', foreign_keys=[workspace_id]) class ExecutiveReport(db.Model): @@ -689,5 +692,5 @@ class ExecutiveReport(db.Model): summary = Column(Text, nullable=True) title = Column(Text, nullable=True) - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship('Workspace', foreign_keys=[workspace_id]) From 6e07d30e810214337140eb48cf10465d412fe1c4 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 4 Sep 2017 17:03:30 -0300 Subject: [PATCH 0154/1506] Fix severities enum in vulnerability model --- server/models.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index 82d557bb69a..d31e7f2fd54 100644 --- a/server/models.py +++ b/server/models.py @@ -159,6 +159,14 @@ class VulnerabilityABC(db.Model): 'difficult', 'infeasible' ] + SEVERITIES = [ + 'critical', + 'high', + 'medium', + 'low', + 'informational', + 'unclassified', + ] __abstract__ = True id = Column(Integer, primary_key=True) @@ -168,7 +176,7 @@ class VulnerabilityABC(db.Model): ease_of_resolution = Column(Enum(*EASE_OF_RESOLUTIONS, name='vulnerability_ease_of_resolution'), nullable=True) name = Column(Text, nullable=False) resolution = Column(Text, nullable=True) - severity = Column(String(50), nullable=False) + severity = Column(Enum(*SEVERITIES, name='vulnerability_severity'), nullable=False) # TODO add evidence impact_accountability = Column(Boolean, default=False) From 2ab1f6e2c73e4803e821d6fa260ecdf367148721 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 4 Sep 2017 17:51:09 -0300 Subject: [PATCH 0155/1506] Add unique constraint --- server/importer.py | 4 ++-- server/models.py | 53 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/server/importer.py b/server/importer.py index 6852170c932..5bcf09dd882 100644 --- a/server/importer.py +++ b/server/importer.py @@ -105,8 +105,8 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation document['os'] = 'unknown' default_gateway = document.get('default_gateway', None) - - host.name = document.get('name') + if not host.ip: + host.ip = document.get('name') host.description = document.get('description') host.os = document.get('os') host.default_gateway_ip = default_gateway and default_gateway[0] diff --git a/server/models.py b/server/models.py index a46f3983287..35de236a492 100644 --- a/server/models.py +++ b/server/models.py @@ -57,7 +57,6 @@ class EntityMetadata(db.Model): class SourceCode(db.Model): - # TODO: add unique constraint -> filename, workspace __tablename__ = 'source_code' id = Column(Integer, primary_key=True) filename = Column(Text, nullable=False) @@ -65,9 +64,12 @@ class SourceCode(db.Model): workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship('Workspace', backref='source_codes') + __table_args__ = ( + UniqueConstraint(filename, workspace_id, name='uix_source_code_filename_workspace'), + ) + class Host(db.Model): - # TODO: add unique constraint -> ip, workspace __tablename__ = 'host' id = Column(Integer, primary_key=True) ip = Column(Text, nullable=False) # IP v4 or v6 @@ -97,16 +99,27 @@ class Host(db.Model): backref='hosts', foreign_keys=[workspace_id] ) + __table_args__ = ( + UniqueConstraint(ip, workspace_id, name='uix_host_ip_workspace'), + ) class Hostname(db.Model): - # TODO: add unique constraint -> name, host, workspace __tablename__ = 'hostname' id = Column(Integer, primary_key=True) name = Column(Text, nullable=False) - host_id = Column(Integer, ForeignKey('host.id'), index=True) + host_id = Column(Integer, ForeignKey('host.id'), index=True, nullable=False) host = relationship('Host', backref='hostnames') + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) + workspace = relationship( + 'Workspace', + backref='hosts', + foreign_keys=[workspace_id] + ) + __table_args__ = ( + UniqueConstraint(name, host_id, workspace_id, name='uix_hostname_host_workspace'), + ) class Service(db.Model): @@ -147,6 +160,9 @@ class Service(db.Model): backref='services', foreign_keys=[workspace_id] ) + __table_args__ = ( + UniqueConstraint(port, protocol, host_id, workspace_id, name='uix_service_port_protocol_host_workspace'), + ) class VulnerabilityABC(db.Model): @@ -180,6 +196,10 @@ class VulnerabilityABC(db.Model): class VulnerabilityTemplate(VulnerabilityABC): __tablename__ = 'vulnerability_template' + __table_args__ = ( + UniqueConstraint('name', name='uix_vulnerability_template_name'), + ) + class VulnerabilityGeneric(VulnerabilityABC): STATUSES = [ @@ -211,6 +231,19 @@ class VulnerabilityGeneric(VulnerabilityABC): 'polymorphic_on': type } + __table_args__ = ( + UniqueConstraint('name', + 'severity', + 'host_id', + 'service_id', + 'method', + 'parameter_name', + 'path', + 'website', + 'workspace_id', + name='uix_vulnerability'), + ) + class Vulnerability(VulnerabilityGeneric): host_id = Column(Integer, ForeignKey(Host.id), index=True) @@ -577,6 +610,10 @@ class TaskTemplate(TaskABC): nullable=False, ) + __table_args__ = ( + UniqueConstraint(TaskABC.name, template_id, name='uix_tass_template_name_desc_template'), + ) + class Task(TaskABC): STATUSES = [ @@ -602,25 +639,29 @@ class Task(TaskABC): assigned_to = relationship('User', backref='assigned_tasks') assigned_to_id = Column(Integer, ForeignKey('user.id'), nullable=True) - methodology = relationship('Methodology', backref='tasks') methodology_id = Column( Integer, ForeignKey('methodology.id'), index=True, nullable=False, ) + methodology = relationship('Methodology', backref='tasks') - template = relationship('TaskTemplate', backref='tasks') template_id = Column( Integer, ForeignKey('task_template.id'), index=True, nullable=True, ) + template = relationship('TaskTemplate', backref='tasks') workspace = relationship('Workspace', backref='tasks') workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) + __table_args__ = ( + UniqueConstraint(TaskABC.name, methodology_id, workspace_id, name='uix_task_name_desc_methodology_workspace'), + ) + class License(db.Model): __tablename__ = 'license' From 623661dddfee582fdd27a6b803693a7429e0c873 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 4 Sep 2017 17:52:04 -0300 Subject: [PATCH 0156/1506] Add License unique constraint --- server/models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index e6d2244bb53..e8c83a08079 100644 --- a/server/models.py +++ b/server/models.py @@ -211,7 +211,7 @@ class VulnerabilityGeneric(VulnerabilityABC): Integer, ForeignKey('workspace.id'), index=True, - nullable = False + nullable=False, ) workspace = relationship('Workspace', backref='vulnerabilities') @@ -640,6 +640,10 @@ class License(db.Model): type = Column(Text, nullable=True) notes = Column(Text, nullable=True) + __table_args__ = ( + UniqueConstraint('product', 'start_date', 'end_date', name='uix_license'), + ) + class Tag(db.Model): __tablename__ = 'tag' From 7192a61f1a640a941cc41d1b6e55661ce0e5aa41 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 4 Sep 2017 17:54:17 -0300 Subject: [PATCH 0157/1506] Fix License constraint name --- server/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index 7692a2fff1b..95da98c7ee1 100644 --- a/server/models.py +++ b/server/models.py @@ -682,7 +682,7 @@ class License(db.Model): notes = Column(Text, nullable=True) __table_args__ = ( - UniqueConstraint('product', 'start_date', 'end_date', name='uix_license'), + UniqueConstraint('product', 'start_date', 'end_date', name='uix_license_product_start_end_dates'), ) From f9dc69fb3f781c3cc757798ee8b0ae48c79d411b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 4 Sep 2017 19:22:09 -0300 Subject: [PATCH 0158/1506] Partial commit Add POST method to API. First attempt --- requirements_server.txt | 1 + server/api/base.py | 48 ++++++++++++++++++++++++++++++++--- server/api/modules/hosts.py | 4 +-- server/models.py | 2 +- test_cases/conftest.py | 8 ++++++ test_cases/test_server_api.py | 32 ++++++++++++++++++++++- 6 files changed, 87 insertions(+), 8 deletions(-) diff --git a/requirements_server.txt b/requirements_server.txt index 56279c73f37..7c9bacffd8a 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -14,3 +14,4 @@ Flask-SQLAlchemy==2.2 Flask-Script==2.0.5 flask-classful marshmallow +webargs diff --git a/server/api/base.py b/server/api/base.py index d8bd2aa41fe..39720295e25 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -4,7 +4,8 @@ from flask_classful import FlaskView from sqlalchemy.orm.exc import NoResultFound from werkzeug.routing import parse_rule -from server.models import Workspace +from webargs.flaskparser import FlaskParser +from server.models import Workspace, db def output_json(data, code, headers=None): @@ -78,6 +79,27 @@ def _get_object(self, workspace_name, object_id): def _dump(self, obj, **kwargs): return self._get_schema_class()(**kwargs).dump(obj) + def _parse_data(self, schema, request, *args, **kwargs): + return FlaskParser().parse(schema, request, locations=('json',), + *args, **kwargs) + + @classmethod + def register(cls, app, *args, **kwargs): + """Register and add JSON error handler. Use error code + 400 instead of 422""" + super(GenericWorkspacedView, cls).register(app, *args, **kwargs) + @app.errorhandler(422) + def handle_unprocessable_entity(err): + # webargs attaches additional metadata to the `data` attribute + exc = getattr(err, 'exc') + if exc: + # Get validations from the ValidationError object + messages = exc.messages + else: + messages = ['Invalid request'] + return flask.jsonify({ + 'messages': messages, + }), 400 class ListWorkspacedMixin(object): """Add GET // route""" @@ -94,8 +116,26 @@ def get(self, workspace_name, object_id): return self._dump(self._get_object(workspace_name, object_id)) -class ReadOnlyWorkspacedView(GenericWorkspacedView, - ListWorkspacedMixin, - RetrieveWorkspacedMixin): +class ReadOnlyWorkspacedView(ListWorkspacedMixin, + RetrieveWorkspacedMixin, + GenericWorkspacedView): """A generic view with list and retrieve endpoints""" pass + + +class CreateWorkspacedMixin(object): + + def post(self, workspace_name): + data = self._parse_data(self._get_schema_class()(strict=True), + flask.request) + obj = self.model_class(**data) + obj.workspace = self._get_workspace(workspace_name) + db.session.add(obj) + db.session.commit() + return self._dump(obj).data, 201 + + +class ReadWriteWorkspacedView(CreateWorkspacedMixin, + ReadOnlyWorkspacedView, + GenericWorkspacedView): + pass diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 42c9d1bbb2d..ff753db757e 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -10,7 +10,7 @@ from server.utils.web import gzipped, validate_workspace,\ get_integer_parameter, filter_request_args from server.dao.host import HostDAO -from server.api.base import ReadOnlyWorkspacedView +from server.api.base import ReadWriteWorkspacedView from server.models import Host host_api = Blueprint('host_api', __name__) @@ -23,7 +23,7 @@ class HostSchema(Schema): os = fields.String() -class HostsView(ReadOnlyWorkspacedView): +class HostsView(ReadWriteWorkspacedView): route_base = 'hosts' model_class = Host schema_class = HostSchema diff --git a/server/models.py b/server/models.py index d24303c7b6b..d6df2eb75d0 100644 --- a/server/models.py +++ b/server/models.py @@ -91,7 +91,7 @@ class Host(db.Model): foreign_keys=[entity_metadata_id] ) - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True) + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship( 'Workspace', backref='hosts', diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 4decffbf2e8..57567edd662 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -24,7 +24,15 @@ class CustomClient(FlaskClient): + def open(self, *args, **kwargs): + if kwargs.pop('use_json_data', True) and 'data' in kwargs: + # JSON-encode data by default + kwargs['data'] = json.dumps(kwargs['data']) + kwargs['headers'] = kwargs.get('headers', []) + [ + ('Content-Type', 'application/json'), + ] + ret = super(CustomClient, self).open(*args, **kwargs) try: ret.json = json.loads(ret.data) diff --git a/test_cases/test_server_api.py b/test_cases/test_server_api.py index 86cd14622a5..69e8fa3b3b1 100644 --- a/test_cases/test_server_api.py +++ b/test_cases/test_server_api.py @@ -1,6 +1,6 @@ import pytest from test_cases import factories -from server.models import db, Workspace +from server.models import db, Workspace, Host PREFIX = '/v2/' HOSTS_COUNT = 5 @@ -46,3 +46,33 @@ def test_retrieve_fails_with_host_of_another_workspace(self, res = test_client.get(self.url(self.workspace.hosts[0], new)) assert res.status_code == 404 + def test_create_a_host_succeeds(self, test_client): + res = test_client.post(self.url(), data={ + "ip": "127.0.0.1", + "description": "aaaaa", + # os is not required + }) + print res.data + assert res.status_code == 201 + host_id = res.json['id'] + host = Host.query.get(host_id) + assert host.ip == "127.0.0.1" + assert host.description == "aaaaa" + assert host.os is None + assert host.workspace == self.workspace + + def test_create_a_host_fails_with_missing_desc(self, test_client): + res = test_client.post(self.url(), data={ + "ip": "127.0.0.1", + }) + assert res.status_code == 400 + + def test_create_a_host_fails_with_existing_ip(self, test_client, + session, host): + session.add(host) + session.commit() + res = test_client.post(self.url(), data={ + "ip": host.ip, + "description": "aaaaa", + }) + assert res.status_code == 400 From 657cb7669f42b3e1792110b3936f83446c12f7a2 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 4 Sep 2017 19:27:11 -0300 Subject: [PATCH 0159/1506] Add check and unique constraints Adds unique to Reference, ReferenceTemplate, PolicyViolation, PolicyViolationTemplate, Credential, Vulnerability. Adds check to Vulnerability and Credential. --- server/models.py | 87 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 66 insertions(+), 21 deletions(-) diff --git a/server/models.py b/server/models.py index 95da98c7ee1..7bdc2a4c9c2 100644 --- a/server/models.py +++ b/server/models.py @@ -3,6 +3,7 @@ # See the file 'doc/LICENSE' for the license information from sqlalchemy import ( Boolean, + CheckConstraint, Column, DateTime, Enum, @@ -123,7 +124,6 @@ class Hostname(db.Model): class Service(db.Model): - # TODO: add unique constraint to -> port, protocol, host_id, workspace STATUSES = [ 'open', 'closed', @@ -166,7 +166,6 @@ class Service(db.Model): class VulnerabilityABC(db.Model): - # TODO: add unique constraint to -> name, description, severity, parent, method, pname, path, website, workspace # revisar plugin nexpose, netspark para terminar de definir uniques. asegurar que se carguen bien EASE_OF_RESOLUTIONS = [ 'trivial', @@ -239,19 +238,6 @@ class VulnerabilityGeneric(VulnerabilityABC): 'polymorphic_on': type } - __table_args__ = ( - UniqueConstraint('name', - 'severity', - 'host_id', - 'service_id', - 'method', - 'parameter_name', - 'path', - 'website', - 'workspace_id', - name='uix_vulnerability'), - ) - class Vulnerability(VulnerabilityGeneric): host_id = Column(Integer, ForeignKey(Host.id), index=True) @@ -336,6 +322,10 @@ class ReferenceTemplate(db.Model): foreign_keys=[vulnerability_id], ) + __table_args__ = ( + UniqueConstraint('name', 'vulnerability_id', name='uix_reference_template_name_vulnerability'), + ) + class Reference(db.Model): __tablename__ = 'reference' @@ -365,6 +355,10 @@ class Reference(db.Model): foreign_keys=[vulnerability_id], ) + __table_args__ = ( + UniqueConstraint('name', 'vulnerability_id', 'workspace_id', name='uix_reference_name_vulnerability_workspace'), + ) + class PolicyViolationTemplate(db.Model): __tablename__ = 'policy_violation_template' @@ -382,6 +376,13 @@ class PolicyViolationTemplate(db.Model): foreign_keys=[vulnerability_id] ) + __table_args__ = ( + UniqueConstraint( + 'name', + 'vulnerability_id', + name='uix_policy_violation_template_name_vulnerability'), + ) + class PolicyViolation(db.Model): __tablename__ = 'policy_violation' @@ -411,6 +412,14 @@ class PolicyViolation(db.Model): foreign_keys=[vulnerability_id] ) + __table_args__ = ( + UniqueConstraint( + 'name', + 'vulnerability_id', + 'workspace_id', + name='uix_policy_violation_template_name_vulnerability_workspace'), + ) + class Credential(db.Model): # TODO: add unique constraint -> username, host o service y workspace @@ -448,6 +457,19 @@ class Credential(db.Model): foreign_keys=[workspace_id], ) + __table_args__ = ( + CheckConstraint('(host_id IS NULL AND service_id IS NOT NULL) OR ' + '(host_id IS NOT NULL AND service_id IS NULL)', + name='check_credential_host_service'), + UniqueConstraint( + 'username', + 'host_id', + 'service_id', + 'workspace_id', + name='uix_credential_username_host_service_workspace' + ), + ) + class Command(db.Model): __tablename__ = 'command' @@ -618,9 +640,9 @@ class TaskTemplate(TaskABC): nullable=False, ) - __table_args__ = ( - UniqueConstraint(TaskABC.name, template_id, name='uix_tass_template_name_desc_template'), - ) + # __table_args__ = ( + # UniqueConstraint(template_id, name='uix_task_template_name_desc_template_delete'), + # ) class Task(TaskABC): @@ -666,9 +688,9 @@ class Task(TaskABC): workspace = relationship('Workspace', backref='tasks') workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) - __table_args__ = ( - UniqueConstraint(TaskABC.name, methodology_id, workspace_id, name='uix_task_name_desc_methodology_workspace'), - ) + # __table_args__ = ( + # UniqueConstraint(TaskABC.name, methodology_id, workspace_id, name='uix_task_name_desc_methodology_workspace'), + # ) class License(db.Model): @@ -747,3 +769,26 @@ class ExecutiveReport(db.Model): workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship('Workspace', foreign_keys=[workspace_id]) + + +# This constraint uses Columns from different classes +# Since it applies to the table vulnerability it should be adVulnerability.ded to the Vulnerability class +# However, since it contains columns from children classes, this cannot be done +# This is a workaround suggested by SQLAlchemy's creator +CheckConstraint('((Vulnerability.host_id IS NULL)::int+' + '(Vulnerability.service_id IS NULL)::int+' + '(Vulnerability.source_code_id IS NULL)::int)=1', + name='check_vulnerability_host_service_source_code', + table=VulnerabilityGeneric.__table__) +UniqueConstraint(VulnerabilityGeneric.name, + VulnerabilityGeneric.severity, + Vulnerability.host_id, + VulnerabilityWeb.service_id, + VulnerabilityWeb.method, + VulnerabilityWeb.parameter_name, + VulnerabilityWeb.path, + VulnerabilityWeb.website, + VulnerabilityGeneric.workspace_id, + VulnerabilityCode.source_code_id, + name='uix_vulnerability' +) \ No newline at end of file From f696e00014001a194abfacfae0069eb8f71a3a50 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 5 Sep 2017 12:43:06 -0300 Subject: [PATCH 0160/1506] Fix check constraint --- server/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/models.py b/server/models.py index 7bdc2a4c9c2..541959b3f9f 100644 --- a/server/models.py +++ b/server/models.py @@ -775,9 +775,9 @@ class ExecutiveReport(db.Model): # Since it applies to the table vulnerability it should be adVulnerability.ded to the Vulnerability class # However, since it contains columns from children classes, this cannot be done # This is a workaround suggested by SQLAlchemy's creator -CheckConstraint('((Vulnerability.host_id IS NULL)::int+' - '(Vulnerability.service_id IS NULL)::int+' - '(Vulnerability.source_code_id IS NULL)::int)=1', +CheckConstraint('((Vulnerability.host_id IS NOT NULL)::int+' + '(Vulnerability.service_id IS NOT NULL)::int+' + '(Vulnerability.source_code_id IS NOT NULL)::int)=1', name='check_vulnerability_host_service_source_code', table=VulnerabilityGeneric.__table__) UniqueConstraint(VulnerabilityGeneric.name, From 3ccd295f41946d7a29d2ff91c4c869577afc6650 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 5 Sep 2017 12:43:53 -0300 Subject: [PATCH 0161/1506] Importer fixes after new unique and check constraints --- server/importer.py | 63 +++++++++++++++++++++++++++++++++------------- server/models.py | 2 +- 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/server/importer.py b/server/importer.py index 5bcf09dd882..67daaf1e1b4 100644 --- a/server/importer.py +++ b/server/importer.py @@ -152,10 +152,10 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation couch_parent_id = '.'.join(document['_id'].split('.')[:-1]) parent_id = couchdb_relational_map[couch_parent_id] interface['parent_id'] = parent_id - host = self.merge_with_host(interface, parent_id) + host = self.merge_with_host(interface, workspace, parent_id) yield interface - def merge_with_host(self, interface, parent_relation_db_id): + def merge_with_host(self, interface, workspace, parent_relation_db_id): host = session.query(Host).filter_by(id=parent_relation_db_id).first() assert host.workspace == interface['workspace'] if interface['mac']: @@ -173,7 +173,13 @@ def merge_with_host(self, interface, parent_relation_db_id): host.description += '\n Interface data: {0}'.format(interface['description']) for hostname in interface['hostnames']: - hostname, created = get_or_create(session, Hostname, name=hostname, host=host) + hostname, created = get_or_create( + session, + Hostname, + name=hostname, + host=host, + workspace=workspace + ) host.owned = host.owned or interface['owned'] return host @@ -189,7 +195,10 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation # some services are missing the parent key parent_id = document['_id'].split('.')[0] host, created = get_or_create(session, Host, id=couchdb_relational_map[parent_id]) - for port in document.get('ports'): + ports = document.get('ports') + if len(ports) > 2: + logger.warn('More than one port found in services!') + for port in ports: service, created = get_or_create(session, Service, name=document.get('name'), @@ -232,7 +241,15 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation vulnerability.data = document.get('data') vulnerability.easeofresolution = document.get('easeofresolution') vulnerability.resolution = document.get('resolution') - vulnerability.severity = document.get('severity') + mapped_severity = { + 'med': 'medium', + 'critical': 'critical', + 'high':'high', + 'low': 'low', + 'info': 'informational', + 'unclassified': 'unclassified', + } + vulnerability.severity = mapped_severity[document.get('severity')] vulnerability.owned = document.get('owned', False) #vulnerability.attachments = json.dumps(document.get('_attachments', {})) vulnerability.impact_accountability = document.get('impact', {}).get('accountability') @@ -291,7 +308,7 @@ def add_references(self, document, vulnerability, workspace): vulnerability=vulnerability ) - def set_parent(self, vulnerability, parent_id, level=2): + def set_parent(self, vulnerability, parent_id, level): logger.debug('Set parent for vulnerabiity level {0}'.format(level)) if level == 2: vulnerability.host_id = session.query(Host).filter_by(id=parent_id).first().id @@ -579,10 +596,18 @@ def run(self): cwes = requests.get(cwe_url).json()['rows'] for cwe in cwes: document = cwe['doc'] + mapped_exploitation = { + 'critical': 'critical', + 'med': 'medium', + 'high':'high', + 'low': 'low', + 'info': 'informational', + 'unclassified': 'unclassified', + } vuln_template, created = get_or_create(session, VulnerabilityTemplate, name=document.get('name'), - severity=document.get('exploitation'), + severity=mapped_exploitation[document.get('exploitation')], description=document.get('description')) vuln_template.resolution = document.get('resolution') for ref_doc in document['references']: @@ -724,17 +749,19 @@ def import_workspace_into_database(self, workspace_name): obj_importer = faraday_importer.get_importer_from_document(obj_type)() objs_dict = self.get_objs(couch_url, obj_type, level) for raw_obj in tqdm(objs_dict.get('rows', [])): - raw_obj = raw_obj['value'] - couchdb_id = raw_obj['_id'] - - for new_obj in obj_importer.update_from_document(raw_obj, workspace, level, couchdb_relational_map): - if not new_obj: - continue - session.commit() - if obj_type not in removed_objs: - couchdb_relational_map[couchdb_id] = new_obj.id - else: - couchdb_relational_map[couchdb_id] = new_obj['parent_id'] + # we use no_autoflush since some queries triggers flush and some relationship are missing in the middle + with session.no_autoflush: + raw_obj = raw_obj['value'] + couchdb_id = raw_obj['_id'] + + for new_obj in obj_importer.update_from_document(raw_obj, workspace, level, couchdb_relational_map): + if not new_obj: + continue + session.commit() + if obj_type not in removed_objs: + couchdb_relational_map[couchdb_id] = new_obj.id + else: + couchdb_relational_map[couchdb_id] = new_obj['parent_id'] couchdb_removed_objs.add(couchdb_id) self.verify_import_data(couchdb_relational_map, couchdb_removed_objs, workspace) return created diff --git a/server/models.py b/server/models.py index 541959b3f9f..87fb9f0ab7e 100644 --- a/server/models.py +++ b/server/models.py @@ -97,7 +97,7 @@ class Host(db.Model): workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship( 'Workspace', - backref='hosts', + backref='workspace_hosts', foreign_keys=[workspace_id] ) __table_args__ = ( From b2f4b2a4b69ca2cdb28709fe21182e74fbc227f1 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 5 Sep 2017 14:24:31 -0300 Subject: [PATCH 0162/1506] Add ip address check --- server/importer.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/server/importer.py b/server/importer.py index 67daaf1e1b4..f676494813f 100644 --- a/server/importer.py +++ b/server/importer.py @@ -155,6 +155,13 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation host = self.merge_with_host(interface, workspace, parent_id) yield interface + def check_ip_address(self, ip_str): + if not ip_str or ip_str == '0.0.0.0': + return False + if not ip_str or ip_str == '0000:0000:0000:0000:0000:0000:0000:0000': + return False + return True + def merge_with_host(self, interface, workspace, parent_relation_db_id): host = session.query(Host).filter_by(id=parent_relation_db_id).first() assert host.workspace == interface['workspace'] @@ -162,9 +169,9 @@ def merge_with_host(self, interface, workspace, parent_relation_db_id): host.mac = interface['mac'] if interface['owned']: host.owned = interface['owned'] - if interface['ipv4_address'] or interface['ipv6_address']: + if self.check_ip_address(interface['ipv4_address']) or self.check_ip_address(interface['ipv6_address']): host.ip = interface['ipv4_address'] or interface['ipv6_address'] - if interface['ipv4_gateway'] or interface['ipv6_gateway']: + if self.check_ip_address(interface['ipv4_gateway']) or self.check_ip_address(interface['ipv6_gateway']): host.default_gateway_ip = interface['ipv4_gateway'] or interface['ipv6_gateway'] #host.default_gateway_mac if interface['network_segment']: From 78f305097f64954883391522c54d94f60cccdb7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 5 Sep 2017 15:39:19 -0300 Subject: [PATCH 0163/1506] Improve creation endpoint (generic and host one) Make it work, add unique logic (with fields that are unique together with workspace id) --- server/api/base.py | 33 +++++++++++++++++++++++++++++---- server/api/modules/hosts.py | 1 + server/models.py | 3 +++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 39720295e25..16e81a060da 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -4,7 +4,8 @@ from flask_classful import FlaskView from sqlalchemy.orm.exc import NoResultFound from werkzeug.routing import parse_rule -from webargs.flaskparser import FlaskParser +from webargs.flaskparser import FlaskParser, abort +from webargs.core import ValidationError from server.models import Workspace, db @@ -33,6 +34,7 @@ class GenericWorkspacedView(FlaskView): base_args = ['workspace_name'] # Required to prevent double usage of representations = {'application/json': output_json} lookup_field = 'id' + unique_fields = [] # Fields unique together with workspace_id @classmethod def get_route_base(cls): @@ -83,6 +85,19 @@ def _parse_data(self, schema, request, *args, **kwargs): return FlaskParser().parse(schema, request, locations=('json',), *args, **kwargs) + def _validate_uniqueness(self, obj): + assert obj.workspace is not None, "Object must have a " \ + "workspace attribute set to call _validate_uniqueness" + for field_name in self.unique_fields: + field = getattr(self.model_class, field_name) + value = getattr(obj, field_name) + query = self._get_base_query(obj.workspace.name).filter( + field==value) + if query.one_or_none(): + abort(422, ValidationError('Existing value for %s field: %s' % ( + field_name, value + ))) + @classmethod def register(cls, app, *args, **kwargs): """Register and add JSON error handler. Use error code @@ -129,13 +144,23 @@ def post(self, workspace_name): data = self._parse_data(self._get_schema_class()(strict=True), flask.request) obj = self.model_class(**data) - obj.workspace = self._get_workspace(workspace_name) - db.session.add(obj) + created = self._perform_create(workspace_name, obj) + return self._dump(created).data, 201 + + def _perform_create(self, workspace_name, obj): + assert not db.session.new + with db.session.no_autoflush: + # Required because _validate_uniqueness does a select. Doing this + # outside a no_autoflush block would result in a premature create. + obj.workspace = self._get_workspace(workspace_name) + self._validate_uniqueness(obj) + db.session.add(obj) db.session.commit() - return self._dump(obj).data, 201 + return obj class ReadWriteWorkspacedView(CreateWorkspacedMixin, ReadOnlyWorkspacedView, GenericWorkspacedView): + """A generic view with list, retrieve and create endpoints""" pass diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index ff753db757e..70cc144cdc8 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -27,6 +27,7 @@ class HostsView(ReadWriteWorkspacedView): route_base = 'hosts' model_class = Host schema_class = HostSchema + unique_fields = ['ip'] HostsView.register(host_api) diff --git a/server/models.py b/server/models.py index d6df2eb75d0..b1b80fa3119 100644 --- a/server/models.py +++ b/server/models.py @@ -97,6 +97,9 @@ class Host(db.Model): backref='hosts', foreign_keys=[workspace_id] ) + __table_args__ = ( + UniqueConstraint(ip, workspace_id, name='uix_host_ip_workspace'), + ) class Hostname(db.Model): From e86b0c57d0142e58bbae2d337815faccbc31ae4a Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 5 Sep 2017 17:36:54 -0300 Subject: [PATCH 0164/1506] More updates after new contraints * Use service and host to retrieve vulns * Sometimes hostnames were string/unicode and not list * Check for validity in interface ip before override host ip --- server/importer.py | 80 +++++++++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 29 deletions(-) diff --git a/server/importer.py b/server/importer.py index f676494813f..cd0f5ad8e87 100644 --- a/server/importer.py +++ b/server/importer.py @@ -160,6 +160,8 @@ def check_ip_address(self, ip_str): return False if not ip_str or ip_str == '0000:0000:0000:0000:0000:0000:0000:0000': return False + if not ip_str or ip_str == '0000:0000:0000:0000:0000:0000:0000:0001': + return False return True def merge_with_host(self, interface, workspace, parent_relation_db_id): @@ -169,8 +171,16 @@ def merge_with_host(self, interface, workspace, parent_relation_db_id): host.mac = interface['mac'] if interface['owned']: host.owned = interface['owned'] - if self.check_ip_address(interface['ipv4_address']) or self.check_ip_address(interface['ipv6_address']): - host.ip = interface['ipv4_address'] or interface['ipv6_address'] + if self.check_ip_address(interface['ipv4_address']): + interface_ip = interface['ipv4_address'] + if host.ip != interface_ip: + logger.warn('Overriding host ip address {0} with new ip {1}'.format(host.ip, interface_ip)) + host.ip = interface_ip + if self.check_ip_address(interface['ipv6_address']): + interface_ip = interface['ipv6_address'] + if host.ip != interface_ip: + logger.warn('Overriding host ip address {0} with new ip {1}'.format(host.ip, interface_ip)) + host.ip = interface_ip if self.check_ip_address(interface['ipv4_gateway']) or self.check_ip_address(interface['ipv6_gateway']): host.default_gateway_ip = interface['ipv4_gateway'] or interface['ipv6_gateway'] #host.default_gateway_mac @@ -178,12 +188,16 @@ def merge_with_host(self, interface, workspace, parent_relation_db_id): host.net_segment = interface['network_segment'] if interface['description']: host.description += '\n Interface data: {0}'.format(interface['description']) - - for hostname in interface['hostnames']: + if type(interface['hostnames']) in (str, unicode): + interface['hostnames'] = [interface['hostnames']] + for hostname_str in interface['hostnames']: + if not hostname_str: + # skip empty hostnames + continue hostname, created = get_or_create( session, Hostname, - name=hostname, + name=hostname_str, host=host, workspace=workspace ) @@ -235,15 +249,37 @@ class VulnerabilityImporter(object): DOC_TYPE = ['Vulnerability', 'VulnerabilityWeb'] def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): - vuln_class = Vulnerability + couch_parent_id = document.get('parent', None) + if not couch_parent_id: + couch_parent_id = '.'.join(document['_id'].split('.')[:-1]) + parent_id = couchdb_relational_map[couch_parent_id] + if level == 2: + parent = session.query(Host).filter_by(id=parent_id).first() + if level == 4: + parent = session.query(Service).filter_by(id=parent_id).first() if document['type'] == 'VulnerabilityWeb': - vuln_class = VulnerabilityWeb - vulnerability, created = get_or_create( - session, - vuln_class, - name=document.get('name'), - description=document.get('desc') - ) + vulnerability, created = get_or_create( + session, + VulnerabilityWeb, + name=document.get('name'), + description=document.get('desc'), + service_id=parent.id, + ) + if document['type'] == 'Vulnerability': + vuln_params = { + 'name': document.get('name'), + 'description': document.get('desc') + } + if type(parent) == Host: + vuln_params.update({'host_id': parent.id}) + elif type(parent) == Service: + vuln_params.update({'service_id': parent.id}) + vulnerability, created = get_or_create( + session, + Vulnerability, + **vuln_params + ) + vulnerability.confirmed = document.get('confirmed', False) or False vulnerability.data = document.get('data') vulnerability.easeofresolution = document.get('easeofresolution') @@ -284,13 +320,6 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation vulnerability.status = status vulnerability.workspace = workspace - - - couch_parent_id = document.get('parent', None) - if not couch_parent_id: - couch_parent_id = '.'.join(document['_id'].split('.')[:-1]) - parent_id = couchdb_relational_map[couch_parent_id] - self.set_parent(vulnerability, parent_id, level) self.add_references(document, vulnerability, workspace) self.add_policy_violations(document, vulnerability, workspace) yield vulnerability @@ -315,13 +344,6 @@ def add_references(self, document, vulnerability, workspace): vulnerability=vulnerability ) - def set_parent(self, vulnerability, parent_id, level): - logger.debug('Set parent for vulnerabiity level {0}'.format(level)) - if level == 2: - vulnerability.host_id = session.query(Host).filter_by(id=parent_id).first().id - if level == 4: - vulnerability.service_id = session.query(Service).filter_by(id=parent_id).first().id - class CommandImporter(object): DOC_TYPE = 'CommandRunInformation' @@ -387,7 +409,6 @@ class WorkspaceImporter(object): DOC_TYPE = 'Workspace' def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): - workspace, created = get_or_create(session, server.models.Workspace, name=document.get('name', None)) workspace.description = document.get('description') if document.get('duration') and document.get('duration')['start']: workspace.start_date = datetime.datetime.fromtimestamp(float(document.get('duration')['start'])/1000) @@ -717,7 +738,8 @@ def verify_import_data(self, couchdb_relational_map, couchdb_removed_objs, works def import_workspace_into_database(self, workspace_name): faraday_importer = FaradayEntityImporter() - workspace, created = get_or_create(session, server.models.Workspace, name=workspace_name) + workspace, created = get_or_create(session, Workspace, name=workspace_name) + session.commit() couch_url = "http://{username}:{password}@{hostname}:{port}/{workspace_name}/_temp_view?include_docs=true".format( username=server.config.couchdb.user, From f5a82d3c4c6a31a7cb416c709597f0489fadf02b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 6 Sep 2017 15:08:24 -0300 Subject: [PATCH 0165/1506] Move interface importer inside host importer. --- server/importer.py | 202 +++++++++++++++++++++++++-------------------- 1 file changed, 111 insertions(+), 91 deletions(-) diff --git a/server/importer.py b/server/importer.py index cd0f5ad8e87..10a5d066923 100644 --- a/server/importer.py +++ b/server/importer.py @@ -47,7 +47,7 @@ ) COUCHDB_USER_PREFIX = 'org.couchdb.user:' -COUCHDB_PASSWORD_PREXFIX = '-pbkdf2-' +COUCHDB_PASSWORD_PREFIX = '-pbkdf2-' logger = server.utils.logger.get_logger(__name__) @@ -90,99 +90,125 @@ def __truncate_to_epoch_in_seconds(self, timestamp): return timestamp -class HostImporter(object): - DOC_TYPE = 'Host' - - def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): - # Ticket #3387: if the 'os' field is None, we default to 'unknown' - try: - IP(document.get('name')) # this will raise ValueError on valid IPs - host, created = get_or_create(session, Host, ip=document.get('name')) - except ValueError: - host, created = get_or_create(session, Host, ip=document.get('ip')) +def check_ip_address(ip_str): + if not ip_str: + return False + if ip_str == '0.0.0.0': + return False + if ip_str == '0000:0000:0000:0000:0000:0000:0000:0000': + return False + if ip_str == '0000:0000:0000:0000:0000:0000:0000:0001': + return False + try: + IP(ip_str) + except ValueError: + return False + return True - if not document.get('os'): - document['os'] = 'unknown' - default_gateway = document.get('default_gateway', None) - if not host.ip: - host.ip = document.get('name') - host.description = document.get('description') - host.os = document.get('os') - host.default_gateway_ip = default_gateway and default_gateway[0] - host.default_gateway_mac = default_gateway and default_gateway[1] - host.owned = document.get('owned', False) - host.workspace = workspace - yield host - - -class InterfaceImporter(object): +class HostImporter(object): """ Class interface was removed in the new model. We will merge the interface data with the host. For ports we will create new services for open ports if it was not previously created. """ - DOC_TYPE = 'Interface' + + def retrieve_ips_from_host_document(self, document): + """ + + :param document: json document from couchdb with host data + :return: str with ip or name if no valid ip was found. + """ + try: + IP(document.get('name')) # this will raise ValueError on invalid IPs + yield document.get('name') + except ValueError: + host_ip = document.get('ipv4') + created_ipv4 = False + created_ipv6 = False + if check_ip_address(host_ip): + yield host_ip + created_ipv4 = True + host_ip = document.get('ipv6') + if check_ip_address(host_ip): + yield host_ip + if not created_ipv4 or not created_ipv6: + # sometimes the host lacks the ip. + yield document.get('name') + if created_ipv4 and created_ipv6: + logger.warn('Two host will be created one with ipv4 and another one with ipv6. Couch id is {0}'.format(document.get('_id'))) def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): - # interface dict is used to merge interface with host and connect it the services to the host - interface = {} - interface['name'] = document.get('name') - interface['description'] = document.get('description') - interface['mac'] = document.get('mac') - interface['owned'] = document.get('owned', False) - interface['hostnames'] = document.get('hostnames') - interface['network_segment'] = document.get('network_segment') - interface['ipv4_address'] = document.get('ipv4').get('address') - interface['ipv4_gateway'] = document.get('ipv4').get('gateway') - interface['ipv4_dns'] = document.get('ipv4').get('DNS') - interface['ipv4_mask'] = document.get('ipv4').get('mask') - interface['ipv6_address'] = document.get('ipv6').get('address') - interface['ipv6_gateway'] = document.get('ipv6').get('gateway') - interface['ipv6_dns'] = document.get('ipv6').get('DNS') - interface['ipv6_prefix'] = str(document.get('ipv6').get('prefix')) - # ports_* are integers with counts - interface['ports_filtered'] = document.get('ports', {}).get('filtered') - interface['ports_opened'] = document.get('ports', {}).get('opened') - interface['ports_closed'] = document.get('ports', {}).get('closed') - interface['workspace'] = workspace - couch_parent_id = document.get('parent', None) - if not couch_parent_id: - couch_parent_id = '.'.join(document['_id'].split('.')[:-1]) - parent_id = couchdb_relational_map[couch_parent_id] - interface['parent_id'] = parent_id - host = self.merge_with_host(interface, workspace, parent_id) - yield interface - - def check_ip_address(self, ip_str): - if not ip_str or ip_str == '0.0.0.0': - return False - if not ip_str or ip_str == '0000:0000:0000:0000:0000:0000:0000:0000': - return False - if not ip_str or ip_str == '0000:0000:0000:0000:0000:0000:0000:0001': - return False - return True - - def merge_with_host(self, interface, workspace, parent_relation_db_id): - host = session.query(Host).filter_by(id=parent_relation_db_id).first() - assert host.workspace == interface['workspace'] + hosts = [] + host_ips = [name_or_ip for name_or_ip in self.retrieve_ips_from_host_document(document)] + interfaces = self.host_interfaces_from_couch(workspace, document.get('_id')) + for interface in interfaces: + interface = interface['value'] + if check_ip_address(interface['ipv4']['address']): + interface_ip = interface['ipv4']['address'] + host, created = get_or_create(session, Host, ip=interface_ip, workspace=workspace) + host.default_gateway_ip = interface['ipv4']['gateway'] + self.merge_with_host(host, interface, workspace) + hosts.append(host) + if check_ip_address(interface['ipv6']['address']): + interface_ip = interface['ipv6']['address'] + host, created = get_or_create(session, Host, ip=interface_ip, workspace=workspace) + host.default_gateway_ip = interface['ipv6']['gateway'] + self.merge_with_host(host, interface, workspace) + hosts.append(host) + if not hosts: + # if not host were created after inspecting interfaces + # we create a host with "name" as ip to avoid losing hosts. + # some hosts lacks of interface + for name_or_ip in host_ips: + host, created = get_or_create(session, Host, ip=name_or_ip, workspace=workspace) + hosts.append(host) + + if len(hosts) > 1: + logger.warning('Total hosts found {0} for couchdb id {1}'.format(len(hosts), document.get('_id'))) + + for host in hosts: + # we update or set other host attributes in this cycle + # Ticket #3387: if the 'os' field is None, we default to 'unknown + if not document.get('os'): + document['os'] = 'unknown' + + default_gateway = document.get('default_gateway', None) + + host.description = document.get('description') + host.os = document.get('os') + host.default_gateway_ip = default_gateway and default_gateway[0] + host.default_gateway_mac = default_gateway and default_gateway[1] + host.owned = document.get('owned', False) + host.workspace = workspace + yield host + + def host_interfaces_from_couch(self, workspace, host_couchdb_id): + couch_url = "http://{username}:{password}@{hostname}:{port}/{workspace_name}/_temp_view?include_docs=true".format( + username=server.config.couchdb.user, + password=server.config.couchdb.password, + hostname=server.config.couchdb.host, + port=server.config.couchdb.port, + workspace_name=workspace.name + ) + data = { + "map": "function(doc) { if(doc.type == '%s' && doc._id.split('.').slice(0, 1).join('.') == '%s') emit(null, doc); }" % ( + 'Interface', host_couchdb_id) + } + + r = requests.post(couch_url, json=data) + try: + return r.json()['rows'] + except Exception as ex: + print('sadsa') + + def merge_with_host(self, host, interface, workspace): if interface['mac']: host.mac = interface['mac'] if interface['owned']: host.owned = interface['owned'] - if self.check_ip_address(interface['ipv4_address']): - interface_ip = interface['ipv4_address'] - if host.ip != interface_ip: - logger.warn('Overriding host ip address {0} with new ip {1}'.format(host.ip, interface_ip)) - host.ip = interface_ip - if self.check_ip_address(interface['ipv6_address']): - interface_ip = interface['ipv6_address'] - if host.ip != interface_ip: - logger.warn('Overriding host ip address {0} with new ip {1}'.format(host.ip, interface_ip)) - host.ip = interface_ip - if self.check_ip_address(interface['ipv4_gateway']) or self.check_ip_address(interface['ipv6_gateway']): - host.default_gateway_ip = interface['ipv4_gateway'] or interface['ipv6_gateway'] + #host.default_gateway_mac if interface['network_segment']: host.net_segment = interface['network_segment'] @@ -190,6 +216,7 @@ def merge_with_host(self, interface, workspace, parent_relation_db_id): host.description += '\n Interface data: {0}'.format(interface['description']) if type(interface['hostnames']) in (str, unicode): interface['hostnames'] = [interface['hostnames']] + for hostname_str in interface['hostnames']: if not hostname_str: # skip empty hostnames @@ -209,7 +236,7 @@ class ServiceImporter(object): DOC_TYPE = 'Service' def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): - # service was always above interface, not it's above host. + # service was always below interface, not it's below host. try: parent_id = document['parent'].split('.')[0] except KeyError: @@ -507,7 +534,6 @@ def get_importer_from_document(self, doc_type): 'Service': ServiceImporter, 'Note': NoteImporter, 'Credential': CredentialImporter, - 'Interface': InterfaceImporter, 'CommandRunInformation': CommandImporter, 'Workspace': WorkspaceImporter, 'Vulnerability': VulnerabilityImporter, @@ -532,11 +558,11 @@ def modular_crypt_pbkdf2_sha1(self, checksum, salt, iterations=1000): ) def convert_couchdb_hash(self, original_hash): - if not original_hash.startswith(COUCHDB_PASSWORD_PREXFIX): + if not original_hash.startswith(COUCHDB_PASSWORD_PREFIX): # Should be a plaintext password return original_hash checksum, salt, iterations = original_hash[ - len(COUCHDB_PASSWORD_PREXFIX):].split(',') + len(COUCHDB_PASSWORD_PREFIX):].split(',') iterations = int(iterations) return self.modular_crypt_pbkdf2_sha1(checksum, salt, iterations) @@ -645,11 +671,6 @@ def run(self): name=ref_doc) - - - - - class ImportCouchDB(FlaskScriptCommand): def _open_couchdb_conn(self): @@ -751,7 +772,7 @@ def import_workspace_into_database(self, workspace_name): # obj_types are tuples. the first value is the level on the tree # for the desired obj. - # the idea is to improt by level from couchdb data. + # the idea is to import by level from couchdb data. obj_types = [ (1, 'Host'), (1, 'EntityMetadata'), @@ -761,7 +782,6 @@ def import_workspace_into_database(self, workspace_name): (1, 'Task'), (1, 'Workspace'), (1, 'Reports'), - (2, 'Interface'), (2, 'Service'), (2, 'Credential'), (2, 'Vulnerability'), From b2b19198b5870441189532965821f879c73245d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 6 Sep 2017 16:41:36 -0300 Subject: [PATCH 0166/1506] Fix rollback issues when creating a host with an existing IP Detailed list of changes: - When creating host with an IP that already exists in that workspace, rollback (so it isn't saved because it would throw an IntegrityError) and abort - Use custom SQLAlchemy to correctly handle rollbacks in SQLite Based on http://docs.sqlalchemy.org/en/rel_1_0/dialects/sqlite.html#serializable-isolation-savepoints-transactional-ddl but with support with Flask-SQLAlchemy - Improve session fixture to support functions doing rollbacks Based on http://docs.sqlalchemy.org/en/latest/orm/session_transaction.html#joining-a-session-into-an-external-transaction-such-as-for-test-suites - Add "second_workspace" fixture - Fixed test_create_a_host_fails_with_existing_ip and test_create_a_host_succeeds test cases to do an assertion based on the number of hosts in the database - Added empty test_create_a_host_with_ip_of_other_workspace test, TBI - Add commented line to activate echo in SQLAlchemy --- server/api/base.py | 1 + server/app.py | 1 + server/models.py | 46 +++++++++++++++++++++++++++- test_cases/conftest.py | 56 ++++++++++++++++++++++++++++++++++- test_cases/test_server_api.py | 12 ++++++-- 5 files changed, 111 insertions(+), 5 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 16e81a060da..554f99b0c9f 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -94,6 +94,7 @@ def _validate_uniqueness(self, obj): query = self._get_base_query(obj.workspace.name).filter( field==value) if query.one_or_none(): + db.session.rollback() abort(422, ValidationError('Existing value for %s field: %s' % ( field_name, value ))) diff --git a/server/app.py b/server/app.py index 957772923c8..d243f2798c4 100644 --- a/server/app.py +++ b/server/app.py @@ -27,6 +27,7 @@ def create_app(db_connection_string=None, testing=None): app.config['SECURITY_POST_LOGOUT_VIEW'] = '/_api/login' app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False if testing: + # app.config['SQLALCHEMY_ECHO'] = True app.config['TESTING'] = testing try: app.config['SQLALCHEMY_DATABASE_URI'] = db_connection_string or server.config.database.connection_string.strip("'") diff --git a/server/models.py b/server/models.py index b1b80fa3119..9f9a66ac2df 100644 --- a/server/models.py +++ b/server/models.py @@ -12,10 +12,14 @@ String, Text, UniqueConstraint, + event ) from sqlalchemy.orm import relationship, backref from sqlalchemy.ext.declarative import declared_attr -from flask_sqlalchemy import SQLAlchemy +from flask_sqlalchemy import ( + SQLAlchemy as OriginalSQLAlchemy, + _EngineConnector +) from flask_security import ( RoleMixin, UserMixin, @@ -23,6 +27,46 @@ import server.config + +class SQLAlchemy(OriginalSQLAlchemy): + """Override to fix issues when doing a rollback with sqlite driver + See http://docs.sqlalchemy.org/en/rel_1_0/dialects/sqlite.html#serializable-isolation-savepoints-transactional-ddl + and https://bitbucket.org/zzzeek/sqlalchemy/issues/3561/sqlite-nested-transactions-fail-with + for furhter information""" + + # TODO: only do this on sqlite, not on postgres! + + def make_connector(self, app=None, bind=None): + """Creates the connector for a given state and bind.""" + return CustomEngineConnector(self, self.get_app(app), bind) + + +class CustomEngineConnector(_EngineConnector): + """Used by overrideb SQLAlchemy class to fix rollback issues""" + + def get_engine(self): + # Use an existent engine and don't register events if possible + uri = self.get_uri() + echo = self._app.config['SQLALCHEMY_ECHO'] + if (uri, echo) == self._connected_for: + return self._engine + + # Call original metohd and register events + rv = super(CustomEngineConnector, self).get_engine() + with self._lock: + @event.listens_for(rv, "connect") + def do_connect(dbapi_connection, connection_record): + # disable pysqlite's emitting of the BEGIN statement entirely. + # also stops it from emitting COMMIT before any DDL. + dbapi_connection.isolation_level = None + + @event.listens_for(rv, "begin") + def do_begin(conn): + # emit our own BEGIN + conn.execute("BEGIN") + return rv + + db = SQLAlchemy() SCHEMA_VERSION = 'W.3.0.0' diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 57567edd662..f67c569e282 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -3,6 +3,7 @@ import json import pytest from flask.testing import FlaskClient +from sqlalchemy import event from pytest_factoryboy import register sys.path.append(os.path.abspath(os.getcwd())) @@ -22,6 +23,8 @@ for factory in enabled_factories: register(factory) +register(factories.WorkspaceFactory, "second_workspace") + class CustomClient(FlaskClient): @@ -74,7 +77,7 @@ def teardown(): @pytest.fixture(scope='function') -def session(database, request): +def fake_session(database, request): connection = database.engine.connect() transaction = connection.begin() @@ -87,9 +90,59 @@ def session(database, request): for factory in enabled_factories: factory._meta.sqlalchemy_session = session + def teardown(): + # rollback - everything that happened with the + # Session above (including calls to commit()) + # is rolled back. + # be careful with this!!!!! + transaction.rollback() + connection.close() + session.remove() + request.addfinalizer(teardown) + return session + + +@pytest.fixture(scope='function') +def session(database, request): + """Use this fixture if the function being tested does a session + rollback. + + See http://docs.sqlalchemy.org/en/latest/orm/session_transaction.html#joining-a-session-into-an-external-transaction-such-as-for-test-suites + for further information + """ + connection = database.engine.connect() + transaction = connection.begin() + + options = {"bind": connection, 'binds': {}} + session = db.create_scoped_session(options=options) + + # start the session in a SAVEPOINT... + session.begin_nested() + + # then each time that SAVEPOINT ends, reopen it + @event.listens_for(session, "after_transaction_end") + def restart_savepoint(session, transaction): + if transaction.nested and not transaction._parent.nested: + + # ensure that state is expired the way + # session.commit() at the top level normally does + # (optional step) + session.expire_all() + + session.begin_nested() + + database.session = session + db.session = session + + for factory in enabled_factories: + factory._meta.sqlalchemy_session = session def teardown(): + # rollback - everything that happened with the + # Session above (including calls to commit()) + # is rolled back. + # be careful with this!!!!! transaction.rollback() connection.close() session.remove() @@ -97,6 +150,7 @@ def teardown(): request.addfinalizer(teardown) return session + @pytest.fixture def test_client(app): return app.test_client() diff --git a/test_cases/test_server_api.py b/test_cases/test_server_api.py index 69e8fa3b3b1..b3b04b4888c 100644 --- a/test_cases/test_server_api.py +++ b/test_cases/test_server_api.py @@ -52,8 +52,8 @@ def test_create_a_host_succeeds(self, test_client): "description": "aaaaa", # os is not required }) - print res.data assert res.status_code == 201 + assert Host.query.count() == HOSTS_COUNT + 1 host_id = res.json['id'] host = Host.query.get(host_id) assert host.ip == "127.0.0.1" @@ -67,12 +67,18 @@ def test_create_a_host_fails_with_missing_desc(self, test_client): }) assert res.status_code == 400 - def test_create_a_host_fails_with_existing_ip(self, test_client, - session, host): + def test_create_a_host_fails_with_existing_ip(self, session, + test_client, host): session.add(host) session.commit() + res = test_client.post(self.url(), data={ "ip": host.ip, "description": "aaaaa", }) assert res.status_code == 400 + assert Host.query.count() == HOSTS_COUNT + 1 + +# def test_create_a_host_with_ip_of_other_workspace(self, test_client, +# second_workspace, host): +# pass From 70142344b95aaf6eb0cc865350a3fee17e875268 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 6 Sep 2017 16:53:07 -0300 Subject: [PATCH 0167/1506] Add generic model to comment any model --- server/models.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/server/models.py b/server/models.py index 87fb9f0ab7e..59e652624f4 100644 --- a/server/models.py +++ b/server/models.py @@ -726,6 +726,17 @@ class TagObject(db.Model): tag_id = Column(Integer, ForeignKey('tag.id'), index=True) +class CommentObject(db.Model): + __tablename__ = 'comment_object' + id = Column(Integer, primary_key=True) + + object_id = Column(Integer, nullable=False) + object_type = Column(Text, nullable=False) + + comment = relationship('Comment', backref='comment_objects') + comment_id = Column(Integer, ForeignKey('comment.id'), index=True) + + class Comment(db.Model): __tablename__ = 'comment' id = Column(Integer, primary_key=True) @@ -739,12 +750,10 @@ class Comment(db.Model): foreign_keys=[reply_to_id] ) - object_id = Column(Integer, nullable=False) - object_type = Column(Text, nullable=False) - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship('Workspace', foreign_keys=[workspace_id]) + class ExecutiveReport(db.Model): STATUSES = [ 'created', From 67d6299e5d738b756615c5f5aa55a6ef625608b7 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 6 Sep 2017 16:53:34 -0300 Subject: [PATCH 0168/1506] Add communication and license importer --- server/importer.py | 62 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 8 deletions(-) diff --git a/server/importer.py b/server/importer.py index 10a5d066923..5b0922a921d 100644 --- a/server/importer.py +++ b/server/importer.py @@ -8,21 +8,17 @@ from binascii import unhexlify import requests -from tqdm import tqdm from IPy import IP from flask_script import Command as FlaskScriptCommand -from restkit.errors import RequestError, Unauthorized -from IPy import IP from passlib.utils.binary import ab64_encode -from passlib.hash import pbkdf2_sha1 +from restkit.errors import RequestError, Unauthorized +from tqdm import tqdm -from server.web import app -import server.utils.logger +import server.config import server.couchdb import server.database import server.models -import server.config -from server.utils.database import get_or_create +import server.utils.logger from server.models import ( db, EntityMetadata, @@ -44,7 +40,12 @@ ExecutiveReport, VulnerabilityTemplate, ReferenceTemplate, + License, + Comment, + CommentObject, ) +from server.utils.database import get_or_create +from server.web import app COUCHDB_USER_PREFIX = 'org.couchdb.user:' COUCHDB_PASSWORD_PREFIX = '-pbkdf2-' @@ -512,6 +513,23 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation yield report +class CommunicationImporter(object): + def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): + comment, created = get_or_create( + session, + Comment, + text=document.get('text'), + workspace=workspace) + + get_or_create( + session, + CommentObject, + object_id=workspace.id, + object_type='Workspace', + comment=comment, + ) + yield comment + class FaradayEntityImporter(object): # Document Types: [u'Service', u'Communication', u'Vulnerability', u'CommandRunInformation', u'Reports', u'Host', u'Workspace'] @@ -541,6 +559,7 @@ def get_importer_from_document(self, doc_type): 'TaskGroup': MethodologyImporter, 'Task': TaskImporter, 'Reports': ReportsImporter, + 'Communication': CommunicationImporter } importer_self = importer_class_mapper.get(doc_type, None) if not importer_self: @@ -671,6 +690,30 @@ def run(self): name=ref_doc) +class ImportLicense(FlaskScriptCommand): + + def run(self): + cwe_url = "http://{username}:{password}@{hostname}:{port}/{path}".format( + username=server.config.couchdb.user, + password=server.config.couchdb.password, + hostname=server.config.couchdb.host, + port=server.config.couchdb.port, + path='faraday_licenses/_all_docs?include_docs=true' + ) + licenses = requests.get(cwe_url).json()['rows'] + for license in licenses: + document = license['doc'] + + license_obj, created = get_or_create(session, + License, + product=document.get('product'), + start_date=datetime.datetime.strptime(document['start'], "%Y-%m-%dT%H:%M:%S.%fZ"), + end_date=datetime.datetime.strptime(document['end'], "%Y-%m-%dT%H:%M:%S.%fZ"), + notes=document.get('notes'), + type=document.get('lictype') + ) + + class ImportCouchDB(FlaskScriptCommand): def _open_couchdb_conn(self): @@ -695,6 +738,8 @@ def run(self): """ Main entry point for couchdb import """ + license_import = ImportLicense() + license_import.run() vuln_templates_import = ImportVulnerabilityTemplates() vuln_templates_import.run() users_import = ImportCouchDBUsers() @@ -780,6 +825,7 @@ def import_workspace_into_database(self, workspace_name): (1, 'CommandRunInformation'), (1, 'TaskGroup'), (1, 'Task'), + (1, 'Communication'), (1, 'Workspace'), (1, 'Reports'), (2, 'Service'), From 0d88d92fba046ba11438722feb2a5b62e0e198dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 6 Sep 2017 18:18:35 -0300 Subject: [PATCH 0169/1506] Add test_create_a_host_with_ip_of_other_workspace test --- test_cases/test_server_api.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test_cases/test_server_api.py b/test_cases/test_server_api.py index b3b04b4888c..d95c97f07d6 100644 --- a/test_cases/test_server_api.py +++ b/test_cases/test_server_api.py @@ -79,6 +79,16 @@ def test_create_a_host_fails_with_existing_ip(self, session, assert res.status_code == 400 assert Host.query.count() == HOSTS_COUNT + 1 -# def test_create_a_host_with_ip_of_other_workspace(self, test_client, -# second_workspace, host): -# pass + def test_create_a_host_with_ip_of_other_workspace(self, test_client, + session, + second_workspace, host): + session.add(host) + session.commit() + + res = test_client.post(self.url(workspace=second_workspace), data={ + "ip": host.ip, + "description": "aaaaa", + }) + assert res.status_code == 201 + # It should create two hosts, one for each workspace + assert Host.query.count() == HOSTS_COUNT + 2 From baed666a962e43133dfb3c6464c95d3a3b1b763e Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 6 Sep 2017 18:34:57 -0300 Subject: [PATCH 0170/1506] Add function to query CouchDB objs by level and parent Adds a function to the importer that uploads a view to CouchDB and then queries it. --- server/importer.py | 101 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 76 insertions(+), 25 deletions(-) diff --git a/server/importer.py b/server/importer.py index cd0f5ad8e87..aff9caad5ec 100644 --- a/server/importer.py +++ b/server/importer.py @@ -53,6 +53,81 @@ logger = server.utils.logger.get_logger(__name__) session = db.session +OBJ_TYPES = [ + (1, 'Host'), + (1, 'EntityMetadata'), + (1, 'Note'), + (1, 'CommandRunInformation'), + (1, 'TaskGroup'), + (1, 'Task'), + (1, 'Workspace'), + (1, 'Reports'), + (2, 'Interface'), + (2, 'Service'), + (2, 'Credential'), + (2, 'Vulnerability'), + (2, 'VulnerabilityWeb'), + (3, 'Service'), + (4, 'Credential'), # Level 4 is for interface + (4, 'Vulnerability'), + (4, 'VulnerabilityWeb'), + ] + +def get_children_from_couch(workspace, parent_couchdb_id, child_type, level): + """ + Performance for temporary views suck, so this method uploads a view and queries it instead + + :param workspace: workspace to upload the view + :param parent_couchdb_id: ID of the parent document + :param child_type: type of the child obj we're looking for + :param level: level of the child obj we're looking for, must match those in OBJ_TYPES + :return: + """ + if (level, child_type) not in OBJ_TYPES: + logger.warn('Unable to retrieve children of type {0} at level {1}'.format(child_type, level)) + return [] + + couch_url = "http://{username}:{password}@{hostname}:{port}/{workspace_name}/".format( + username=server.config.couchdb.user, + password=server.config.couchdb.password, + hostname=server.config.couchdb.host, + port=server.config.couchdb.port, + workspace_name=workspace.name, + ) + + # create the new view + view_url = "{}_design/importer".format(couch_url) + view_data = { + "views": { + "children_by_parent_and_type": { + "map": "function(doc) { id_parent = doc._id.split('.').slice(0, -1).join('.');" + "key = [id_parent,doc.type]; emit(key, doc); }" + } + } + } + + try: + r = requests.put(view_url, json=view_data) + except requests.exceptions.RequestException as e: + logger.warn(e) + return [] + + # and now, finally query it! + couch_url += "_design/importer/_view/children_by_parent_and_type?" \ + "startkey=[\"{parent_id}\",\"{child_type}\"]&" \ + "endkey=[\"{parent_id}\",\"{child_type}\"]".format( + parent_id=parent_couchdb_id, + child_type=child_type, + ) + + try: + r = requests.get(couch_url) + except requests.exceptions.RequestException as e: + logger.warn(e) + return [] + + return r.json()['rows'] + class EntityNotFound(Exception): def __init__(self, entity_id): @@ -645,13 +720,7 @@ def run(self): name=ref_doc) - - - - - class ImportCouchDB(FlaskScriptCommand): - def _open_couchdb_conn(self): try: couchdb_server_conn = server.couchdb.CouchDBServer() @@ -752,25 +821,7 @@ def import_workspace_into_database(self, workspace_name): # obj_types are tuples. the first value is the level on the tree # for the desired obj. # the idea is to improt by level from couchdb data. - obj_types = [ - (1, 'Host'), - (1, 'EntityMetadata'), - (1, 'Note'), - (1, 'CommandRunInformation'), - (1, 'TaskGroup'), - (1, 'Task'), - (1, 'Workspace'), - (1, 'Reports'), - (2, 'Interface'), - (2, 'Service'), - (2, 'Credential'), - (2, 'Vulnerability'), - (2, 'VulnerabilityWeb'), - (3, 'Service'), - (4, 'Credential'), # Level 4 is for interface - (4, 'Vulnerability'), - (4, 'VulnerabilityWeb'), - ] + obj_types = OBJ_TYPES couchdb_relational_map = {} couchdb_removed_objs = set() removed_objs = ['Interface'] From 82bbcbc7893319b29a2125c5d0ea6e0224c189d3 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 7 Sep 2017 13:03:18 -0300 Subject: [PATCH 0171/1506] Use new function to retrieve children. Generate multiple hosts * We generate multiple host for each interface and also for ipv4 and ipv6, one for each. * For each new host we generate vulns. So if a host with ipv4 and ipv6 set the vulns will be duplicate, the same for services. Not the host has only one field * Temporal views were very slow for retrieve multiple children and also caused couchdb to collapse. --- server/importer.py | 287 ++++++++++++++++++++++----------------------- 1 file changed, 137 insertions(+), 150 deletions(-) diff --git a/server/importer.py b/server/importer.py index 597ed78f018..baa57341396 100644 --- a/server/importer.py +++ b/server/importer.py @@ -5,6 +5,7 @@ import sys import json import datetime +from collections import defaultdict from binascii import unhexlify import requests @@ -74,19 +75,15 @@ (4, 'VulnerabilityWeb'), ] -def get_children_from_couch(workspace, parent_couchdb_id, child_type, level): +def get_children_from_couch(workspace, parent_couchdb_id, child_type): """ Performance for temporary views suck, so this method uploads a view and queries it instead :param workspace: workspace to upload the view :param parent_couchdb_id: ID of the parent document :param child_type: type of the child obj we're looking for - :param level: level of the child obj we're looking for, must match those in OBJ_TYPES :return: """ - if (level, child_type) not in OBJ_TYPES: - logger.warn('Unable to retrieve children of type {0} at level {1}'.format(child_type, level)) - return [] couch_url = "http://{username}:{password}@{hostname}:{port}/{workspace_name}/".format( username=server.config.couchdb.user, @@ -173,8 +170,6 @@ def check_ip_address(ip_str): return False if ip_str == '0000:0000:0000:0000:0000:0000:0000:0000': return False - if ip_str == '0000:0000:0000:0000:0000:0000:0000:0001': - return False try: IP(ip_str) except ValueError: @@ -218,7 +213,7 @@ def retrieve_ips_from_host_document(self, document): def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): hosts = [] host_ips = [name_or_ip for name_or_ip in self.retrieve_ips_from_host_document(document)] - interfaces = self.host_interfaces_from_couch(workspace, document.get('_id')) + interfaces = get_children_from_couch(workspace, document.get('_id'), 'Interface') for interface in interfaces: interface = interface['value'] if check_ip_address(interface['ipv4']['address']): @@ -260,25 +255,6 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation host.workspace = workspace yield host - def host_interfaces_from_couch(self, workspace, host_couchdb_id): - couch_url = "http://{username}:{password}@{hostname}:{port}/{workspace_name}/_temp_view?include_docs=true".format( - username=server.config.couchdb.user, - password=server.config.couchdb.password, - hostname=server.config.couchdb.host, - port=server.config.couchdb.port, - workspace_name=workspace.name - ) - data = { - "map": "function(doc) { if(doc.type == '%s' && doc._id.split('.').slice(0, 1).join('.') == '%s') emit(null, doc); }" % ( - 'Interface', host_couchdb_id) - } - - r = requests.post(couch_url, json=data) - try: - return r.json()['rows'] - except Exception as ex: - print('sadsa') - def merge_with_host(self, host, interface, workspace): if interface['mac']: host.mac = interface['mac'] @@ -318,34 +294,35 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation except KeyError: # some services are missing the parent key parent_id = document['_id'].split('.')[0] - host, created = get_or_create(session, Host, id=couchdb_relational_map[parent_id]) - ports = document.get('ports') - if len(ports) > 2: - logger.warn('More than one port found in services!') - for port in ports: - service, created = get_or_create(session, - Service, - name=document.get('name'), - port=port, - host=host) - service.description = document.get('description') - service.owned = document.get('owned', False) - service.banner = document.get('banner') - service.protocol = document.get('protocol') - if not document.get('status'): - logger.warning('Service {0} with empty status. Using open as status'.format(document['_id'])) - document['status'] = 'open' - status_mapper = { - 'open': 'open', - 'closed': 'closed', - 'filtered': 'filtered', - 'open|filtered': 'filtered' - } - service.status = status_mapper[document.get('status')] - service.version = document.get('version') - service.workspace = workspace - - yield service + for relational_parent_id in couchdb_relational_map[parent_id]: + host, created = get_or_create(session, Host, id=relational_parent_id) + ports = document.get('ports') + if len(ports) > 2: + logger.warn('More than one port found in services!') + for port in ports: + service, created = get_or_create(session, + Service, + name=document.get('name'), + port=port, + host=host) + service.description = document.get('description') + service.owned = document.get('owned', False) + service.banner = document.get('banner') + service.protocol = document.get('protocol') + if not document.get('status'): + logger.warning('Service {0} with empty status. Using open as status'.format(document['_id'])) + document['status'] = 'open' + status_mapper = { + 'open': 'open', + 'closed': 'closed', + 'filtered': 'filtered', + 'open|filtered': 'filtered' + } + service.status = status_mapper[document.get('status')] + service.version = document.get('version') + service.workspace = workspace + + yield service class VulnerabilityImporter(object): @@ -355,76 +332,77 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation couch_parent_id = document.get('parent', None) if not couch_parent_id: couch_parent_id = '.'.join(document['_id'].split('.')[:-1]) - parent_id = couchdb_relational_map[couch_parent_id] - if level == 2: - parent = session.query(Host).filter_by(id=parent_id).first() - if level == 4: - parent = session.query(Service).filter_by(id=parent_id).first() - if document['type'] == 'VulnerabilityWeb': - vulnerability, created = get_or_create( - session, - VulnerabilityWeb, - name=document.get('name'), - description=document.get('desc'), - service_id=parent.id, - ) - if document['type'] == 'Vulnerability': - vuln_params = { - 'name': document.get('name'), - 'description': document.get('desc') - } - if type(parent) == Host: - vuln_params.update({'host_id': parent.id}) - elif type(parent) == Service: - vuln_params.update({'service_id': parent.id}) - vulnerability, created = get_or_create( - session, - Vulnerability, - **vuln_params - ) + parent_ids = couchdb_relational_map[couch_parent_id] + for parent_id in parent_ids: + if level == 2: + parent = session.query(Host).filter_by(id=parent_id).first() + if level == 4: + parent = session.query(Service).filter_by(id=parent_id).first() + if document['type'] == 'VulnerabilityWeb': + vulnerability, created = get_or_create( + session, + VulnerabilityWeb, + name=document.get('name'), + description=document.get('desc'), + service_id=parent.id, + ) + if document['type'] == 'Vulnerability': + vuln_params = { + 'name': document.get('name'), + 'description': document.get('desc') + } + if type(parent) == Host: + vuln_params.update({'host_id': parent.id}) + elif type(parent) == Service: + vuln_params.update({'service_id': parent.id}) + vulnerability, created = get_or_create( + session, + Vulnerability, + **vuln_params + ) - vulnerability.confirmed = document.get('confirmed', False) or False - vulnerability.data = document.get('data') - vulnerability.easeofresolution = document.get('easeofresolution') - vulnerability.resolution = document.get('resolution') - mapped_severity = { - 'med': 'medium', - 'critical': 'critical', - 'high':'high', - 'low': 'low', - 'info': 'informational', - 'unclassified': 'unclassified', - } - vulnerability.severity = mapped_severity[document.get('severity')] - vulnerability.owned = document.get('owned', False) - #vulnerability.attachments = json.dumps(document.get('_attachments', {})) - vulnerability.impact_accountability = document.get('impact', {}).get('accountability') - vulnerability.impact_availability = document.get('impact', {}).get('availability') - vulnerability.impact_confidentiality = document.get('impact', {}).get('confidentiality') - vulnerability.impact_integrity = document.get('impact', {}).get('integrity') - if document['type'] == 'VulnerabilityWeb': - vulnerability.method = document.get('method') - vulnerability.path = document.get('path') - vulnerability.pname = document.get('pname') - vulnerability.query = document.get('query') - vulnerability.request = document.get('request') - vulnerability.response = document.get('response') - vulnerability.website = document.get('website') - params = document.get('params', u'') - if isinstance(params, (list, tuple)): - vulnerability.parameters = (u' '.join(params)).strip() - else: - vulnerability.parameters = params if params is not None else u'' - status_map = { - 'opened': 'open', - 'closed': 'closed', - } - status = status_map[document.get('status', 'opened')] - vulnerability.status = status - vulnerability.workspace = workspace + vulnerability.confirmed = document.get('confirmed', False) or False + vulnerability.data = document.get('data') + vulnerability.easeofresolution = document.get('easeofresolution') + vulnerability.resolution = document.get('resolution') + mapped_severity = { + 'med': 'medium', + 'critical': 'critical', + 'high':'high', + 'low': 'low', + 'info': 'informational', + 'unclassified': 'unclassified', + } + vulnerability.severity = mapped_severity[document.get('severity')] + vulnerability.owned = document.get('owned', False) + #vulnerability.attachments = json.dumps(document.get('_attachments', {})) + vulnerability.impact_accountability = document.get('impact', {}).get('accountability') + vulnerability.impact_availability = document.get('impact', {}).get('availability') + vulnerability.impact_confidentiality = document.get('impact', {}).get('confidentiality') + vulnerability.impact_integrity = document.get('impact', {}).get('integrity') + if document['type'] == 'VulnerabilityWeb': + vulnerability.method = document.get('method') + vulnerability.path = document.get('path') + vulnerability.pname = document.get('pname') + vulnerability.query = document.get('query') + vulnerability.request = document.get('request') + vulnerability.response = document.get('response') + vulnerability.website = document.get('website') + params = document.get('params', u'') + if isinstance(params, (list, tuple)): + vulnerability.parameters = (u' '.join(params)).strip() + else: + vulnerability.parameters = params if params is not None else u'' + status_map = { + 'opened': 'open', + 'closed': 'closed', + } + status = status_map[document.get('status', 'opened')] + vulnerability.status = status + vulnerability.workspace = workspace - self.add_references(document, vulnerability, workspace) - self.add_policy_violations(document, vulnerability, workspace) + self.add_references(document, vulnerability, workspace) + self.add_policy_violations(document, vulnerability, workspace) yield vulnerability def add_policy_violations(self, document, vulnerability, workspace): @@ -489,22 +467,28 @@ class CredentialImporter(object): DOC_TYPE = 'Cred' def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): - host = None - service = None + parents = [] if level == 2: - parent_id = couchdb_relational_map[document['_id'].split('.')[0]] - host = session.query(Host).filter_by(id=parent_id).first() + parent_ids = couchdb_relational_map[document['_id'].split('.')[0]] + parents = session.query(Host).filter(Host.id.in_(parent_ids)).all() if level == 4: - parent_id = couchdb_relational_map['.'.join(document['_id'].split('.')[:3])] - service = session.query(Service).filter_by(id=parent_id).first() - if not host and not service: + parent_ids = couchdb_relational_map['.'.join(document['_id'].split('.')[:3])] + parents = session.query(Service).filter(Host.id.in_(parent_ids)).all() + if not parents: raise Exception('Missing host or service for credential {0}'.format(document['_id'])) - credential, created = get_or_create(session, Credential, username=document.get('username'), host=host, service=service) - credential.password = document.get('password', None) - credential.owned = document.get('owned', False) - credential.description = document.get('description', None) - credential.name = document.get('name', None) - credential.workspace = workspace + for parent in parents: + service = None + host = None + if isinstance(parent, Host): + host = parent + if isinstance(parent, Service): + service = parent + credential, created = get_or_create(session, Credential, username=document.get('username'), host=host, service=service) + credential.password = document.get('password', None) + credential.owned = document.get('owned', False) + credential.description = document.get('description', None) + credential.name = document.get('name', None) + credential.workspace = workspace yield credential @@ -539,10 +523,13 @@ class TaskImporter(object): def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): try: - methodology_id = couchdb_relational_map[document.get('group_id')] + methodology_id = couchdb_relational_map[document.get('group_id')][0] except KeyError: logger.warn('Could not found methodology with id {0}'.format(document.get('group_id'))) return [] + if len(couchdb_relational_map[document.get('group_id')]) > 1: + logger.error('It was expected only one parent in methodology {0}'.format(document.get('_id'))) + methodology = session.query(Methodology).filter_by(id=methodology_id).first() task_class = Task if not methodology: @@ -841,7 +828,7 @@ def get_objs(self, host, obj_type, level): return r.json() - def verify_import_data(self, couchdb_relational_map, couchdb_removed_objs, workspace): + def verify_import_data(self, couchdb_relational_map, workspace): all_docs_url = "http://{username}:{password}@{hostname}:{port}/{workspace_name}/_all_docs?include_docs=true".format( username=server.config.couchdb.user, password=server.config.couchdb.password, @@ -850,11 +837,14 @@ def verify_import_data(self, couchdb_relational_map, couchdb_removed_objs, works workspace_name=workspace.name ) all_ids = map(lambda x: x['doc']['_id'], requests.get(all_docs_url).json()['rows']) - if len(all_ids) != len(couchdb_relational_map.keys()) + len(couchdb_removed_objs): + if len(all_ids) != len(couchdb_relational_map.keys()): missing_objs_filename = os.path.join(os.path.expanduser('~/.faraday'), 'logs', 'import_missing_objects_{0}.json'.format(workspace.name)) - missing_ids = set(all_ids) - set(couchdb_relational_map.keys()).union(couchdb_removed_objs) + missing_ids = set(all_ids) - set(couchdb_relational_map.keys()) + missing_ids = missing_ids - set([u'_design/commands', u'_design/hosts', u'_design/comms', u'_design/tags', u'_design/vulns', u'_design/utils', u'_design/importer', u'_design/auth', u'_design/mapper', u'_design/services', u'_design/interfaces', u'_design/changes', u'_design/reports', +]) objs_diff = [] - logger.info('Downloading missing couchdb docs') + if missing_ids: + logger.info('Downloading missing couchdb docs') for missing_id in tqdm(missing_ids): doc_url = 'http://{username}:{password}@{hostname}:{port}/{workspace_name}/{doc_id}'.format( username=server.config.couchdb.user, @@ -865,6 +855,9 @@ def verify_import_data(self, couchdb_relational_map, couchdb_removed_objs, works doc_id=missing_id ) not_imported_obj = requests.get(doc_url).json() + if not_imported_obj['type'] == 'Interface': + # we know that interface obj was not imported + continue filter_keys = ['views', 'validate_doc_update'] if not any(map(lambda x: x not in filter_keys, not_imported_obj.keys())): # we filter custom views, validation funcs, etc @@ -892,9 +885,7 @@ def import_workspace_into_database(self, workspace_name): # obj_types are tuples. the first value is the level on the tree # for the desired obj. obj_types = OBJ_TYPES - couchdb_relational_map = {} - couchdb_removed_objs = set() - removed_objs = ['Interface'] + couchdb_relational_map = defaultdict(list) for level, obj_type in obj_types: obj_importer = faraday_importer.get_importer_from_document(obj_type)() objs_dict = self.get_objs(couch_url, obj_type, level) @@ -908,10 +899,6 @@ def import_workspace_into_database(self, workspace_name): if not new_obj: continue session.commit() - if obj_type not in removed_objs: - couchdb_relational_map[couchdb_id] = new_obj.id - else: - couchdb_relational_map[couchdb_id] = new_obj['parent_id'] - couchdb_removed_objs.add(couchdb_id) - self.verify_import_data(couchdb_relational_map, couchdb_removed_objs, workspace) + couchdb_relational_map[couchdb_id].append(new_obj.id) + self.verify_import_data(couchdb_relational_map, workspace) return created From c0cd3c7877d5b741149ca24765e91db3fdf3cef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 7 Sep 2017 13:22:54 -0300 Subject: [PATCH 0172/1506] Add update endpoint logic --- server/api/base.py | 30 +++++++++++++++++++++++++++++- test_cases/test_server_api.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/server/api/base.py b/server/api/base.py index 554f99b0c9f..ccf493a2da2 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -3,6 +3,7 @@ from flask_classful import FlaskView from sqlalchemy.orm.exc import NoResultFound +from sqlalchemy.inspection import inspect from werkzeug.routing import parse_rule from webargs.flaskparser import FlaskParser, abort from webargs.core import ValidationError @@ -85,14 +86,19 @@ def _parse_data(self, schema, request, *args, **kwargs): return FlaskParser().parse(schema, request, locations=('json',), *args, **kwargs) - def _validate_uniqueness(self, obj): + def _validate_uniqueness(self, obj, object_id=None): assert obj.workspace is not None, "Object must have a " \ "workspace attribute set to call _validate_uniqueness" + primary_key_field = inspect(self.model_class).primary_key[0] for field_name in self.unique_fields: field = getattr(self.model_class, field_name) value = getattr(obj, field_name) query = self._get_base_query(obj.workspace.name).filter( field==value) + if object_id is not None: + # The object already exists in DB, we want to fetch an object + # different to this one but with the same unique field + query = query.filter(primary_key_field != object_id) if query.one_or_none(): db.session.rollback() abort(422, ValidationError('Existing value for %s field: %s' % ( @@ -160,7 +166,29 @@ def _perform_create(self, workspace_name, obj): return obj +class UpdateWorkspacedMixin(object): + def put(self, workspace_name, object_id): + data = self._parse_data(self._get_schema_class()(strict=True), + flask.request) + obj = self._get_object(workspace_name, object_id) + self._update_object(obj, data) + updated = self._perform_update(workspace_name, object_id, obj) + return self._dump(obj).data, 200 + + def _update_object(self, obj, data): + for (key, value) in data.items(): + setattr(obj, key, value) + + def _perform_update(self, workspace_name, object_id, obj): + with db.session.no_autoflush: + obj.workspace = self._get_workspace(workspace_name) + self._validate_uniqueness(obj, object_id) + db.session.add(obj) + db.session.commit() + + class ReadWriteWorkspacedView(CreateWorkspacedMixin, + UpdateWorkspacedMixin, ReadOnlyWorkspacedView, GenericWorkspacedView): """A generic view with list, retrieve and create endpoints""" diff --git a/test_cases/test_server_api.py b/test_cases/test_server_api.py index d95c97f07d6..37e25444242 100644 --- a/test_cases/test_server_api.py +++ b/test_cases/test_server_api.py @@ -92,3 +92,35 @@ def test_create_a_host_with_ip_of_other_workspace(self, test_client, assert res.status_code == 201 # It should create two hosts, one for each workspace assert Host.query.count() == HOSTS_COUNT + 2 + + def test_update_a_host(self, test_client): + host = self.workspace.hosts[0] + res = test_client.put(self.url(host), data={ + "ip": host.ip, + "description": "bbbbb", + }) + assert res.status_code == 200 + assert res.json['description'] == 'bbbbb' + assert Host.query.get(res.json['id']).description == 'bbbbb' + assert Host.query.count() == HOSTS_COUNT + + def test_update_a_host_fails_with_existing_ip(self, test_client, session): + host = self.workspace.hosts[0] + original_ip = host.ip + original_desc = host.description + res = test_client.put(self.url(host), data={ + "ip": self.workspace.hosts[1].ip, # Existing IP + "description": "bbbbb", + }) + assert res.status_code == 400 + session.refresh(host) + assert host.ip == original_ip + assert host.description == original_desc # It shouldn't do a partial update + + def test_update_a_host_fails_with_missing_fields(self, test_client): + """To do this the user should use a PATCH request""" + host = self.workspace.hosts[0] + res = test_client.put(self.url(host), data={ + "ip": "1.2.3.4", # Existing IP + }) + assert res.status_code == 400 From 8b8425ff98d58a32329be134b83fa8e72d1490cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 7 Sep 2017 13:27:06 -0300 Subject: [PATCH 0173/1506] Add delete endpoint logic --- server/api/base.py | 12 ++++++++++++ test_cases/test_server_api.py | 14 ++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/server/api/base.py b/server/api/base.py index ccf493a2da2..5020b03b530 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -187,8 +187,20 @@ def _perform_update(self, workspace_name, object_id, obj): db.session.commit() +class DeleteWorkspacedMixin(object): + def delete(self, workspace_name, object_id): + obj = self._get_object(workspace_name, object_id) + self._perform_delete(obj) + return None, 204 + + def _perform_delete(self, obj): + db.session.delete(obj) + db.session.commit() + + class ReadWriteWorkspacedView(CreateWorkspacedMixin, UpdateWorkspacedMixin, + DeleteWorkspacedMixin, ReadOnlyWorkspacedView, GenericWorkspacedView): """A generic view with list, retrieve and create endpoints""" diff --git a/test_cases/test_server_api.py b/test_cases/test_server_api.py index 37e25444242..1855e17e28d 100644 --- a/test_cases/test_server_api.py +++ b/test_cases/test_server_api.py @@ -1,4 +1,5 @@ import pytest +from sqlalchemy.orm.util import was_deleted from test_cases import factories from server.models import db, Workspace, Host @@ -124,3 +125,16 @@ def test_update_a_host_fails_with_missing_fields(self, test_client): "ip": "1.2.3.4", # Existing IP }) assert res.status_code == 400 + + def test_delete_a_host(self, test_client): + host = self.workspace.hosts[0] + res = test_client.delete(self.url(host)) + assert res.status_code == 204 # No content + assert was_deleted(host) + + def test_delete_host_from_other_workspace_fails(self, test_client, + second_workspace): + host = self.workspace.hosts[0] + res = test_client.delete(self.url(host, workspace=second_workspace)) + assert res.status_code == 404 # No content + assert not was_deleted(host) From 77a063cd56d6cbbb58ab34a0e09bbf3ceb17e3e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 7 Sep 2017 15:05:05 -0300 Subject: [PATCH 0174/1506] Improve Host API tests Check that list doesn't show hosts of other workspace, use session instead of database.session and remove unused old comment --- test_cases/test_server_api.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test_cases/test_server_api.py b/test_cases/test_server_api.py index 1855e17e28d..2e19d3ac62c 100644 --- a/test_cases/test_server_api.py +++ b/test_cases/test_server_api.py @@ -12,7 +12,7 @@ class TestHostAPI: @pytest.fixture(autouse=True) def load_workspace_with_hosts(self, database, session, workspace, host_factory): host_factory.create_batch(HOSTS_COUNT, workspace=workspace) - database.session.commit() + session.commit() assert workspace.id is not None assert workspace.hosts[0].id is not None self.workspace = workspace @@ -25,13 +25,17 @@ def url(self, host=None, workspace=None): url += str(host.id) return url - def test_list_retrieves_all_items(self, test_client): + def test_list_retrieves_all_items_from_workspace(self, test_client, + second_workspace, + session, + host_factory): + other_host = host_factory.create(workspace=second_workspace) + session.commit() res = test_client.get(self.url()) assert res.status_code == 200 assert len(res.json) == HOSTS_COUNT def test_retrieve_one_host(self, test_client, database): - # self.workspace = Workspace.query.first() host = self.workspace.hosts[0] assert host.id is not None res = test_client.get(self.url(host)) From 4afc8dfe044cabbbaf79e80086b16675e119ae66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 7 Sep 2017 15:09:29 -0300 Subject: [PATCH 0175/1506] Do sqlite transaction hacks only when SQListe is configured Otherwise, using postresql or other RDBMS will fail --- server/models.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/server/models.py b/server/models.py index 9f9a66ac2df..5772835a083 100644 --- a/server/models.py +++ b/server/models.py @@ -34,8 +34,6 @@ class SQLAlchemy(OriginalSQLAlchemy): and https://bitbucket.org/zzzeek/sqlalchemy/issues/3561/sqlite-nested-transactions-fail-with for furhter information""" - # TODO: only do this on sqlite, not on postgres! - def make_connector(self, app=None, bind=None): """Creates the connector for a given state and bind.""" return CustomEngineConnector(self, self.get_app(app), bind) @@ -53,17 +51,19 @@ def get_engine(self): # Call original metohd and register events rv = super(CustomEngineConnector, self).get_engine() - with self._lock: - @event.listens_for(rv, "connect") - def do_connect(dbapi_connection, connection_record): - # disable pysqlite's emitting of the BEGIN statement entirely. - # also stops it from emitting COMMIT before any DDL. - dbapi_connection.isolation_level = None - - @event.listens_for(rv, "begin") - def do_begin(conn): - # emit our own BEGIN - conn.execute("BEGIN") + if uri.startswith('sqlite://'): + with self._lock: + @event.listens_for(rv, "connect") + def do_connect(dbapi_connection, connection_record): + # disable pysqlite's emitting of the BEGIN statement + # entirely. also stops it from emitting COMMIT before any + # DDL. + dbapi_connection.isolation_level = None + + @event.listens_for(rv, "begin") + def do_begin(conn): + # emit our own BEGIN + conn.execute("BEGIN") return rv From b0ad9a1ecbaf69196a13643b12b18130d08c78f5 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 7 Sep 2017 15:32:46 -0300 Subject: [PATCH 0176/1506] When methodology was deleted some task were not deleted --- server/importer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/importer.py b/server/importer.py index baa57341396..25972964d7e 100644 --- a/server/importer.py +++ b/server/importer.py @@ -527,6 +527,10 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation except KeyError: logger.warn('Could not found methodology with id {0}'.format(document.get('group_id'))) return [] + except IndexError: + logger.warn('Could not find methodology {0} of task {1}'.format(document.get('group_id'), document.get('_id'))) + return [] + if len(couchdb_relational_map[document.get('group_id')]) > 1: logger.error('It was expected only one parent in methodology {0}'.format(document.get('_id'))) From 99862fda190f7c3ac391d008c573af52dbae2a5e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 7 Sep 2017 16:55:19 -0300 Subject: [PATCH 0177/1506] Add new function get_object_from_couchdb and add Note importer --- server/importer.py | 50 ++++++++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/server/importer.py b/server/importer.py index 25972964d7e..8ebaaa2bbe2 100644 --- a/server/importer.py +++ b/server/importer.py @@ -65,6 +65,7 @@ (1, 'Workspace'), (1, 'Reports'), (1, 'Communication'), + (1, 'Note'), (2, 'Service'), (2, 'Credential'), (2, 'Vulnerability'), @@ -75,6 +76,19 @@ (4, 'VulnerabilityWeb'), ] + +def get_object_from_couchdb(couchdb_id, workspace): + doc_url = 'http://{username}:{password}@{hostname}:{port}/{workspace_name}/{doc_id}'.format( + username=server.config.couchdb.user, + password=server.config.couchdb.password, + hostname=server.config.couchdb.host, + port=server.config.couchdb.port, + workspace_name=workspace.name, + doc_id=couchdb_id + ) + return requests.get(doc_url).json() + + def get_children_from_couch(workspace, parent_couchdb_id, child_type): """ Performance for temporary views suck, so this method uploads a view and queries it instead @@ -452,15 +466,24 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation class NoteImporter(object): - DOC_TYPE = 'Note' def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): - note = Note() - note.name = document.get('name') - note.text = document.get('text', None) - note.description = document.get('description', None) - note.owned = document.get('owned', False) - yield note + couch_parent_id = '.'.join(document['_id'].split('.')[:-1]) + parent_document = get_object_from_couchdb(couch_parent_id, workspace) + comment, created = get_or_create( + session, + Comment, + text='{0}\n{1}'.format(document.get('text', ''), document.get('description', '')), + workspace=workspace) + + get_or_create( + session, + CommentObject, + object_id=couchdb_relational_map[parent_document.get('_id')], + object_type=parent_document['type'], + comment=comment, + ) + yield comment class CredentialImporter(object): @@ -741,7 +764,7 @@ def run(self): 'high':'high', 'low': 'low', 'info': 'informational', - 'unclassified': 'unclassified', + 'unclassified': 'unclassified', } vuln_template, created = get_or_create(session, VulnerabilityTemplate, @@ -850,15 +873,8 @@ def verify_import_data(self, couchdb_relational_map, workspace): if missing_ids: logger.info('Downloading missing couchdb docs') for missing_id in tqdm(missing_ids): - doc_url = 'http://{username}:{password}@{hostname}:{port}/{workspace_name}/{doc_id}'.format( - username=server.config.couchdb.user, - password=server.config.couchdb.password, - hostname=server.config.couchdb.host, - port=server.config.couchdb.port, - workspace_name=workspace.name, - doc_id=missing_id - ) - not_imported_obj = requests.get(doc_url).json() + not_imported_obj = get_object_from_couchdb(missing_id, workspace) + if not_imported_obj['type'] == 'Interface': # we know that interface obj was not imported continue From bd8244f90486eca951a9b4cb352acfd90cd7d5c6 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 7 Sep 2017 16:55:33 -0300 Subject: [PATCH 0178/1506] Remove TODO --- server/models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/models.py b/server/models.py index 59e652624f4..c2694f6b454 100644 --- a/server/models.py +++ b/server/models.py @@ -422,8 +422,6 @@ class PolicyViolation(db.Model): class Credential(db.Model): - # TODO: add unique constraint -> username, host o service y workspace - # TODO: add constraint host y service, uno o el otro __tablename__ = 'credential' id = Column(Integer, primary_key=True) username = Column(String(250), nullable=False) From fff9348960327f86aeae5e9bc6bb36ef2272d507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 7 Sep 2017 17:02:41 -0300 Subject: [PATCH 0179/1506] Add generic API tests --- test_cases/factories.py | 6 ++ test_cases/test_server_api.py | 163 ++++++++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+) diff --git a/test_cases/factories.py b/test_cases/factories.py index e6001554460..6001cc90b28 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -33,6 +33,12 @@ class Meta: class WorkspaceObjectFactory(FaradayFactory): workspace = factory.SubFactory(WorkspaceFactory) + @classmethod + def build_dict(cls, **kwargs): + ret = factory.build(dict, FACTORY_CLASS=cls) + del ret['workspace'] # It is passed in the URL, not in POST data + return ret + class HostFactory(WorkspaceObjectFactory): ip = factory.Faker('ipv4') diff --git a/test_cases/test_server_api.py b/test_cases/test_server_api.py index 2e19d3ac62c..16599ee0374 100644 --- a/test_cases/test_server_api.py +++ b/test_cases/test_server_api.py @@ -142,3 +142,166 @@ def test_delete_host_from_other_workspace_fails(self, test_client, res = test_client.delete(self.url(host, workspace=second_workspace)) assert res.status_code == 404 # No content assert not was_deleted(host) + + +PREFIX = '/v2/' +OBJECT_COUNT = 5 + + +@pytest.mark.usefixtures('logged_user') +class GenericAPITest: + + model = None + factory = None + api_endpoint = None + pk_field = 'id' + unique_fields = [] + update_fields = [] + + @pytest.fixture(autouse=True) + def load_workspace_with_objects(self, database, session, workspace): + objects = self.factory.create_batch(OBJECT_COUNT, workspace=workspace) + self.first_object = objects[0] + session.commit() + assert workspace.id is not None + self.workspace = workspace + return workspace + + @pytest.fixture + def object_instance(self, session, workspace): + """An object instance with the correct workspace assigned, + saved in the database""" + obj = self.factory.create(workspace=workspace) + session.commit() + return obj + + def url(self, obj=None, workspace=None): + workspace = workspace or self.workspace + url = PREFIX + workspace.name + '/' + self.api_endpoint + '/' + if obj is not None: + url += str(obj.id) + return url + + +class ListTestsMixin: + + def test_list_retrieves_all_items_from_workspace(self, test_client, + second_workspace, + session): + self.factory.create(workspace=second_workspace) + session.commit() + res = test_client.get(self.url()) + assert res.status_code == 200 + assert len(res.json) == OBJECT_COUNT + + +class RetrieveTestsMixin: + + def test_retrieve_one_object(self, test_client): + res = test_client.get(self.url(self.first_object)) + assert res.status_code == 200 + assert isinstance(res.json, dict) + + def test_retrieve_fails_object_of_other_workspcae(self, + test_client, + session, + second_workspace): + res = test_client.get(self.url(self.first_object, second_workspace)) + assert res.status_code == 404 + + +class CreateTestsMixin: + + def test_create_succeeds(self, test_client): + res = test_client.post(self.url(), + data=self.factory.build_dict()) + assert res.status_code == 201 + assert self.model.query.count() == OBJECT_COUNT + 1 + object_id = res.json['id'] + obj = self.model.query.get(object_id) + assert obj.workspace == self.workspace + + def test_create_fails_with_empty_dict(self, test_client): + res = test_client.post(self.url(), data={}) + assert res.status_code == 400 + + def test_create_fails_with_existing(self, session, test_client): + for unique_field in self.unique_fields: + data = self.factory.build_dict() + data[unique_field] = getattr(self.first_object, unique_field) + res = test_client.post(self.url(), data=data) + assert res.status_code == 400 + assert self.model.query.count() == OBJECT_COUNT + + def test_create_with_existing_in_other_workspace(self, test_client, + session, + second_workspace): + unique_field = self.unique_fields[0] + other_object = self.factory.create(workspace=second_workspace) + session.commit() + + data = self.factory.build_dict() + data[unique_field] = getattr(other_object, unique_field) + res = test_client.post(self.url(), data=data) + assert res.status_code == 201 + # It should create two hosts, one for each workspace + assert self.model.query.count() == OBJECT_COUNT + 2 + + +class UpdateTestsMixin: + + def test_update_a_host(self, test_client): + host = self.workspace.hosts[0] + res = test_client.put(self.url(self.first_object), + data=self.factory.build_dict()) + assert res.status_code == 200 + assert self.model.query.count() == OBJECT_COUNT + + def test_update_fails_with_existing(self, test_client, session): + for unique_field in self.unique_fields: + data = self.factory.build_dict() + data[unique_field] = getattr(self.first_object, unique_field) + res = test_client.put(self.url(self.workspace.hosts[1]), data=data) + assert res.status_code == 400 + assert self.model.query.count() == OBJECT_COUNT + + def test_update_a_host_fails_with_empty_dict(self, test_client): + """To do this the user should use a PATCH request""" + host = self.workspace.hosts[0] + res = test_client.put(self.url(host), data={}) + assert res.status_code == 400 + + +class DeleteTestsMixin: + + def test_delete(self, test_client): + res = test_client.delete(self.url(self.first_object)) + assert res.status_code == 204 # No content + assert was_deleted(self.first_object) + assert self.model.query.count() == OBJECT_COUNT - 1 + + def test_delete_from_other_workspace_fails(self, test_client, + second_workspace): + res = test_client.delete(self.url(self.first_object, + workspace=second_workspace)) + assert res.status_code == 404 # No content + assert not was_deleted(self.first_object) + assert self.model.query.count() == OBJECT_COUNT + + +class ReadWriteTestsMixin(ListTestsMixin, + RetrieveTestsMixin, + CreateTestsMixin, + UpdateTestsMixin, + DeleteTestsMixin): + pass + + +class TestHostAPIGeneric(ReadWriteTestsMixin, + GenericAPITest): + model = Host + factory = factories.HostFactory + api_endpoint = 'hosts' + unique_fields = ['ip'] + update_fields = ['ip', 'description', 'os'] + From 492d53bd860854112804caff02c9e95dc757ffa7 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 7 Sep 2017 17:12:09 -0300 Subject: [PATCH 0180/1506] Fix a bug on Service query --- server/importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index 8ebaaa2bbe2..a3e5fc2f7d4 100644 --- a/server/importer.py +++ b/server/importer.py @@ -496,7 +496,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation parents = session.query(Host).filter(Host.id.in_(parent_ids)).all() if level == 4: parent_ids = couchdb_relational_map['.'.join(document['_id'].split('.')[:3])] - parents = session.query(Service).filter(Host.id.in_(parent_ids)).all() + parents = session.query(Service).filter(Service.id.in_(parent_ids)).all() if not parents: raise Exception('Missing host or service for credential {0}'.format(document['_id'])) for parent in parents: From 8ff9a7b4b2cc50670e7d86422799cfd110bd2eef Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 8 Sep 2017 13:16:38 -0300 Subject: [PATCH 0181/1506] Add required field on get or create vulns --- server/importer.py | 70 +++++++++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/server/importer.py b/server/importer.py index a3e5fc2f7d4..7f48939aaab 100644 --- a/server/importer.py +++ b/server/importer.py @@ -347,23 +347,42 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation if not couch_parent_id: couch_parent_id = '.'.join(document['_id'].split('.')[:-1]) parent_ids = couchdb_relational_map[couch_parent_id] + mapped_severity = { + 'med': 'medium', + 'critical': 'critical', + 'high': 'high', + 'low': 'low', + 'info': 'informational', + 'unclassified': 'unclassified', + } + severity = mapped_severity[document.get('severity')] for parent_id in parent_ids: if level == 2: parent = session.query(Host).filter_by(id=parent_id).first() if level == 4: parent = session.query(Service).filter_by(id=parent_id).first() if document['type'] == 'VulnerabilityWeb': + method = document.get('method') + path = document.get('path') + pname = document.get('pname') + website = document.get('website') vulnerability, created = get_or_create( session, VulnerabilityWeb, name=document.get('name'), - description=document.get('desc'), + severity=severity, service_id=parent.id, + method=method, + parameter_name=pname, + path=path, + website=website, + workspace=workspace, ) if document['type'] == 'Vulnerability': vuln_params = { 'name': document.get('name'), - 'description': document.get('desc') + 'severity': severity, + 'workspace': workspace, } if type(parent) == Host: vuln_params.update({'host_id': parent.id}) @@ -374,20 +393,12 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation Vulnerability, **vuln_params ) - + vulnerability.description = document.get('desc'), vulnerability.confirmed = document.get('confirmed', False) or False vulnerability.data = document.get('data') vulnerability.easeofresolution = document.get('easeofresolution') vulnerability.resolution = document.get('resolution') - mapped_severity = { - 'med': 'medium', - 'critical': 'critical', - 'high':'high', - 'low': 'low', - 'info': 'informational', - 'unclassified': 'unclassified', - } - vulnerability.severity = mapped_severity[document.get('severity')] + vulnerability.owned = document.get('owned', False) #vulnerability.attachments = json.dumps(document.get('_attachments', {})) vulnerability.impact_accountability = document.get('impact', {}).get('accountability') @@ -395,13 +406,12 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation vulnerability.impact_confidentiality = document.get('impact', {}).get('confidentiality') vulnerability.impact_integrity = document.get('impact', {}).get('integrity') if document['type'] == 'VulnerabilityWeb': - vulnerability.method = document.get('method') - vulnerability.path = document.get('path') - vulnerability.pname = document.get('pname') + + vulnerability.query = document.get('query') vulnerability.request = document.get('request') vulnerability.response = document.get('response') - vulnerability.website = document.get('website') + params = document.get('params', u'') if isinstance(params, (list, tuple)): vulnerability.parameters = (u' '.join(params)).strip() @@ -413,7 +423,6 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation } status = status_map[document.get('status', 'opened')] vulnerability.status = status - vulnerability.workspace = workspace self.add_references(document, vulnerability, workspace) self.add_policy_violations(document, vulnerability, workspace) @@ -855,7 +864,28 @@ def get_objs(self, host, obj_type, level): return r.json() - def verify_import_data(self, couchdb_relational_map, workspace): + def verify_host_vulns_count_is_correct(self, couchdb_relational_map, couchdb_relational_map_by_type, workspace): + hosts = session.query(Host).filter_by(workspace=workspace) + for host in hosts: + parent_couchdb_id = None + for couchdb_id, relational_ids in couchdb_relational_map_by_type.items(): + for obj_data in relational_ids: + if obj_data['type'] == 'Host' and host.id == obj_data['id']: + parent_couchdb_id = couchdb_id + break + if parent_couchdb_id: + break + if not parent_couchdb_id: + raise Exception('Could not found couchdb id!') + vulns = get_children_from_couch(workspace, parent_couchdb_id, 'Vulnerability') + interfaces = get_children_from_couch(workspace, parent_couchdb_id, 'Interface') + for interface in interfaces: + interface = interface['value'] + vulns += get_children_from_couch(workspace, interface.get('_id'), 'Vulnerability') + assert len(vulns) == len(host.vulnerabilities) + + def verify_import_data(self, couchdb_relational_map, couchdb_relational_map_by_type, workspace): + self.verify_host_vulns_count_is_correct(couchdb_relational_map, couchdb_relational_map_by_type, workspace) all_docs_url = "http://{username}:{password}@{hostname}:{port}/{workspace_name}/_all_docs?include_docs=true".format( username=server.config.couchdb.user, password=server.config.couchdb.password, @@ -906,6 +936,7 @@ def import_workspace_into_database(self, workspace_name): # for the desired obj. obj_types = OBJ_TYPES couchdb_relational_map = defaultdict(list) + couchdb_relational_map_by_type = defaultdict(list) for level, obj_type in obj_types: obj_importer = faraday_importer.get_importer_from_document(obj_type)() objs_dict = self.get_objs(couch_url, obj_type, level) @@ -919,6 +950,7 @@ def import_workspace_into_database(self, workspace_name): if not new_obj: continue session.commit() + couchdb_relational_map_by_type[couchdb_id].append({'type': obj_type, 'id': new_obj.id}) couchdb_relational_map[couchdb_id].append(new_obj.id) - self.verify_import_data(couchdb_relational_map, workspace) + self.verify_import_data(couchdb_relational_map, couchdb_relational_map_by_type, workspace) return created From 9d21f19fac4e2ea3d575833b468bf5359e27412c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 8 Sep 2017 15:08:56 -0300 Subject: [PATCH 0182/1506] Make Service.status field not nullable It must be set and have one of the three allowed values --- server/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/models.py b/server/models.py index 7bdc2a4c9c2..bfaf785cefd 100644 --- a/server/models.py +++ b/server/models.py @@ -137,7 +137,7 @@ class Service(db.Model): owned = Column(Boolean, nullable=False, default=False) protocol = Column(Text, nullable=False) - status = Column(Enum(*STATUSES, name='service_statuses'), nullable=True) + status = Column(Enum(*STATUSES, name='service_statuses'), nullable=False) version = Column(Text, nullable=True) banner = Column(Text, nullable=True) @@ -791,4 +791,4 @@ class ExecutiveReport(db.Model): VulnerabilityGeneric.workspace_id, VulnerabilityCode.source_code_id, name='uix_vulnerability' -) \ No newline at end of file +) From f1dee4a376365e7caebfd69ea152823424091a2f Mon Sep 17 00:00:00 2001 From: micabot Date: Fri, 8 Sep 2017 15:41:10 -0300 Subject: [PATCH 0183/1506] Fix importer to avoid Design Docs The importer was only ignoring some particular design docs. --- server/importer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/importer.py b/server/importer.py index 7f48939aaab..a0ba813f5f3 100644 --- a/server/importer.py +++ b/server/importer.py @@ -2,6 +2,7 @@ # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information import os +import re import sys import json import datetime @@ -897,8 +898,7 @@ def verify_import_data(self, couchdb_relational_map, couchdb_relational_map_by_t if len(all_ids) != len(couchdb_relational_map.keys()): missing_objs_filename = os.path.join(os.path.expanduser('~/.faraday'), 'logs', 'import_missing_objects_{0}.json'.format(workspace.name)) missing_ids = set(all_ids) - set(couchdb_relational_map.keys()) - missing_ids = missing_ids - set([u'_design/commands', u'_design/hosts', u'_design/comms', u'_design/tags', u'_design/vulns', u'_design/utils', u'_design/importer', u'_design/auth', u'_design/mapper', u'_design/services', u'_design/interfaces', u'_design/changes', u'_design/reports', -]) + missing_ids = set([x for x in missing_ids if not re.match(r'^\_design', x)]) objs_diff = [] if missing_ids: logger.info('Downloading missing couchdb docs') From 58e6828b4242e21a47f8d1495764813ad06567b6 Mon Sep 17 00:00:00 2001 From: micabot Date: Fri, 8 Sep 2017 15:42:18 -0300 Subject: [PATCH 0184/1506] Fix typo --- server/importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index a0ba813f5f3..17a916c34fe 100644 --- a/server/importer.py +++ b/server/importer.py @@ -771,7 +771,7 @@ def run(self): mapped_exploitation = { 'critical': 'critical', 'med': 'medium', - 'high':'high', + 'high': 'high', 'low': 'low', 'info': 'informational', 'unclassified': 'unclassified', From 4bd0871c3549c1d26b415ec6ce2d74c55e9647b1 Mon Sep 17 00:00:00 2001 From: micabot Date: Fri, 8 Sep 2017 15:43:23 -0300 Subject: [PATCH 0185/1506] Fix Licenses importer If the Licenses database was not present in CouchDB, the script failed. Fixed it to check the existence of this DB before trying to import it. --- server/importer.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index 17a916c34fe..c2a21dd0e5e 100644 --- a/server/importer.py +++ b/server/importer.py @@ -799,7 +799,17 @@ def run(self): port=server.config.couchdb.port, path='faraday_licenses/_all_docs?include_docs=true' ) - licenses = requests.get(cwe_url).json()['rows'] + + if not requests.head(cwe_url).ok: + logger.info('No Licenses database found, nothing to see here, move along!') + return + + try: + licenses = requests.get(cwe_url).json()['rows'] + except requests.exceptions.RequestException as e: + logger.warn(e) + return + for license in licenses: document = license['doc'] From fbba0e00fb40bfa6df06dec33c27855e65dae057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 8 Sep 2017 15:47:34 -0300 Subject: [PATCH 0186/1506] Add nested services endpoint //hosts//services/ Also improve service and host factories and schemas --- server/api/modules/hosts.py | 17 ++++++++++++++++- test_cases/factories.py | 7 +++++-- test_cases/test_server_api.py | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 70cc144cdc8..d16bf98229d 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -4,6 +4,7 @@ import flask from flask import Blueprint +from flask_classful import route from marshmallow import Schema, fields from server.utils.logger import get_logger @@ -17,18 +18,32 @@ class HostSchema(Schema): - id = fields.String(required=True, dump_only=True) + id = fields.Integer(required=True, dump_only=True) ip = fields.String(required=True) description = fields.String(required=True) os = fields.String() +class ServiceSchema(Schema): + id = fields.Integer(required=True, dump_only=True) + name = fields.String(required=True) + description = fields.String(required=False) + port = fields.Integer(required=True) + protocol = fields.String(required=True) + status = fields.String(required=True) + + class HostsView(ReadWriteWorkspacedView): route_base = 'hosts' model_class = Host schema_class = HostSchema unique_fields = ['ip'] + @route('//services/') + def service_list(self, workspace_name, host_id): + services = self._get_object(workspace_name, host_id).services + return ServiceSchema(many=True).dump(services) + HostsView.register(host_api) diff --git a/test_cases/factories.py b/test_cases/factories.py index 6001cc90b28..85c94b8ece9 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -1,7 +1,8 @@ import factory from factory.fuzzy import ( + FuzzyChoice, + FuzzyInteger, FuzzyText, - FuzzyChoice ) from server.models import ( db, @@ -61,8 +62,10 @@ class Meta: class ServiceFactory(WorkspaceObjectFactory): name = FuzzyText() description = FuzzyText() - ports = FuzzyChoice(['443', '80', '22']) + port = FuzzyInteger(1, 65535) + protocol = FuzzyChoice(['tcp', 'udp']) host = factory.SubFactory(HostFactory) + status = FuzzyChoice(Service.STATUSES) class Meta: model = Service diff --git a/test_cases/test_server_api.py b/test_cases/test_server_api.py index 16599ee0374..19c80af5ed4 100644 --- a/test_cases/test_server_api.py +++ b/test_cases/test_server_api.py @@ -11,7 +11,9 @@ class TestHostAPI: @pytest.fixture(autouse=True) def load_workspace_with_hosts(self, database, session, workspace, host_factory): - host_factory.create_batch(HOSTS_COUNT, workspace=workspace) + self.hosts = host_factory.create_batch(HOSTS_COUNT, + workspace=workspace) + self.first_host = self.hosts[0] session.commit() assert workspace.id is not None assert workspace.hosts[0].id is not None @@ -25,6 +27,9 @@ def url(self, host=None, workspace=None): url += str(host.id) return url + def services_url(self, host, workspace=None): + return self.url(host, workspace) + '/services/' + def test_list_retrieves_all_items_from_workspace(self, test_client, second_workspace, session, @@ -143,6 +148,32 @@ def test_delete_host_from_other_workspace_fails(self, test_client, assert res.status_code == 404 # No content assert not was_deleted(host) + def test_get_host_services(self, test_client, session, + second_workspace, + service_factory): + SERVICE_COUNT = 10 + + # Create the services that must be shown + real = service_factory.create_batch( + SERVICE_COUNT, + host=self.first_host, + workspace=self.first_host.workspace) + + # Create a service of other host, must not be shown + other_host = service_factory.create( + host=self.hosts[1], + workspace=self.hosts[1].workspace) + + session.commit() + ids_expected = {host.id for host in real} + + res = test_client.get(self.services_url(self.first_host)) + assert res.status_code == 200 + ids_returned = {host['id'] for host in res.json} + assert other_host.id not in ids_returned + assert ids_expected == ids_returned + + PREFIX = '/v2/' OBJECT_COUNT = 5 From d965796b7f31bcd1d3b9517003924aa2464b368e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 8 Sep 2017 18:32:42 -0300 Subject: [PATCH 0187/1506] Add metadata model --- server/models.py | 63 +++++++++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/server/models.py b/server/models.py index c2694f6b454..67ffa01491f 100644 --- a/server/models.py +++ b/server/models.py @@ -1,6 +1,9 @@ # Faraday Penetration Test IDE # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information +from datetime import datetime + +import pytz from sqlalchemy import ( Boolean, CheckConstraint, @@ -36,6 +39,22 @@ class DatabaseMetadata(db.Model): value = Column(String(250), nullable=False) +class Metadata(db.Model): + + __abstract__ = True + + @declared_attr + def creator_id(cls): + return Column(Integer, ForeignKey('user.id'), nullable=True) + + @declared_attr + def creator(cls): + return relationship('User', foreign_keys=[cls.creator_id]) + + create_date = Column(DateTime, default=datetime.utcnow) + update_date = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + + class EntityMetadata(db.Model): __tablename__ = 'metadata' __table_args__ = ( @@ -57,7 +76,7 @@ class EntityMetadata(db.Model): document_type = Column(String(250)) -class SourceCode(db.Model): +class SourceCode(Metadata): __tablename__ = 'source_code' id = Column(Integer, primary_key=True) filename = Column(Text, nullable=False) @@ -70,7 +89,7 @@ class SourceCode(db.Model): ) -class Host(db.Model): +class Host(Metadata): __tablename__ = 'host' id = Column(Integer, primary_key=True) ip = Column(Text, nullable=False) # IP v4 or v6 @@ -105,7 +124,7 @@ class Host(db.Model): ) -class Hostname(db.Model): +class Hostname(Metadata): __tablename__ = 'hostname' id = Column(Integer, primary_key=True) name = Column(Text, nullable=False) @@ -123,7 +142,7 @@ class Hostname(db.Model): ) -class Service(db.Model): +class Service(Metadata): STATUSES = [ 'open', 'closed', @@ -165,7 +184,7 @@ class Service(db.Model): ) -class VulnerabilityABC(db.Model): +class VulnerabilityABC(Metadata): # revisar plugin nexpose, netspark para terminar de definir uniques. asegurar que se carguen bien EASE_OF_RESOLUTIONS = [ 'trivial', @@ -306,7 +325,7 @@ class VulnerabilityCode(VulnerabilityGeneric): } -class ReferenceTemplate(db.Model): +class ReferenceTemplate(Metadata): __tablename__ = 'reference_template' id = Column(Integer, primary_key=True) name = Column(Text, nullable=False) @@ -327,7 +346,7 @@ class ReferenceTemplate(db.Model): ) -class Reference(db.Model): +class Reference(Metadata): __tablename__ = 'reference' id = Column(Integer, primary_key=True) name = Column(Text, nullable=False) @@ -360,7 +379,7 @@ class Reference(db.Model): ) -class PolicyViolationTemplate(db.Model): +class PolicyViolationTemplate(Metadata): __tablename__ = 'policy_violation_template' id = Column(Integer, primary_key=True) name = Column(Text, nullable=False) @@ -384,7 +403,7 @@ class PolicyViolationTemplate(db.Model): ) -class PolicyViolation(db.Model): +class PolicyViolation(Metadata): __tablename__ = 'policy_violation' id = Column(Integer, primary_key=True) name = Column(Text, nullable=False) @@ -421,7 +440,7 @@ class PolicyViolation(db.Model): ) -class Credential(db.Model): +class Credential(Metadata): __tablename__ = 'credential' id = Column(Integer, primary_key=True) username = Column(String(250), nullable=False) @@ -469,7 +488,7 @@ class Credential(db.Model): ) -class Command(db.Model): +class Command(Metadata): __tablename__ = 'command' id = Column(Integer, primary_key=True) command = Column(Text(), nullable=False) @@ -498,7 +517,7 @@ class Command(db.Model): ) -class Workspace(db.Model): +class Workspace(Metadata): __tablename__ = 'workspace' id = Column(Integer, primary_key=True) # TODO: change nullable=True for appropriate fields @@ -527,14 +546,14 @@ class RolesUsers(db.Model): role_id = Column('role_id', Integer(), ForeignKey('role.id')) -class Role(db.Model, RoleMixin): +class Role(Metadata, RoleMixin): __tablename__ = 'role' id = Column(Integer(), primary_key=True) name = Column(String(80), unique=True) description = Column(String(255), nullable=True) -class User(db.Model, UserMixin): +class User(Metadata, UserMixin): __tablename__ = 'user' id = Column(Integer, primary_key=True) username = Column(String(255), unique=True, nullable=False) @@ -576,14 +595,14 @@ def __repr__(self): self.username) -class MethodologyTemplate(db.Model): +class MethodologyTemplate(Metadata): # TODO: reset template_id in methodologies when deleting meth template __tablename__ = 'methodology_template' id = Column(Integer, primary_key=True) name = Column(Text, nullable=False) -class Methodology(db.Model): +class Methodology(Metadata): # TODO: add unique constraint -> name, workspace __tablename__ = 'methodology' id = Column(Integer, primary_key=True) @@ -614,7 +633,7 @@ class Methodology(db.Model): workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) -class TaskABC(db.Model): +class TaskABC(Metadata): __abstract__ = True id = Column(Integer, primary_key=True) @@ -664,8 +683,8 @@ class Task(TaskABC): entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) - assigned_to = relationship('User', backref='assigned_tasks') assigned_to_id = Column(Integer, ForeignKey('user.id'), nullable=True) + assigned_to = relationship('User', backref='assigned_tasks', foreign_keys=[assigned_to_id]) methodology_id = Column( Integer, @@ -691,7 +710,7 @@ class Task(TaskABC): # ) -class License(db.Model): +class License(Metadata): __tablename__ = 'license' id = Column(Integer, primary_key=True) product = Column(Text, nullable=False) @@ -706,7 +725,7 @@ class License(db.Model): ) -class Tag(db.Model): +class Tag(Metadata): __tablename__ = 'tag' id = Column(Integer, primary_key=True) name = Column(Text, nullable=False, unique=True) @@ -735,7 +754,7 @@ class CommentObject(db.Model): comment_id = Column(Integer, ForeignKey('comment.id'), index=True) -class Comment(db.Model): +class Comment(Metadata): __tablename__ = 'comment' id = Column(Integer, primary_key=True) @@ -752,7 +771,7 @@ class Comment(db.Model): workspace = relationship('Workspace', foreign_keys=[workspace_id]) -class ExecutiveReport(db.Model): +class ExecutiveReport(Metadata): STATUSES = [ 'created', 'error', From b2499d611e75a5cc760ff42eade9856f9c864849 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 8 Sep 2017 19:00:03 -0300 Subject: [PATCH 0188/1506] Add nullable to creator_id --- server/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index 67ffa01491f..42bc4edfa64 100644 --- a/server/models.py +++ b/server/models.py @@ -45,7 +45,7 @@ class Metadata(db.Model): @declared_attr def creator_id(cls): - return Column(Integer, ForeignKey('user.id'), nullable=True) + return Column(Integer, ForeignKey('user.id'), nullable=False) @declared_attr def creator(cls): From d269adc87d7a698d72a04ed46482827580eeda48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 8 Sep 2017 19:59:24 -0300 Subject: [PATCH 0189/1506] Add service_count field to Host Optimized version via undeferring an originally deferred column_property --- server/api/modules/hosts.py | 8 ++++++++ server/models.py | 9 +++++++++ test_cases/test_server_api.py | 34 ++++++++++++++++++++++++++++++---- 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index d16bf98229d..880e027d8ec 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -6,6 +6,7 @@ from flask import Blueprint from flask_classful import route from marshmallow import Schema, fields +from sqlalchemy.orm import undefer from server.utils.logger import get_logger from server.utils.web import gzipped, validate_workspace,\ @@ -22,6 +23,7 @@ class HostSchema(Schema): ip = fields.String(required=True) description = fields.String(required=True) os = fields.String() + service_count = fields.Integer() class ServiceSchema(Schema): @@ -44,6 +46,12 @@ def service_list(self, workspace_name, host_id): services = self._get_object(workspace_name, host_id).services return ServiceSchema(many=True).dump(services) + def _get_base_query(self, workspace_name): + """Get services_count in one query and not deferred, that doe + one query per host""" + original = super(HostsView, self)._get_base_query(workspace_name) + return original.options(undefer(Host.service_count)) + HostsView.register(host_api) diff --git a/server/models.py b/server/models.py index 5772835a083..42f31bbdf44 100644 --- a/server/models.py +++ b/server/models.py @@ -15,6 +15,9 @@ event ) from sqlalchemy.orm import relationship, backref +from sqlalchemy.sql import select +from sqlalchemy import func +from sqlalchemy.orm import column_property from sqlalchemy.ext.declarative import declared_attr from flask_sqlalchemy import ( SQLAlchemy as OriginalSQLAlchemy, @@ -195,6 +198,12 @@ class Service(db.Model): foreign_keys=[workspace_id] ) +# TODO: Move this to Host definition. I need a way to reference Service before +# it is declared +Host.service_count = column_property((select([func.count(Service.id)]). + where(Service.host_id == Host.id)#.label('service_count') + ), deferred=True) + class VulnerabilityABC(db.Model): # TODO: add unique constraint to -> name, description, severity, parent, method, pname, path, website, workspace diff --git a/test_cases/test_server_api.py b/test_cases/test_server_api.py index 19c80af5ed4..be8ab9b1a19 100644 --- a/test_cases/test_server_api.py +++ b/test_cases/test_server_api.py @@ -5,6 +5,7 @@ PREFIX = '/v2/' HOSTS_COUNT = 5 +SERVICE_COUNT = [10, 5] # 10 services to the first host, 5 to the second @pytest.mark.usefixtures('database', 'logged_user') class TestHostAPI: @@ -20,6 +21,20 @@ def load_workspace_with_hosts(self, database, session, workspace, host_factory): self.workspace = workspace return workspace + @pytest.fixture + def host_services(self, session, service_factory): + """ + Add some services to the first len(SERVICE_COUNT) hosts. + + Return a dictionary mapping hosts to a list of services + """ + ret = {} + for (count, host) in zip(SERVICE_COUNT, self.hosts): + ret[host] = service_factory.create_batch( + count, host=host, workspace=host.workspace) + session.commit() + return ret + def url(self, host=None, workspace=None): workspace = workspace or self.workspace url = PREFIX + workspace.name + '/hosts/' @@ -149,13 +164,10 @@ def test_delete_host_from_other_workspace_fails(self, test_client, assert not was_deleted(host) def test_get_host_services(self, test_client, session, - second_workspace, service_factory): - SERVICE_COUNT = 10 - # Create the services that must be shown real = service_factory.create_batch( - SERVICE_COUNT, + SERVICE_COUNT[0], host=self.first_host, workspace=self.first_host.workspace) @@ -173,6 +185,20 @@ def test_get_host_services(self, test_client, session, assert other_host.id not in ids_returned assert ids_expected == ids_returned + def test_retrieve_shows_service_count(self, test_client, host_services): + for (host, services) in host_services.items(): + res = test_client.get(self.url(host)) + assert res.json['service_count'] == len(services) + + def test_index_shows_service_count(self, test_client, host_services): + ids_map = {host.id: services + for (host, services) in host_services.items()} + res = test_client.get(self.url()) + assert len(res.json) >= len(ids_map) # Some hosts can have no services + for host in res.json: + if host['id'] in ids_map: + assert host['service_count'] == len(ids_map[host['id']]) + PREFIX = '/v2/' From 0717ba12951044e6dd88ecbf7a579fed44ff93d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 11 Sep 2017 15:22:15 -0300 Subject: [PATCH 0190/1506] Require ids to be numeric by default Otherwise, postgresql query will fail when comparing numeric fields with strings. This behavior doesn't happen in SQLite --- server/api/base.py | 8 ++++++++ test_cases/test_server_api.py | 12 +++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/server/api/base.py b/server/api/base.py index 5020b03b530..eb16fd7abfb 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -35,6 +35,7 @@ class GenericWorkspacedView(FlaskView): base_args = ['workspace_name'] # Required to prevent double usage of representations = {'application/json': output_json} lookup_field = 'id' + lookup_field_type = int unique_fields = [] # Fields unique together with workspace_id @classmethod @@ -72,6 +73,7 @@ def _get_base_query(self, workspace_name): .filter(Workspace.id==self._get_workspace(workspace_name).id) def _get_object(self, workspace_name, object_id): + self._validate_object_id(object_id) try: obj = self._get_base_query(workspace_name).filter( self._get_lookup_field() == object_id).one() @@ -86,6 +88,12 @@ def _parse_data(self, schema, request, *args, **kwargs): return FlaskParser().parse(schema, request, locations=('json',), *args, **kwargs) + def _validate_object_id(self, object_id): + try: + self.lookup_field_type(object_id) + except ValueError: + flask.abort(404, 'Invalid format of lookup field') + def _validate_uniqueness(self, obj, object_id=None): assert obj.workspace is not None, "Object must have a " \ "workspace attribute set to call _validate_uniqueness" diff --git a/test_cases/test_server_api.py b/test_cases/test_server_api.py index be8ab9b1a19..bedf2cab962 100644 --- a/test_cases/test_server_api.py +++ b/test_cases/test_server_api.py @@ -1,3 +1,4 @@ +#-*- coding: utf8 -*- import pytest from sqlalchemy.orm.util import was_deleted from test_cases import factories @@ -236,7 +237,9 @@ def url(self, obj=None, workspace=None): workspace = workspace or self.workspace url = PREFIX + workspace.name + '/' + self.api_endpoint + '/' if obj is not None: - url += str(obj.id) + id_ = unicode(obj.id) if isinstance( + obj, self.model) else unicode(obj) + url += id_ return url @@ -266,6 +269,13 @@ def test_retrieve_fails_object_of_other_workspcae(self, res = test_client.get(self.url(self.first_object, second_workspace)) assert res.status_code == 404 + @pytest.mark.parametrize('object_id', [123, -1, 'xxx', u'áá']) + def test_404_when_retrieving_unexistent_object(self, test_client, + object_id): + url = self.url(object_id) + res = test_client.get(url) + assert res.status_code == 404 + class CreateTestsMixin: From 1ae42310451c82e6e61edb14e46db017ae72aba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 11 Sep 2017 18:00:52 -0300 Subject: [PATCH 0191/1506] Upgrade to flask classful v0.14.0-flask-c - Install dependency from github instead of from PyPI (it wasn't published yet) - Add trailing slashes to endpoints in test cases(otherwise they would fail because of 301 redirect status codes) - Remove fix for https://github.com/teracyhq/flask-classful/issues/50 since it was fixed on the package --- requirements_server.txt | 2 +- server/api/base.py | 16 ---------------- test_cases/test_server_api.py | 6 +++--- 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/requirements_server.txt b/requirements_server.txt index 7c9bacffd8a..371a6f8d32d 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -12,6 +12,6 @@ Flask-Security==3.0.0 bcrypt Flask-SQLAlchemy==2.2 Flask-Script==2.0.5 -flask-classful +https://github.com/teracyhq/flask-classful/archive/develop.zip marshmallow webargs diff --git a/server/api/base.py b/server/api/base.py index eb16fd7abfb..a4a7c4e74af 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -38,22 +38,6 @@ class GenericWorkspacedView(FlaskView): lookup_field_type = int unique_fields = [] # Fields unique together with workspace_id - @classmethod - def get_route_base(cls): - """Fix issue with base_args overriding - - See https://github.com/teracyhq/flask-classful/issues/50 for - more information""" - - if cls.route_base is not None: - route_base = cls.route_base - base_rule = parse_rule(route_base) - cls.base_args += [r[2] for r in base_rule] - else: - route_base = cls.default_route_base() - - return route_base.strip("/") - def _get_schema_class(self): assert self.schema_class is not None, "You must define schema_class" return self.schema_class diff --git a/test_cases/test_server_api.py b/test_cases/test_server_api.py index bedf2cab962..5491676931f 100644 --- a/test_cases/test_server_api.py +++ b/test_cases/test_server_api.py @@ -40,11 +40,11 @@ def url(self, host=None, workspace=None): workspace = workspace or self.workspace url = PREFIX + workspace.name + '/hosts/' if host is not None: - url += str(host.id) + url += str(host.id) + '/' return url def services_url(self, host, workspace=None): - return self.url(host, workspace) + '/services/' + return self.url(host, workspace) + 'services/' def test_list_retrieves_all_items_from_workspace(self, test_client, second_workspace, @@ -239,7 +239,7 @@ def url(self, obj=None, workspace=None): if obj is not None: id_ = unicode(obj.id) if isinstance( obj, self.model) else unicode(obj) - url += id_ + url += id_ + u'/' return url From ce2a62fc57a30e575abf37bbd755105c46560b38 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 12 Sep 2017 11:53:11 -0300 Subject: [PATCH 0192/1506] Fix a bug in the model. --- server/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index 42bc4edfa64..232e4c1057e 100644 --- a/server/models.py +++ b/server/models.py @@ -553,7 +553,7 @@ class Role(Metadata, RoleMixin): description = Column(String(255), nullable=True) -class User(Metadata, UserMixin): +class User(db.Model, UserMixin): __tablename__ = 'user' id = Column(Integer, primary_key=True) username = Column(String(255), unique=True, nullable=False) From 99b70dcf12c85e5d976f6688bd4cf2247e556696 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 12 Sep 2017 11:53:27 -0300 Subject: [PATCH 0193/1506] Update factories to the new model --- test_cases/factories.py | 46 ++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/test_cases/factories.py b/test_cases/factories.py index 195ba120cc4..7513492f062 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -6,10 +6,10 @@ from pytest_factoryboy import register from server.models import ( db, + User, Host, Command, Service, - Interface, Workspace, Credential, Vulnerability, @@ -22,9 +22,18 @@ class FaradayFactory(factory.alchemy.SQLAlchemyModelFactory): id = factory.Sequence(lambda n: n) +class UserFactory(FaradayFactory): + + username = FuzzyText() + + class Meta: + model = User + sqlalchemy_session = db.session + class WorkspaceFactory(FaradayFactory): name = FuzzyText() + creator = factory.SubFactory(UserFactory) class Meta: model = Workspace @@ -32,9 +41,11 @@ class Meta: class HostFactory(FaradayFactory): - name = FuzzyText() + ip = FuzzyText() description = FuzzyText() os = FuzzyChoice(['Linux', 'Windows', 'OSX', 'Android', 'iOS']) + workspace = factory.SubFactory(WorkspaceFactory) + creator = factory.SubFactory(UserFactory) class Meta: model = Host @@ -49,23 +60,14 @@ class Meta: sqlalchemy_session = db.session -class InterfaceFactory(FaradayFactory): - name = FuzzyText() - description = FuzzyText() - mac = FuzzyText() - host = factory.SubFactory(HostFactory) - - class Meta: - model = Interface - sqlalchemy_session = db.session - - class ServiceFactory(FaradayFactory): name = FuzzyText() description = FuzzyText() - ports = FuzzyChoice(['443', '80', '22']) - interface = factory.SubFactory(InterfaceFactory) + port = FuzzyChoice(['443', '80', '22']) + protocol = FuzzyChoice(['TCP', 'UDP']) host = factory.SubFactory(HostFactory) + workspace = factory.SubFactory(WorkspaceFactory) + creator = factory.SubFactory(UserFactory) class Meta: model = Service @@ -76,14 +78,11 @@ class VulnerabilityFactory(FaradayFactory): name = FuzzyText() description = FuzzyText() - host = factory.SubFactory(HostFactory) - entity_metadata = factory.SubFactory(EntityMetadataFactory) - service = factory.SubFactory(ServiceFactory) + # host = factory.SubFactory(HostFactory) + # service = factory.SubFactory(ServiceFactory) workspace = factory.SubFactory(WorkspaceFactory) - vuln_type = FuzzyChoice(['Vulnerability', 'VulnerabilityWeb']) - attachments = '[]' - policyviolations = '[]' - refs = '[]' + creator = factory.SubFactory(UserFactory) + severity = FuzzyChoice(['critical', 'high']) class Meta: model = Vulnerability @@ -106,10 +105,9 @@ class Meta: model = Command sqlalchemy_session = db.session - +register(UserFactory) register(WorkspaceFactory) register(HostFactory) register(ServiceFactory) -register(InterfaceFactory) register(VulnerabilityFactory) register(CredentialFactory) From 08ac9bbc828c750851c6894371745baed1d587a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 12 Sep 2017 15:00:29 -0300 Subject: [PATCH 0194/1506] Move SQLALCHEMY_ECHO commented code It is useful to have printed queries when running the server. Now it was only enabled when running the server (and with the line uncommented) --- server/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/app.py b/server/app.py index d243f2798c4..2f4778817fc 100644 --- a/server/app.py +++ b/server/app.py @@ -26,8 +26,8 @@ def create_app(db_connection_string=None, testing=None): app.config['SECURITY_POST_LOGIN_VIEW'] = '/_api/session' app.config['SECURITY_POST_LOGOUT_VIEW'] = '/_api/login' app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False + # app.config['SQLALCHEMY_ECHO'] = True if testing: - # app.config['SQLALCHEMY_ECHO'] = True app.config['TESTING'] = testing try: app.config['SQLALCHEMY_DATABASE_URI'] = db_connection_string or server.config.database.connection_string.strip("'") From 78d33ab4a93de74845f9cbf812e9f53ca71588ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 12 Sep 2017 15:18:07 -0300 Subject: [PATCH 0195/1506] Split base api tests and host api tests Make generic tests available in test_api_workspaced_base --- .../{test_server_api.py => test_api_hosts.py} | 174 +----------------- test_cases/test_api_workspaced_base.py | 170 +++++++++++++++++ 2 files changed, 175 insertions(+), 169 deletions(-) rename test_cases/{test_server_api.py => test_api_hosts.py} (55%) create mode 100644 test_cases/test_api_workspaced_base.py diff --git a/test_cases/test_server_api.py b/test_cases/test_api_hosts.py similarity index 55% rename from test_cases/test_server_api.py rename to test_cases/test_api_hosts.py index 5491676931f..c94a1cf79a0 100644 --- a/test_cases/test_server_api.py +++ b/test_cases/test_api_hosts.py @@ -1,10 +1,10 @@ -#-*- coding: utf8 -*- import pytest from sqlalchemy.orm.util import was_deleted + from test_cases import factories -from server.models import db, Workspace, Host +from test_api_workspaced_base import API_PREFIX, ReadWriteAPITests +from server.models import db, Host -PREFIX = '/v2/' HOSTS_COUNT = 5 SERVICE_COUNT = [10, 5] # 10 services to the first host, 5 to the second @@ -38,7 +38,7 @@ def host_services(self, session, service_factory): def url(self, host=None, workspace=None): workspace = workspace or self.workspace - url = PREFIX + workspace.name + '/hosts/' + url = API_PREFIX + workspace.name + '/hosts/' if host is not None: url += str(host.id) + '/' return url @@ -201,171 +201,7 @@ def test_index_shows_service_count(self, test_client, host_services): assert host['service_count'] == len(ids_map[host['id']]) - -PREFIX = '/v2/' -OBJECT_COUNT = 5 - - -@pytest.mark.usefixtures('logged_user') -class GenericAPITest: - - model = None - factory = None - api_endpoint = None - pk_field = 'id' - unique_fields = [] - update_fields = [] - - @pytest.fixture(autouse=True) - def load_workspace_with_objects(self, database, session, workspace): - objects = self.factory.create_batch(OBJECT_COUNT, workspace=workspace) - self.first_object = objects[0] - session.commit() - assert workspace.id is not None - self.workspace = workspace - return workspace - - @pytest.fixture - def object_instance(self, session, workspace): - """An object instance with the correct workspace assigned, - saved in the database""" - obj = self.factory.create(workspace=workspace) - session.commit() - return obj - - def url(self, obj=None, workspace=None): - workspace = workspace or self.workspace - url = PREFIX + workspace.name + '/' + self.api_endpoint + '/' - if obj is not None: - id_ = unicode(obj.id) if isinstance( - obj, self.model) else unicode(obj) - url += id_ + u'/' - return url - - -class ListTestsMixin: - - def test_list_retrieves_all_items_from_workspace(self, test_client, - second_workspace, - session): - self.factory.create(workspace=second_workspace) - session.commit() - res = test_client.get(self.url()) - assert res.status_code == 200 - assert len(res.json) == OBJECT_COUNT - - -class RetrieveTestsMixin: - - def test_retrieve_one_object(self, test_client): - res = test_client.get(self.url(self.first_object)) - assert res.status_code == 200 - assert isinstance(res.json, dict) - - def test_retrieve_fails_object_of_other_workspcae(self, - test_client, - session, - second_workspace): - res = test_client.get(self.url(self.first_object, second_workspace)) - assert res.status_code == 404 - - @pytest.mark.parametrize('object_id', [123, -1, 'xxx', u'áá']) - def test_404_when_retrieving_unexistent_object(self, test_client, - object_id): - url = self.url(object_id) - res = test_client.get(url) - assert res.status_code == 404 - - -class CreateTestsMixin: - - def test_create_succeeds(self, test_client): - res = test_client.post(self.url(), - data=self.factory.build_dict()) - assert res.status_code == 201 - assert self.model.query.count() == OBJECT_COUNT + 1 - object_id = res.json['id'] - obj = self.model.query.get(object_id) - assert obj.workspace == self.workspace - - def test_create_fails_with_empty_dict(self, test_client): - res = test_client.post(self.url(), data={}) - assert res.status_code == 400 - - def test_create_fails_with_existing(self, session, test_client): - for unique_field in self.unique_fields: - data = self.factory.build_dict() - data[unique_field] = getattr(self.first_object, unique_field) - res = test_client.post(self.url(), data=data) - assert res.status_code == 400 - assert self.model.query.count() == OBJECT_COUNT - - def test_create_with_existing_in_other_workspace(self, test_client, - session, - second_workspace): - unique_field = self.unique_fields[0] - other_object = self.factory.create(workspace=second_workspace) - session.commit() - - data = self.factory.build_dict() - data[unique_field] = getattr(other_object, unique_field) - res = test_client.post(self.url(), data=data) - assert res.status_code == 201 - # It should create two hosts, one for each workspace - assert self.model.query.count() == OBJECT_COUNT + 2 - - -class UpdateTestsMixin: - - def test_update_a_host(self, test_client): - host = self.workspace.hosts[0] - res = test_client.put(self.url(self.first_object), - data=self.factory.build_dict()) - assert res.status_code == 200 - assert self.model.query.count() == OBJECT_COUNT - - def test_update_fails_with_existing(self, test_client, session): - for unique_field in self.unique_fields: - data = self.factory.build_dict() - data[unique_field] = getattr(self.first_object, unique_field) - res = test_client.put(self.url(self.workspace.hosts[1]), data=data) - assert res.status_code == 400 - assert self.model.query.count() == OBJECT_COUNT - - def test_update_a_host_fails_with_empty_dict(self, test_client): - """To do this the user should use a PATCH request""" - host = self.workspace.hosts[0] - res = test_client.put(self.url(host), data={}) - assert res.status_code == 400 - - -class DeleteTestsMixin: - - def test_delete(self, test_client): - res = test_client.delete(self.url(self.first_object)) - assert res.status_code == 204 # No content - assert was_deleted(self.first_object) - assert self.model.query.count() == OBJECT_COUNT - 1 - - def test_delete_from_other_workspace_fails(self, test_client, - second_workspace): - res = test_client.delete(self.url(self.first_object, - workspace=second_workspace)) - assert res.status_code == 404 # No content - assert not was_deleted(self.first_object) - assert self.model.query.count() == OBJECT_COUNT - - -class ReadWriteTestsMixin(ListTestsMixin, - RetrieveTestsMixin, - CreateTestsMixin, - UpdateTestsMixin, - DeleteTestsMixin): - pass - - -class TestHostAPIGeneric(ReadWriteTestsMixin, - GenericAPITest): +class TestHostAPIGeneric(ReadWriteAPITests): model = Host factory = factories.HostFactory api_endpoint = 'hosts' diff --git a/test_cases/test_api_workspaced_base.py b/test_cases/test_api_workspaced_base.py new file mode 100644 index 00000000000..3d9b9d3d583 --- /dev/null +++ b/test_cases/test_api_workspaced_base.py @@ -0,0 +1,170 @@ +#-*- coding: utf8 -*- +import pytest +from sqlalchemy.orm.util import was_deleted +from server.models import db, Workspace + +API_PREFIX = '/v2/' +OBJECT_COUNT = 5 + + +@pytest.mark.usefixtures('logged_user') +class GenericAPITest: + + model = None + factory = None + api_endpoint = None + pk_field = 'id' + unique_fields = [] + update_fields = [] + + @pytest.fixture(autouse=True) + def load_workspace_with_objects(self, database, session, workspace): + objects = self.factory.create_batch(OBJECT_COUNT, workspace=workspace) + self.first_object = objects[0] + session.commit() + assert workspace.id is not None + self.workspace = workspace + return workspace + + @pytest.fixture + def object_instance(self, session, workspace): + """An object instance with the correct workspace assigned, + saved in the database""" + obj = self.factory.create(workspace=workspace) + session.commit() + return obj + + def url(self, obj=None, workspace=None): + workspace = workspace or self.workspace + url = API_PREFIX + workspace.name + '/' + self.api_endpoint + '/' + if obj is not None: + id_ = unicode(obj.id) if isinstance( + obj, self.model) else unicode(obj) + url += id_ + u'/' + return url + + +class ListTestsMixin: + + def test_list_retrieves_all_items_from_workspace(self, test_client, + second_workspace, + session): + self.factory.create(workspace=second_workspace) + session.commit() + res = test_client.get(self.url()) + assert res.status_code == 200 + assert len(res.json) == OBJECT_COUNT + + +class RetrieveTestsMixin: + + def test_retrieve_one_object(self, test_client): + res = test_client.get(self.url(self.first_object)) + assert res.status_code == 200 + assert isinstance(res.json, dict) + + def test_retrieve_fails_object_of_other_workspcae(self, + test_client, + session, + second_workspace): + res = test_client.get(self.url(self.first_object, second_workspace)) + assert res.status_code == 404 + + @pytest.mark.parametrize('object_id', [123, -1, 'xxx', u'áá']) + def test_404_when_retrieving_unexistent_object(self, test_client, + object_id): + url = self.url(object_id) + res = test_client.get(url) + assert res.status_code == 404 + + +class CreateTestsMixin: + + def test_create_succeeds(self, test_client): + res = test_client.post(self.url(), + data=self.factory.build_dict()) + assert res.status_code == 201 + assert self.model.query.count() == OBJECT_COUNT + 1 + object_id = res.json['id'] + obj = self.model.query.get(object_id) + assert obj.workspace == self.workspace + + def test_create_fails_with_empty_dict(self, test_client): + res = test_client.post(self.url(), data={}) + assert res.status_code == 400 + + def test_create_fails_with_existing(self, session, test_client): + for unique_field in self.unique_fields: + data = self.factory.build_dict() + data[unique_field] = getattr(self.first_object, unique_field) + res = test_client.post(self.url(), data=data) + assert res.status_code == 400 + assert self.model.query.count() == OBJECT_COUNT + + def test_create_with_existing_in_other_workspace(self, test_client, + session, + second_workspace): + unique_field = self.unique_fields[0] + other_object = self.factory.create(workspace=second_workspace) + session.commit() + + data = self.factory.build_dict() + data[unique_field] = getattr(other_object, unique_field) + res = test_client.post(self.url(), data=data) + assert res.status_code == 201 + # It should create two hosts, one for each workspace + assert self.model.query.count() == OBJECT_COUNT + 2 + + +class UpdateTestsMixin: + + def test_update_a_host(self, test_client): + host = self.workspace.hosts[0] + res = test_client.put(self.url(self.first_object), + data=self.factory.build_dict()) + assert res.status_code == 200 + assert self.model.query.count() == OBJECT_COUNT + + def test_update_fails_with_existing(self, test_client, session): + for unique_field in self.unique_fields: + data = self.factory.build_dict() + data[unique_field] = getattr(self.first_object, unique_field) + res = test_client.put(self.url(self.workspace.hosts[1]), data=data) + assert res.status_code == 400 + assert self.model.query.count() == OBJECT_COUNT + + def test_update_a_host_fails_with_empty_dict(self, test_client): + """To do this the user should use a PATCH request""" + host = self.workspace.hosts[0] + res = test_client.put(self.url(host), data={}) + assert res.status_code == 400 + + +class DeleteTestsMixin: + + def test_delete(self, test_client): + res = test_client.delete(self.url(self.first_object)) + assert res.status_code == 204 # No content + assert was_deleted(self.first_object) + assert self.model.query.count() == OBJECT_COUNT - 1 + + def test_delete_from_other_workspace_fails(self, test_client, + second_workspace): + res = test_client.delete(self.url(self.first_object, + workspace=second_workspace)) + assert res.status_code == 404 # No content + assert not was_deleted(self.first_object) + assert self.model.query.count() == OBJECT_COUNT + + +class ReadWriteTestsMixin(ListTestsMixin, + RetrieveTestsMixin, + CreateTestsMixin, + UpdateTestsMixin, + DeleteTestsMixin): + pass + + +class ReadWriteAPITests(ReadWriteTestsMixin, + GenericAPITest): + pass From b4005aafa8b51c32e0a4dae22069baa0b9496c35 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 12 Sep 2017 15:29:57 -0300 Subject: [PATCH 0196/1506] Add Workspace name to FaradayEntity importer Allows for logging of the ws name when importing objects. --- server/importer.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/importer.py b/server/importer.py index c2a21dd0e5e..e470369836b 100644 --- a/server/importer.py +++ b/server/importer.py @@ -632,6 +632,9 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation class FaradayEntityImporter(object): # Document Types: [u'Service', u'Communication', u'Vulnerability', u'CommandRunInformation', u'Reports', u'Host', u'Workspace'] + def __init__(self, workspace_name): + self.workspace_name = workspace_name + def parse(self, document): """Get an instance of a DAO object given a document""" importer_class = self.get_importer_from_document(document) @@ -644,7 +647,7 @@ def parse(self, document): return None, None def get_importer_from_document(self, doc_type): - logger.info('Getting class importer for {0}'.format(doc_type)) + logger.info('Getting class importer for {0} in workspace {1}'.format(doc_type, self.workspace_name)) importer_class_mapper = { 'EntityMetadata': EntityMetadataImporter, 'Host': HostImporter, @@ -930,7 +933,7 @@ def verify_import_data(self, couchdb_relational_map, couchdb_relational_map_by_t def import_workspace_into_database(self, workspace_name): - faraday_importer = FaradayEntityImporter() + faraday_importer = FaradayEntityImporter(workspace_name) workspace, created = get_or_create(session, Workspace, name=workspace_name) session.commit() From 31545b44995fa480c1c3b90f976b0bf9623dd7cd Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 12 Sep 2017 15:30:49 -0300 Subject: [PATCH 0197/1506] Fix error when CouchDB is not running When trying to import from a turned off CouchDB, a nice error is shown to the user. --- server/importer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index e470369836b..7bf5eaff6d0 100644 --- a/server/importer.py +++ b/server/importer.py @@ -849,13 +849,14 @@ def run(self): """ Main entry point for couchdb import """ + couchdb_server_conn, workspaces_list = self._open_couchdb_conn() license_import = ImportLicense() license_import.run() vuln_templates_import = ImportVulnerabilityTemplates() vuln_templates_import.run() users_import = ImportCouchDBUsers() users_import.run() - couchdb_server_conn, workspaces_list = self._open_couchdb_conn() + for workspace_name in workspaces_list: logger.info(u'Setting up workspace {}'.format(workspace_name)) From a772fa458485bfb1d747630f9c4f51e7371fa097 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 12 Sep 2017 15:31:50 -0300 Subject: [PATCH 0198/1506] Add description to unique contraint for vulns A hashing function is needed in order to avoid exceeding Postgres max length. The constraint was replaced by an index to allow using md5() over the description. --- server/models.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/server/models.py b/server/models.py index c2694f6b454..a286dd15c1d 100644 --- a/server/models.py +++ b/server/models.py @@ -7,6 +7,7 @@ Column, DateTime, Enum, + event, Float, ForeignKey, Integer, @@ -15,7 +16,7 @@ UniqueConstraint, ) from sqlalchemy.orm import relationship, backref -from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.schema import DDL from flask_sqlalchemy import SQLAlchemy from flask_security import ( RoleMixin, @@ -787,15 +788,15 @@ class ExecutiveReport(db.Model): '(Vulnerability.source_code_id IS NOT NULL)::int)=1', name='check_vulnerability_host_service_source_code', table=VulnerabilityGeneric.__table__) -UniqueConstraint(VulnerabilityGeneric.name, - VulnerabilityGeneric.severity, - Vulnerability.host_id, - VulnerabilityWeb.service_id, - VulnerabilityWeb.method, - VulnerabilityWeb.parameter_name, - VulnerabilityWeb.path, - VulnerabilityWeb.website, - VulnerabilityGeneric.workspace_id, - VulnerabilityCode.source_code_id, - name='uix_vulnerability' + +vulnerability_uniqueness = DDL( + "CREATE UNIQUE INDEX uix_vulnerability ON %(fullname)s " + "(name, md5(description), severity, host_id, service_id, " + "method, parameter_name, path, website, workspace_id, source_code_id);" +) + +event.listen( + VulnerabilityGeneric.__table__, + 'after_create', + vulnerability_uniqueness.execute_if(dialect='postgresql') ) \ No newline at end of file From e4e955d518aa175df6d919e7916acd0670965d45 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 12 Sep 2017 16:15:22 -0300 Subject: [PATCH 0199/1506] Allow creator to be nullalble --- server/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index 32fc42933a5..30381cc1170 100644 --- a/server/models.py +++ b/server/models.py @@ -46,7 +46,7 @@ class Metadata(db.Model): @declared_attr def creator_id(cls): - return Column(Integer, ForeignKey('user.id'), nullable=False) + return Column(Integer, ForeignKey('user.id'), nullable=True) @declared_attr def creator(cls): From e23a491ecbcac4d10437ad52ec71d6dbb8df4791 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 12 Sep 2017 16:52:08 -0300 Subject: [PATCH 0200/1506] Refactor API generic views To make it support views that don't depend on a workspace, this make them really generic. I changed the order of _get_object when inheriting from ReadWriteWorkspacedView, it first takes the object_id and then the workspace_name --- server/api/base.py | 109 ++++++++++++++++--------- server/api/modules/hosts.py | 2 +- test_cases/test_api_workspaced_base.py | 3 + 3 files changed, 73 insertions(+), 41 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index a4a7c4e74af..79d01db4f81 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -22,21 +22,21 @@ def output_json(data, code, headers=None): # TODO: Require @view decorator to enable custom routes -class GenericWorkspacedView(FlaskView): - """Abstract class for a view that depends on the workspace, that is - passed in the URL""" +class GenericView(FlaskView): + """Abstract class to provide helpers. Inspired in Django REST + Framework generic viewsets""" # Must-implement attributes model_class = None schema_class = None # Default attributes - route_prefix = '/v2//' - base_args = ['workspace_name'] # Required to prevent double usage of + route_prefix = '/v2/' + base_args = [] representations = {'application/json': output_json} lookup_field = 'id' lookup_field_type = int - unique_fields = [] # Fields unique together with workspace_id + unique_fields = [] # Fields unique def _get_schema_class(self): assert self.schema_class is not None, "You must define schema_class" @@ -45,21 +45,19 @@ def _get_schema_class(self): def _get_lookup_field(self): return getattr(self.model_class, self.lookup_field) - def _get_workspace(self, workspace_name): + def _validate_object_id(self, object_id): try: - ws = Workspace.query.filter_by(name=workspace_name).one() - except NoResultFound: - flask.abort(404, "No such workspace: %s" % workspace_name) - return ws + self.lookup_field_type(object_id) + except ValueError: + flask.abort(404, 'Invalid format of lookup field') - def _get_base_query(self, workspace_name): - return self.model_class.query.join(Workspace) \ - .filter(Workspace.id==self._get_workspace(workspace_name).id) + def _get_base_query(self): + return self.model_class.query - def _get_object(self, workspace_name, object_id): + def _get_object(self, object_id, **kwargs): self._validate_object_id(object_id) try: - obj = self._get_base_query(workspace_name).filter( + obj = self._get_base_query(**kwargs).filter( self._get_lookup_field() == object_id).one() except NoResultFound: flask.abort(404, 'Object with id "%s" not found' % object_id) @@ -72,13 +70,61 @@ def _parse_data(self, schema, request, *args, **kwargs): return FlaskParser().parse(schema, request, locations=('json',), *args, **kwargs) - def _validate_object_id(self, object_id): + def _validate_uniqueness(self, obj, object_id=None): + # TODO: Implement this + return True + + @classmethod + def register(cls, app, *args, **kwargs): + """Register and add JSON error handler. Use error code + 400 instead of 422""" + super(GenericView, cls).register(app, *args, **kwargs) + @app.errorhandler(422) + def handle_unprocessable_entity(err): + # webargs attaches additional metadata to the `data` attribute + exc = getattr(err, 'exc') + if exc: + # Get validations from the ValidationError object + messages = exc.messages + else: + messages = ['Invalid request'] + return flask.jsonify({ + 'messages': messages, + }), 400 + + +class GenericWorkspacedView(GenericView): + """Abstract class for a view that depends on the workspace, that is + passed in the URL""" + + # Default attributes + route_prefix = '/v2//' + base_args = ['workspace_name'] # Required to prevent double usage of + unique_fields = [] # Fields unique together with workspace_id + + def _get_workspace(self, workspace_name): try: - self.lookup_field_type(object_id) - except ValueError: - flask.abort(404, 'Invalid format of lookup field') + ws = Workspace.query.filter_by(name=workspace_name).one() + except NoResultFound: + flask.abort(404, "No such workspace: %s" % workspace_name) + return ws + + def _get_base_query(self, workspace_name): + base = super(GenericWorkspacedView, self)._get_base_query() + return base.join(Workspace).filter( + Workspace.id==self._get_workspace(workspace_name).id) + + def _get_object(self, object_id, workspace_name): + self._validate_object_id(object_id) + try: + obj = self._get_base_query(workspace_name).filter( + self._get_lookup_field() == object_id).one() + except NoResultFound: + flask.abort(404, 'Object with id "%s" not found' % object_id) + return obj def _validate_uniqueness(self, obj, object_id=None): + # TODO: Use implementation of GenericView assert obj.workspace is not None, "Object must have a " \ "workspace attribute set to call _validate_uniqueness" primary_key_field = inspect(self.model_class).primary_key[0] @@ -97,23 +143,6 @@ def _validate_uniqueness(self, obj, object_id=None): field_name, value ))) - @classmethod - def register(cls, app, *args, **kwargs): - """Register and add JSON error handler. Use error code - 400 instead of 422""" - super(GenericWorkspacedView, cls).register(app, *args, **kwargs) - @app.errorhandler(422) - def handle_unprocessable_entity(err): - # webargs attaches additional metadata to the `data` attribute - exc = getattr(err, 'exc') - if exc: - # Get validations from the ValidationError object - messages = exc.messages - else: - messages = ['Invalid request'] - return flask.jsonify({ - 'messages': messages, - }), 400 class ListWorkspacedMixin(object): """Add GET // route""" @@ -127,7 +156,7 @@ class RetrieveWorkspacedMixin(object): """Add GET /// route""" def get(self, workspace_name, object_id): - return self._dump(self._get_object(workspace_name, object_id)) + return self._dump(self._get_object(object_id, workspace_name)) class ReadOnlyWorkspacedView(ListWorkspacedMixin, @@ -162,7 +191,7 @@ class UpdateWorkspacedMixin(object): def put(self, workspace_name, object_id): data = self._parse_data(self._get_schema_class()(strict=True), flask.request) - obj = self._get_object(workspace_name, object_id) + obj = self._get_object(object_id, workspace_name) self._update_object(obj, data) updated = self._perform_update(workspace_name, object_id, obj) return self._dump(obj).data, 200 @@ -181,7 +210,7 @@ def _perform_update(self, workspace_name, object_id, obj): class DeleteWorkspacedMixin(object): def delete(self, workspace_name, object_id): - obj = self._get_object(workspace_name, object_id) + obj = self._get_object(object_id, workspace_name) self._perform_delete(obj) return None, 204 diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 880e027d8ec..598d169a2b7 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -43,7 +43,7 @@ class HostsView(ReadWriteWorkspacedView): @route('//services/') def service_list(self, workspace_name, host_id): - services = self._get_object(workspace_name, host_id).services + services = self._get_object(host_id, workspace_name).services return ServiceSchema(many=True).dump(services) def _get_base_query(self, workspace_name): diff --git a/test_cases/test_api_workspaced_base.py b/test_cases/test_api_workspaced_base.py index 3d9b9d3d583..8fa9b4ce324 100644 --- a/test_cases/test_api_workspaced_base.py +++ b/test_cases/test_api_workspaced_base.py @@ -1,4 +1,7 @@ #-*- coding: utf8 -*- + +"""Generic tests for APIs prefixed with a workspace_name""" + import pytest from sqlalchemy.orm.util import was_deleted from server.models import db, Workspace From 52dec81a6adb7ee458ed247b1c9b89c7daba4257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 12 Sep 2017 18:31:56 -0300 Subject: [PATCH 0201/1506] Add List generic mixin and License API with that mixin --- server/api/base.py | 35 +++- server/app.py | 2 + test_cases/conftest.py | 1 + test_cases/factories.py | 37 +++- test_cases/test_api_non_workspaced_base.py | 167 ++++++++++++++++++ test_cases/test_api_non_workspaced_various.py | 17 ++ 6 files changed, 248 insertions(+), 11 deletions(-) create mode 100644 test_cases/test_api_non_workspaced_base.py create mode 100644 test_cases/test_api_non_workspaced_various.py diff --git a/server/api/base.py b/server/api/base.py index 79d01db4f81..13370089d63 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -144,14 +144,20 @@ def _validate_uniqueness(self, obj, object_id=None): ))) -class ListWorkspacedMixin(object): +class ListMixin(object): """Add GET // route""" - def index(self, workspace_name): - return self._dump(self._get_base_query(workspace_name).all(), + def index(self, **kwargs): + return self._dump(self._get_base_query(**kwargs).all(), many=True) +class ListWorkspacedMixin(ListMixin): + # There are no differences with the non-workspaced implementations. The code + # inside the view generic methods is enough + pass + + class RetrieveWorkspacedMixin(object): """Add GET /// route""" @@ -159,10 +165,17 @@ def get(self, workspace_name, object_id): return self._dump(self._get_object(object_id, workspace_name)) +class ReadOnlyView(ListMixin, + # RetrieveMixin, + GenericView): + """A generic view with list and retrieve endpoints""" + pass + + class ReadOnlyWorkspacedView(ListWorkspacedMixin, RetrieveWorkspacedMixin, GenericWorkspacedView): - """A generic view with list and retrieve endpoints""" + """A workspaced generic view with list and retrieve endpoints""" pass @@ -219,10 +232,18 @@ def _perform_delete(self, obj): db.session.commit() +class ReadWriteView(#CreateWorkspacedMixin, + #UpdateWorkspacedMixin, + # DeleteWorkspacedMixin, + ReadOnlyView): + """A generic view with list, retrieve and create endpoints""" + pass + + class ReadWriteWorkspacedView(CreateWorkspacedMixin, UpdateWorkspacedMixin, DeleteWorkspacedMixin, - ReadOnlyWorkspacedView, - GenericWorkspacedView): - """A generic view with list, retrieve and create endpoints""" + ReadOnlyWorkspacedView): + """A generic workspaced view with list, retrieve and create + endpoints""" pass diff --git a/server/app.py b/server/app.py index 2f4778817fc..a25f3f992bd 100644 --- a/server/app.py +++ b/server/app.py @@ -59,6 +59,7 @@ def create_app(db_connection_string=None, testing=None): from server.api.modules.doc import doc_api from server.api.modules.vuln_csv import vuln_csv_api from server.api.modules.hosts import host_api + from server.api.modules.licenses import license_api from server.api.modules.commandsrun import commandsrun_api from server.api.modules.services import services_api from server.api.modules.credentials import credentials_api @@ -68,6 +69,7 @@ def create_app(db_connection_string=None, testing=None): app.register_blueprint(doc_api) app.register_blueprint(vuln_csv_api) app.register_blueprint(host_api) + app.register_blueprint(license_api) app.register_blueprint(commandsrun_api) app.register_blueprint(services_api) app.register_blueprint(credentials_api) diff --git a/test_cases/conftest.py b/test_cases/conftest.py index f67c569e282..34ce9c29cde 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -19,6 +19,7 @@ factories.ServiceFactory, factories.VulnerabilityFactory, factories.CredentialFactory, + factories.LicenseFactory, ] for factory in enabled_factories: register(factory) diff --git a/test_cases/factories.py b/test_cases/factories.py index 85c94b8ece9..e2d2c55af9d 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -1,21 +1,39 @@ import factory +import datetime from factory.fuzzy import ( FuzzyChoice, + FuzzyNaiveDateTime, FuzzyInteger, FuzzyText, ) from server.models import ( db, - Host, Command, - Service, - Workspace, Credential, - Vulnerability, EntityMetadata, + Host, + License, + Service, + Vulnerability, + Workspace, +) + +# Make partials for start and end date. End date must be after start date +FuzzyStartTime = lambda: ( + FuzzyNaiveDateTime( + datetime.datetime.now() - datetime.timedelta(days=40), + datetime.datetime.now() - datetime.timedelta(days=20), + ) +) +FuzzyEndTime = lambda: ( + FuzzyNaiveDateTime( + datetime.datetime.now() - datetime.timedelta(days=19), + datetime.datetime.now() + ) ) + class FaradayFactory(factory.alchemy.SQLAlchemyModelFactory): # id = factory.Sequence(lambda n: n) @@ -105,3 +123,14 @@ class CommandFactory(WorkspaceObjectFactory): class Meta: model = Command sqlalchemy_session = db.session + + +class LicenseFactory(FaradayFactory): + product = FuzzyText() + start_date = FuzzyStartTime() + end_date = FuzzyEndTime() + type = FuzzyText() + + class Meta: + model = License + sqlalchemy_session = db.session diff --git a/test_cases/test_api_non_workspaced_base.py b/test_cases/test_api_non_workspaced_base.py new file mode 100644 index 00000000000..fdaabb1bffd --- /dev/null +++ b/test_cases/test_api_non_workspaced_base.py @@ -0,0 +1,167 @@ +#-*- coding: utf8 -*- + +"""Generic tests for APIs NOT prefixed with a workspace_name""" + +import pytest +from sqlalchemy.orm.util import was_deleted +from server.models import db + +API_PREFIX = '/v2/' +OBJECT_COUNT = 5 + +@pytest.mark.usefixtures('logged_user') +class GenericAPITest: + + model = None + factory = None + api_endpoint = None + pk_field = 'id' + unique_fields = [] + update_fields = [] + + @pytest.fixture(autouse=True) + def load_many_objects(self, database, session): + objects = self.factory.create_batch(OBJECT_COUNT) + self.first_object = objects[0] + session.commit() + assert self.model.query.count() == OBJECT_COUNT + return objects + + @pytest.fixture + def object_instance(self, session): + """An object instance saved in the database""" + obj = self.factory.create() + session.commit() + return obj + + def url(self, obj=None): + url = API_PREFIX + self.api_endpoint + '/' + if obj is not None: + id_ = unicode(obj.id) if isinstance( + obj, self.model) else unicode(obj) + url += id_ + u'/' + return url + + +class ListTestsMixin: + + def test_list_retrieves_all_items_from(self, test_client, + session): + res = test_client.get(self.url()) + assert res.status_code == 200 + assert len(res.json) == OBJECT_COUNT + + +class RetrieveTestsMixin: + + def test_retrieve_one_object(self, test_client): + res = test_client.get(self.url(self.first_object)) + assert res.status_code == 200 + assert isinstance(res.json, dict) + + def test_retrieve_fails_object_of_other_workspcae(self, + test_client, + session, + second_workspace): + res = test_client.get(self.url(self.first_object, second_workspace)) + assert res.status_code == 404 + + @pytest.mark.parametrize('object_id', [123, -1, 'xxx', u'áá']) + def test_404_when_retrieving_unexistent_object(self, test_client, + object_id): + url = self.url(object_id) + res = test_client.get(url) + assert res.status_code == 404 + + +class CreateTestsMixin: + + def test_create_succeeds(self, test_client): + res = test_client.post(self.url(), + data=self.factory.build_dict()) + assert res.status_code == 201 + assert self.model.query.count() == OBJECT_COUNT + 1 + object_id = res.json['id'] + obj = self.model.query.get(object_id) + assert obj.workspace == self.workspace + + def test_create_fails_with_empty_dict(self, test_client): + res = test_client.post(self.url(), data={}) + assert res.status_code == 400 + + def test_create_fails_with_existing(self, session, test_client): + for unique_field in self.unique_fields: + data = self.factory.build_dict() + data[unique_field] = getattr(self.first_object, unique_field) + res = test_client.post(self.url(), data=data) + assert res.status_code == 400 + assert self.model.query.count() == OBJECT_COUNT + + def test_create_with_existing_in_other_workspace(self, test_client, + session, + second_workspace): + unique_field = self.unique_fields[0] + other_object = self.factory.create(workspace=second_workspace) + session.commit() + + data = self.factory.build_dict() + data[unique_field] = getattr(other_object, unique_field) + res = test_client.post(self.url(), data=data) + assert res.status_code == 201 + # It should create two hosts, one for each workspace + assert self.model.query.count() == OBJECT_COUNT + 2 + + +class UpdateTestsMixin: + + def test_update_a_host(self, test_client): + host = self.workspace.hosts[0] + res = test_client.put(self.url(self.first_object), + data=self.factory.build_dict()) + assert res.status_code == 200 + assert self.model.query.count() == OBJECT_COUNT + + def test_update_fails_with_existing(self, test_client, session): + for unique_field in self.unique_fields: + data = self.factory.build_dict() + data[unique_field] = getattr(self.first_object, unique_field) + res = test_client.put(self.url(self.workspace.hosts[1]), data=data) + assert res.status_code == 400 + assert self.model.query.count() == OBJECT_COUNT + + def test_update_a_host_fails_with_empty_dict(self, test_client): + """To do this the user should use a PATCH request""" + host = self.workspace.hosts[0] + res = test_client.put(self.url(host), data={}) + assert res.status_code == 400 + + +class DeleteTestsMixin: + + def test_delete(self, test_client): + res = test_client.delete(self.url(self.first_object)) + assert res.status_code == 204 # No content + assert was_deleted(self.first_object) + assert self.model.query.count() == OBJECT_COUNT - 1 + + def test_delete_from_other_workspace_fails(self, test_client, + second_workspace): + res = test_client.delete(self.url(self.first_object, + workspace=second_workspace)) + assert res.status_code == 404 # No content + assert not was_deleted(self.first_object) + assert self.model.query.count() == OBJECT_COUNT + + +class ReadWriteTestsMixin(ListTestsMixin, + # RetrieveTestsMixin, + # CreateTestsMixin, + # UpdateTestsMixin, + # DeleteTestsMixin + ): + pass + + +class ReadWriteAPITests(ReadWriteTestsMixin, + GenericAPITest): + pass diff --git a/test_cases/test_api_non_workspaced_various.py b/test_cases/test_api_non_workspaced_various.py new file mode 100644 index 00000000000..36ed35b50ee --- /dev/null +++ b/test_cases/test_api_non_workspaced_various.py @@ -0,0 +1,17 @@ +#-*- coding: utf8 -*- + +"""Tests for many API endpoints that do not depend on workspace_name""" + +import pytest +from test_cases import factories +from test_api_non_workspaced_base import ReadWriteAPITests +from server.models import License + +class TestLicensesAPI(ReadWriteAPITests): + model = License + factory = factories.LicenseFactory + api_endpoint = 'licenses' + unique_fields = ['ip'] + update_fields = ['ip', 'description', 'os'] + + From 743429a2fca2127d27ea2e96e1fd535ccb1ffce5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 12 Sep 2017 19:01:13 -0300 Subject: [PATCH 0202/1506] Add generic retrieve mixin and apply it to licenses API --- server/api/base.py | 22 +++++++++++++++------- test_cases/test_api_non_workspaced_base.py | 9 +-------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 13370089d63..2e3d98c7188 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -145,7 +145,7 @@ def _validate_uniqueness(self, obj, object_id=None): class ListMixin(object): - """Add GET // route""" + """Add GET / route""" def index(self, **kwargs): return self._dump(self._get_base_query(**kwargs).all(), @@ -153,20 +153,28 @@ def index(self, **kwargs): class ListWorkspacedMixin(ListMixin): + """Add GET // route""" # There are no differences with the non-workspaced implementations. The code # inside the view generic methods is enough pass -class RetrieveWorkspacedMixin(object): - """Add GET /// route""" +class RetrieveMixin(object): + """Add GET // route""" + + def get(self, object_id, **kwargs): + return self._dump(self._get_object(object_id, **kwargs)) + - def get(self, workspace_name, object_id): - return self._dump(self._get_object(object_id, workspace_name)) +class RetrieveWorkspacedMixin(RetrieveMixin): + """Add GET /// route""" + # There are no differences with the non-workspaced implementations. The code + # inside the view generic methods is enough + pass class ReadOnlyView(ListMixin, - # RetrieveMixin, + RetrieveMixin, GenericView): """A generic view with list and retrieve endpoints""" pass @@ -234,7 +242,7 @@ def _perform_delete(self, obj): class ReadWriteView(#CreateWorkspacedMixin, #UpdateWorkspacedMixin, - # DeleteWorkspacedMixin, + #DeleteWorkspacedMixin, ReadOnlyView): """A generic view with list, retrieve and create endpoints""" pass diff --git a/test_cases/test_api_non_workspaced_base.py b/test_cases/test_api_non_workspaced_base.py index fdaabb1bffd..795e147232c 100644 --- a/test_cases/test_api_non_workspaced_base.py +++ b/test_cases/test_api_non_workspaced_base.py @@ -59,13 +59,6 @@ def test_retrieve_one_object(self, test_client): assert res.status_code == 200 assert isinstance(res.json, dict) - def test_retrieve_fails_object_of_other_workspcae(self, - test_client, - session, - second_workspace): - res = test_client.get(self.url(self.first_object, second_workspace)) - assert res.status_code == 404 - @pytest.mark.parametrize('object_id', [123, -1, 'xxx', u'áá']) def test_404_when_retrieving_unexistent_object(self, test_client, object_id): @@ -154,7 +147,7 @@ def test_delete_from_other_workspace_fails(self, test_client, class ReadWriteTestsMixin(ListTestsMixin, - # RetrieveTestsMixin, + RetrieveTestsMixin, # CreateTestsMixin, # UpdateTestsMixin, # DeleteTestsMixin From 8a812ccb1909de3c482a92f68e7b5b56ac2876b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 12 Sep 2017 19:47:35 -0300 Subject: [PATCH 0203/1506] Add generic create mixin and apply it to licences API It fails when POSTing with an empty dict --- server/api/base.py | 27 ++++++++++++------- test_cases/factories.py | 15 ++++++++++- test_cases/test_api_non_workspaced_base.py | 17 +----------- test_cases/test_api_non_workspaced_various.py | 4 +-- 4 files changed, 35 insertions(+), 28 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 2e3d98c7188..0c7f2d4ab7b 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -187,27 +187,36 @@ class ReadOnlyWorkspacedView(ListWorkspacedMixin, pass -class CreateWorkspacedMixin(object): +class CreateMixin(object): + """Add POST / route""" - def post(self, workspace_name): + def post(self, **kwargs): data = self._parse_data(self._get_schema_class()(strict=True), flask.request) obj = self.model_class(**data) - created = self._perform_create(workspace_name, obj) + created = self._perform_create(obj, **kwargs) return self._dump(created).data, 201 - def _perform_create(self, workspace_name, obj): - assert not db.session.new + def _perform_create(self, obj): + # assert not db.session.new with db.session.no_autoflush: # Required because _validate_uniqueness does a select. Doing this # outside a no_autoflush block would result in a premature create. - obj.workspace = self._get_workspace(workspace_name) self._validate_uniqueness(obj) db.session.add(obj) db.session.commit() return obj +class CreateWorkspacedMixin(CreateMixin): + """Add POST // route""" + + def _perform_create(self, obj, workspace_name): + assert not db.session.new + obj.workspace = self._get_workspace(workspace_name) + return super(CreateWorkspacedMixin, self)._perform_create(obj) + + class UpdateWorkspacedMixin(object): def put(self, workspace_name, object_id): data = self._parse_data(self._get_schema_class()(strict=True), @@ -240,9 +249,9 @@ def _perform_delete(self, obj): db.session.commit() -class ReadWriteView(#CreateWorkspacedMixin, - #UpdateWorkspacedMixin, - #DeleteWorkspacedMixin, +class ReadWriteView(CreateMixin, + #UpdateMixin, + #DeleteMixin, ReadOnlyView): """A generic view with list, retrieve and create endpoints""" pass diff --git a/test_cases/factories.py b/test_cases/factories.py index e2d2c55af9d..cbea807eab2 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -39,6 +39,11 @@ class FaradayFactory(factory.alchemy.SQLAlchemyModelFactory): # id = factory.Sequence(lambda n: n) pass + @classmethod + def build_dict(cls, **kwargs): + return factory.build(dict, FACTORY_CLASS=cls) + + class WorkspaceFactory(FaradayFactory): @@ -54,7 +59,7 @@ class WorkspaceObjectFactory(FaradayFactory): @classmethod def build_dict(cls, **kwargs): - ret = factory.build(dict, FACTORY_CLASS=cls) + ret = super(WorkspaceObjectFactory, cls).build_dict(**kwargs) del ret['workspace'] # It is passed in the URL, not in POST data return ret @@ -134,3 +139,11 @@ class LicenseFactory(FaradayFactory): class Meta: model = License sqlalchemy_session = db.session + + @classmethod + def build_dict(cls, **kwargs): + # Ugly hack to JSON-serialize datetimes + ret = super(LicenseFactory, cls).build_dict(**kwargs) + ret['start_date'] = ret['start_date'].isoformat() + ret['end_date'] = ret['end_date'].isoformat() + return ret diff --git a/test_cases/test_api_non_workspaced_base.py b/test_cases/test_api_non_workspaced_base.py index 795e147232c..f0f7bdb2073 100644 --- a/test_cases/test_api_non_workspaced_base.py +++ b/test_cases/test_api_non_workspaced_base.py @@ -76,7 +76,6 @@ def test_create_succeeds(self, test_client): assert self.model.query.count() == OBJECT_COUNT + 1 object_id = res.json['id'] obj = self.model.query.get(object_id) - assert obj.workspace == self.workspace def test_create_fails_with_empty_dict(self, test_client): res = test_client.post(self.url(), data={}) @@ -90,20 +89,6 @@ def test_create_fails_with_existing(self, session, test_client): assert res.status_code == 400 assert self.model.query.count() == OBJECT_COUNT - def test_create_with_existing_in_other_workspace(self, test_client, - session, - second_workspace): - unique_field = self.unique_fields[0] - other_object = self.factory.create(workspace=second_workspace) - session.commit() - - data = self.factory.build_dict() - data[unique_field] = getattr(other_object, unique_field) - res = test_client.post(self.url(), data=data) - assert res.status_code == 201 - # It should create two hosts, one for each workspace - assert self.model.query.count() == OBJECT_COUNT + 2 - class UpdateTestsMixin: @@ -148,7 +133,7 @@ def test_delete_from_other_workspace_fails(self, test_client, class ReadWriteTestsMixin(ListTestsMixin, RetrieveTestsMixin, - # CreateTestsMixin, + CreateTestsMixin, # UpdateTestsMixin, # DeleteTestsMixin ): diff --git a/test_cases/test_api_non_workspaced_various.py b/test_cases/test_api_non_workspaced_various.py index 36ed35b50ee..37471b925e0 100644 --- a/test_cases/test_api_non_workspaced_various.py +++ b/test_cases/test_api_non_workspaced_various.py @@ -11,7 +11,7 @@ class TestLicensesAPI(ReadWriteAPITests): model = License factory = factories.LicenseFactory api_endpoint = 'licenses' - unique_fields = ['ip'] - update_fields = ['ip', 'description', 'os'] + # unique_fields = ['ip'] + # update_fields = ['ip', 'description', 'os'] From 6e549eb2eb7e2f22e348495c287c39e432750612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 13 Sep 2017 14:51:13 -0300 Subject: [PATCH 0204/1506] Add dev requirements Dependencies for testing --- requirements_dev.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 requirements_dev.txt diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 00000000000..dbf901a1aab --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,3 @@ +pytest +pytest-factoryboy +factory-boy<=2.8.1 From 8cdb33a18197c3f7f49570345148a09b0954063a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 13 Sep 2017 14:51:57 -0300 Subject: [PATCH 0205/1506] Add licenses API code I forgot to add this before --- server/api/modules/licenses.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 server/api/modules/licenses.py diff --git a/server/api/modules/licenses.py b/server/api/modules/licenses.py new file mode 100644 index 00000000000..61173568f14 --- /dev/null +++ b/server/api/modules/licenses.py @@ -0,0 +1,28 @@ +# Faraday Penetration Test IDE +# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) +# See the file 'doc/LICENSE' for the license information + +import flask +from flask import Blueprint +from marshmallow import Schema, fields + +from server.models import License +from server.api.base import ReadWriteView + +license_api = Blueprint('license_api', __name__) + + +class LicenseSchema(Schema): + id = fields.Integer(required=True, dump_only=True) + product = fields.String() + start_date = fields.DateTime() + end_date = fields.DateTime() + + +class LicenseView(ReadWriteView): + route_base = 'licenses' + model_class = License + schema_class = LicenseSchema + unique_fields = [] + +LicenseView.register(license_api) From 99c752597e718d71ec56fa5067cbec3ae00012cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 13 Sep 2017 15:12:13 -0300 Subject: [PATCH 0206/1506] Set required fields in LicenseSchema Now all API tests succeed --- server/api/modules/licenses.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/api/modules/licenses.py b/server/api/modules/licenses.py index 61173568f14..914f3177aab 100644 --- a/server/api/modules/licenses.py +++ b/server/api/modules/licenses.py @@ -14,9 +14,9 @@ class LicenseSchema(Schema): id = fields.Integer(required=True, dump_only=True) - product = fields.String() - start_date = fields.DateTime() - end_date = fields.DateTime() + product = fields.String(required=True) + start_date = fields.DateTime(required=True) + end_date = fields.DateTime(required=True) class LicenseView(ReadWriteView): From aacf27a489af07b24dc62a12d6ac9aa38101d368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 13 Sep 2017 15:19:12 -0300 Subject: [PATCH 0207/1506] Add generic delete API mixin and apply it to licenses API --- server/api/base.py | 14 ++++++++++---- test_cases/test_api_non_workspaced_base.py | 11 +++++------ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 0c7f2d4ab7b..75bc6409b20 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -238,9 +238,10 @@ def _perform_update(self, workspace_name, object_id, obj): db.session.commit() -class DeleteWorkspacedMixin(object): - def delete(self, workspace_name, object_id): - obj = self._get_object(object_id, workspace_name) +class DeleteMixin(object): + """Add DELETE // route""" + def delete(self, object_id, **kwargs): + obj = self._get_object(object_id, **kwargs) self._perform_delete(obj) return None, 204 @@ -249,9 +250,14 @@ def _perform_delete(self, obj): db.session.commit() +class DeleteWorkspacedMixin(DeleteMixin): + """Add DELETE /// route""" + pass + + class ReadWriteView(CreateMixin, #UpdateMixin, - #DeleteMixin, + DeleteMixin, ReadOnlyView): """A generic view with list, retrieve and create endpoints""" pass diff --git a/test_cases/test_api_non_workspaced_base.py b/test_cases/test_api_non_workspaced_base.py index f0f7bdb2073..fdb603702b0 100644 --- a/test_cases/test_api_non_workspaced_base.py +++ b/test_cases/test_api_non_workspaced_base.py @@ -122,12 +122,11 @@ def test_delete(self, test_client): assert was_deleted(self.first_object) assert self.model.query.count() == OBJECT_COUNT - 1 - def test_delete_from_other_workspace_fails(self, test_client, - second_workspace): - res = test_client.delete(self.url(self.first_object, - workspace=second_workspace)) + @pytest.mark.parametrize('object_id', [123, -1, 'xxx', u'áá']) + def test_delete_non_existent_raises_404(self, test_client, + object_id): + res = test_client.delete(self.url(object_id)) assert res.status_code == 404 # No content - assert not was_deleted(self.first_object) assert self.model.query.count() == OBJECT_COUNT @@ -135,7 +134,7 @@ class ReadWriteTestsMixin(ListTestsMixin, RetrieveTestsMixin, CreateTestsMixin, # UpdateTestsMixin, - # DeleteTestsMixin + DeleteTestsMixin, ): pass From 988afb83a626ded2d9a0b5fead1ce8403cd3436d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 13 Sep 2017 15:35:15 -0300 Subject: [PATCH 0208/1506] Fix generic update tests They were only valid for host tests, and not for other models. Also add validation that the fields in `updated_fields` are efectively updated --- test_cases/test_api_non_workspaced_base.py | 15 ++++++++------- test_cases/test_api_workspaced_base.py | 20 +++++++++++--------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/test_cases/test_api_non_workspaced_base.py b/test_cases/test_api_non_workspaced_base.py index fdb603702b0..e6fd8a17645 100644 --- a/test_cases/test_api_non_workspaced_base.py +++ b/test_cases/test_api_non_workspaced_base.py @@ -92,25 +92,26 @@ def test_create_fails_with_existing(self, session, test_client): class UpdateTestsMixin: - def test_update_a_host(self, test_client): - host = self.workspace.hosts[0] + def test_update_an_object(self, test_client): res = test_client.put(self.url(self.first_object), data=self.factory.build_dict()) assert res.status_code == 200 assert self.model.query.count() == OBJECT_COUNT + for updated_field in self.update_fields: + assert res.json[updated_field] == getattr(self.first_object, + updated_field) def test_update_fails_with_existing(self, test_client, session): for unique_field in self.unique_fields: data = self.factory.build_dict() - data[unique_field] = getattr(self.first_object, unique_field) - res = test_client.put(self.url(self.workspace.hosts[1]), data=data) + data[unique_field] = getattr(self.objects[1], unique_field) + res = test_client.put(self.url(self.first_object), data=data) assert res.status_code == 400 assert self.model.query.count() == OBJECT_COUNT - def test_update_a_host_fails_with_empty_dict(self, test_client): + def test_update_an_object_fails_with_empty_dict(self, test_client): """To do this the user should use a PATCH request""" - host = self.workspace.hosts[0] - res = test_client.put(self.url(host), data={}) + res = test_client.put(self.url(self.first_object), data={}) assert res.status_code == 400 diff --git a/test_cases/test_api_workspaced_base.py b/test_cases/test_api_workspaced_base.py index 8fa9b4ce324..a73d9a8029c 100644 --- a/test_cases/test_api_workspaced_base.py +++ b/test_cases/test_api_workspaced_base.py @@ -22,8 +22,9 @@ class GenericAPITest: @pytest.fixture(autouse=True) def load_workspace_with_objects(self, database, session, workspace): - objects = self.factory.create_batch(OBJECT_COUNT, workspace=workspace) - self.first_object = objects[0] + self.objects = self.factory.create_batch( + OBJECT_COUNT, workspace=workspace) + self.first_object = self.objects[0] session.commit() assert workspace.id is not None self.workspace = workspace @@ -121,25 +122,26 @@ def test_create_with_existing_in_other_workspace(self, test_client, class UpdateTestsMixin: - def test_update_a_host(self, test_client): - host = self.workspace.hosts[0] + def test_update_an_object(self, test_client): res = test_client.put(self.url(self.first_object), data=self.factory.build_dict()) assert res.status_code == 200 assert self.model.query.count() == OBJECT_COUNT + for updated_field in self.update_fields: + assert res.json[updated_field] == getattr(self.first_object, + updated_field) def test_update_fails_with_existing(self, test_client, session): for unique_field in self.unique_fields: data = self.factory.build_dict() - data[unique_field] = getattr(self.first_object, unique_field) - res = test_client.put(self.url(self.workspace.hosts[1]), data=data) + data[unique_field] = getattr(self.objects[1], unique_field) + res = test_client.put(self.url(self.first_object), data=data) assert res.status_code == 400 assert self.model.query.count() == OBJECT_COUNT - def test_update_a_host_fails_with_empty_dict(self, test_client): + def test_update_an_object_fails_with_empty_dict(self, test_client): """To do this the user should use a PATCH request""" - host = self.workspace.hosts[0] - res = test_client.put(self.url(host), data={}) + res = test_client.put(self.url(self.first_object), data={}) assert res.status_code == 400 From f6fa24c8d38ec324350c0d458ba28b458d77d524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 13 Sep 2017 16:02:17 -0300 Subject: [PATCH 0209/1506] Add generic update API mixin and apply it to licenses API --- server/api/base.py | 26 ++++++++++++++++------ test_cases/test_api_non_workspaced_base.py | 2 +- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 75bc6409b20..cedc575926b 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -217,27 +217,39 @@ def _perform_create(self, obj, workspace_name): return super(CreateWorkspacedMixin, self)._perform_create(obj) -class UpdateWorkspacedMixin(object): - def put(self, workspace_name, object_id): +class UpdateMixin(object): + """Add PUT /// route""" + + def put(self, object_id, **kwargs): data = self._parse_data(self._get_schema_class()(strict=True), flask.request) - obj = self._get_object(object_id, workspace_name) + obj = self._get_object(object_id, **kwargs) self._update_object(obj, data) - updated = self._perform_update(workspace_name, object_id, obj) + updated = self._perform_update(object_id, obj, **kwargs) return self._dump(obj).data, 200 def _update_object(self, obj, data): for (key, value) in data.items(): setattr(obj, key, value) - def _perform_update(self, workspace_name, object_id, obj): + def _perform_update(self, object_id, obj): with db.session.no_autoflush: - obj.workspace = self._get_workspace(workspace_name) self._validate_uniqueness(obj, object_id) db.session.add(obj) db.session.commit() +class UpdateWorkspacedMixin(UpdateMixin): + """Add PUT // route""" + + def _perform_update(self, object_id, obj, workspace_name): + assert not db.session.new + with db.session.no_autoflush: + obj.workspace = self._get_workspace(workspace_name) + return super(UpdateWorkspacedMixin, self)._perform_update( + object_id, obj) + + class DeleteMixin(object): """Add DELETE // route""" def delete(self, object_id, **kwargs): @@ -256,7 +268,7 @@ class DeleteWorkspacedMixin(DeleteMixin): class ReadWriteView(CreateMixin, - #UpdateMixin, + UpdateMixin, DeleteMixin, ReadOnlyView): """A generic view with list, retrieve and create endpoints""" diff --git a/test_cases/test_api_non_workspaced_base.py b/test_cases/test_api_non_workspaced_base.py index e6fd8a17645..f7ea1b54676 100644 --- a/test_cases/test_api_non_workspaced_base.py +++ b/test_cases/test_api_non_workspaced_base.py @@ -134,7 +134,7 @@ def test_delete_non_existent_raises_404(self, test_client, class ReadWriteTestsMixin(ListTestsMixin, RetrieveTestsMixin, CreateTestsMixin, - # UpdateTestsMixin, + UpdateTestsMixin, DeleteTestsMixin, ): pass From 4f594c4d88b192a3e7131a2a7d419d207043639e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 13 Sep 2017 17:08:47 -0300 Subject: [PATCH 0210/1506] Check for deleted items on couchdb --- server/importer.py | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/server/importer.py b/server/importer.py index 7bf5eaff6d0..c01837e0a64 100644 --- a/server/importer.py +++ b/server/importer.py @@ -868,16 +868,30 @@ def run(self): self.import_workspace_into_database(workspace_name) - def get_objs(self, host, obj_type, level): + def get_objs(self, host, obj_type, level, workspace): if obj_type == 'Credential': obj_type = 'Cred' data = { "map": "function(doc) { if(doc.type == '%s' && doc._id.split('.').length == %d) emit(null, doc); }" % (obj_type, level) } - r = requests.post(host, json=data) + documents = requests.post(host, json=data).json() + for doc in documents['rows']: + doc['value']['deleted'] = False + doc_status_url = "http://{username}:{password}@{hostname}:{port}/{workspace_name}/_all_docs?keys=[\"{doc_id}\"]".format( + username=server.config.couchdb.user, + password=server.config.couchdb.password, + hostname=server.config.couchdb.host, + port=server.config.couchdb.port, + workspace_name=workspace.name, + doc_id=doc['value']['_id'] + ) + document_status = requests.get(doc_status_url).json() + assert len(document_status['rows']) == 1 + if any(map(lambda doc_hist: 'error' in doc_hist, document_status['rows'])): + doc['value']['deleted'] = True - return r.json() + return documents def verify_host_vulns_count_is_correct(self, couchdb_relational_map, couchdb_relational_map_by_type, workspace): hosts = session.query(Host).filter_by(workspace=workspace) @@ -897,7 +911,9 @@ def verify_host_vulns_count_is_correct(self, couchdb_relational_map, couchdb_rel for interface in interfaces: interface = interface['value'] vulns += get_children_from_couch(workspace, interface.get('_id'), 'Vulnerability') - assert len(vulns) == len(host.vulnerabilities) + # some vulns had the same name but different description. + # we check that we have the same vuln names now + assert len(set(map(lambda vuln: vuln['value'].get('name'), vulns))) == len(set(map(lambda vuln: vuln.name, host.vulnerabilities))) def verify_import_data(self, couchdb_relational_map, couchdb_relational_map_by_type, workspace): self.verify_host_vulns_count_is_correct(couchdb_relational_map, couchdb_relational_map_by_type, workspace) @@ -939,12 +955,12 @@ def import_workspace_into_database(self, workspace_name): session.commit() couch_url = "http://{username}:{password}@{hostname}:{port}/{workspace_name}/_temp_view?include_docs=true".format( - username=server.config.couchdb.user, - password=server.config.couchdb.password, - hostname=server.config.couchdb.host, - port=server.config.couchdb.port, - workspace_name=workspace_name - ) + username=server.config.couchdb.user, + password=server.config.couchdb.password, + hostname=server.config.couchdb.host, + port=server.config.couchdb.port, + workspace_name=workspace_name + ) # obj_types are tuples. the first value is the level on the tree # for the desired obj. @@ -953,8 +969,11 @@ def import_workspace_into_database(self, workspace_name): couchdb_relational_map_by_type = defaultdict(list) for level, obj_type in obj_types: obj_importer = faraday_importer.get_importer_from_document(obj_type)() - objs_dict = self.get_objs(couch_url, obj_type, level) + objs_dict = self.get_objs(couch_url, obj_type, level, workspace) for raw_obj in tqdm(objs_dict.get('rows', [])): + if raw_obj['value']['deleted']: + logger.warn('Skipping deleted object from couchdb {0}'.format(raw_obj['value']['_id'])) + continue # we use no_autoflush since some queries triggers flush and some relationship are missing in the middle with session.no_autoflush: raw_obj = raw_obj['value'] From 4fad47479aaf0883ce4245262b67e3788cea5f89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 13 Sep 2017 17:10:06 -0300 Subject: [PATCH 0211/1506] Change workspaced views URL From /v2/// to /v2/ws///. This way makes it more compatible with the old version, and allows accessing workspaces named "licences", "users", etc. --- server/api/base.py | 2 +- test_cases/test_api_workspaced_base.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index cedc575926b..36d57c675c8 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -98,7 +98,7 @@ class GenericWorkspacedView(GenericView): passed in the URL""" # Default attributes - route_prefix = '/v2//' + route_prefix = '/v2/ws//' base_args = ['workspace_name'] # Required to prevent double usage of unique_fields = [] # Fields unique together with workspace_id diff --git a/test_cases/test_api_workspaced_base.py b/test_cases/test_api_workspaced_base.py index a73d9a8029c..d7d757ffd4f 100644 --- a/test_cases/test_api_workspaced_base.py +++ b/test_cases/test_api_workspaced_base.py @@ -6,7 +6,7 @@ from sqlalchemy.orm.util import was_deleted from server.models import db, Workspace -API_PREFIX = '/v2/' +API_PREFIX = '/v2/ws/' OBJECT_COUNT = 5 From d77b67aef1d767236e5731fc6d59d6abe2a9c531 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 13 Sep 2017 18:40:21 -0300 Subject: [PATCH 0212/1506] Use description to check if vuln already exists --- server/importer.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/importer.py b/server/importer.py index c01837e0a64..ed6d134baef 100644 --- a/server/importer.py +++ b/server/importer.py @@ -371,6 +371,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation session, VulnerabilityWeb, name=document.get('name'), + description=document.get('desc'), severity=severity, service_id=parent.id, method=method, @@ -384,6 +385,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation 'name': document.get('name'), 'severity': severity, 'workspace': workspace, + 'description': document.get('desc') } if type(parent) == Host: vuln_params.update({'host_id': parent.id}) @@ -394,7 +396,6 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation Vulnerability, **vuln_params ) - vulnerability.description = document.get('desc'), vulnerability.confirmed = document.get('confirmed', False) or False vulnerability.data = document.get('data') vulnerability.easeofresolution = document.get('easeofresolution') @@ -911,9 +912,7 @@ def verify_host_vulns_count_is_correct(self, couchdb_relational_map, couchdb_rel for interface in interfaces: interface = interface['value'] vulns += get_children_from_couch(workspace, interface.get('_id'), 'Vulnerability') - # some vulns had the same name but different description. - # we check that we have the same vuln names now - assert len(set(map(lambda vuln: vuln['value'].get('name'), vulns))) == len(set(map(lambda vuln: vuln.name, host.vulnerabilities))) + assert len(vulns) == len(host.vulnerabilities) def verify_import_data(self, couchdb_relational_map, couchdb_relational_map_by_type, workspace): self.verify_host_vulns_count_is_correct(couchdb_relational_map, couchdb_relational_map_by_type, workspace) From add50db56fe57ce732e6ef5f65bb7bea5065502f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 13 Sep 2017 19:04:43 -0300 Subject: [PATCH 0213/1506] Change assert to verify without duplicates The assert could return false positive but we found some old couchdb data that has duplicate vulns. --- server/importer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index ed6d134baef..483f8357af1 100644 --- a/server/importer.py +++ b/server/importer.py @@ -912,7 +912,8 @@ def verify_host_vulns_count_is_correct(self, couchdb_relational_map, couchdb_rel for interface in interfaces: interface = interface['value'] vulns += get_children_from_couch(workspace, interface.get('_id'), 'Vulnerability') - assert len(vulns) == len(host.vulnerabilities) + + assert len(set(map(lambda vuln: vuln['value'].get('name'), vulns))) == len(set(map(lambda vuln: vuln.name, host.vulnerabilities))) def verify_import_data(self, couchdb_relational_map, couchdb_relational_map_by_type, workspace): self.verify_host_vulns_count_is_correct(couchdb_relational_map, couchdb_relational_map_by_type, workspace) From 3768e8af457e22c9cb119754805574de1bfce1cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 14 Sep 2017 11:56:09 -0300 Subject: [PATCH 0214/1506] Don't use flask classful from GitHub It breaks the dependency checker. The API endpoints won't work properly until a new version of flask classful is released, in a few weeks. Meawhile, you can manually execute pip install https://github.com/teracyhq/flask-classful/archive/develop.zip --- requirements_server.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_server.txt b/requirements_server.txt index 371a6f8d32d..7c9bacffd8a 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -12,6 +12,6 @@ Flask-Security==3.0.0 bcrypt Flask-SQLAlchemy==2.2 Flask-Script==2.0.5 -https://github.com/teracyhq/flask-classful/archive/develop.zip +flask-classful marshmallow webargs From 235a6b282d393878a810de6a1494b6eddad08c19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 14 Sep 2017 11:58:42 -0300 Subject: [PATCH 0215/1506] Add plain text password scheme It is used in the createsuperuser.py script. It should be deleted soon --- server/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/app.py b/server/app.py index f70f6df7ea4..c9bc560bfe8 100644 --- a/server/app.py +++ b/server/app.py @@ -34,6 +34,7 @@ def create_app(db_connection_string=None, testing=None): # 'pbkdf2_sha512', # 'sha256_crypt', # 'sha512_crypt', + 'plaintext', # TODO: remove it ] if testing: app.config['TESTING'] = testing From 9cee36d234d07de5a6f39bfab88191e3410c7fc3 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 14 Sep 2017 14:15:19 -0300 Subject: [PATCH 0216/1506] add more cases for vuln template exploitation --- server/importer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index 483f8357af1..464ac9c292d 100644 --- a/server/importer.py +++ b/server/importer.py @@ -775,15 +775,19 @@ def run(self): mapped_exploitation = { 'critical': 'critical', 'med': 'medium', + 'high to very high': 'high', 'high': 'high', 'low': 'low', 'info': 'informational', + 'unknown': 'unclassified', 'unclassified': 'unclassified', } + if document.get('exploitation') not in mapped_exploitation.values(): + logger.warn('Vuln template exploitation {0} not found. using unclassified'.format(document.get('exploitation'))) vuln_template, created = get_or_create(session, VulnerabilityTemplate, name=document.get('name'), - severity=mapped_exploitation[document.get('exploitation')], + severity=mapped_exploitation[document.get('exploitation', 'unclassified').lower()], description=document.get('description')) vuln_template.resolution = document.get('resolution') for ref_doc in document['references']: From 94110fde8ce8b43023b657945f9f4bd424411083 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 14 Sep 2017 14:17:05 -0300 Subject: [PATCH 0217/1506] filter deleted docs --- server/importer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/importer.py b/server/importer.py index 464ac9c292d..fa6f58b785d 100644 --- a/server/importer.py +++ b/server/importer.py @@ -877,11 +877,11 @@ def get_objs(self, host, obj_type, level, workspace): if obj_type == 'Credential': obj_type = 'Cred' data = { - "map": "function(doc) { if(doc.type == '%s' && doc._id.split('.').length == %d) emit(null, doc); }" % (obj_type, level) + "map": "function(doc) { if(doc.type == '%s' && doc._id.split('.').length == %d && !doc._deleted) emit(null, doc); }" % (obj_type, level) } documents = requests.post(host, json=data).json() - for doc in documents['rows']: + for doc in documents.get('rows', []): doc['value']['deleted'] = False doc_status_url = "http://{username}:{password}@{hostname}:{port}/{workspace_name}/_all_docs?keys=[\"{doc_id}\"]".format( username=server.config.couchdb.user, From e3f864bc2cc8c4406850e537897f2efa972e864c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 14 Sep 2017 14:18:14 -0300 Subject: [PATCH 0218/1506] Remove manual deleted check from couchdb --- server/importer.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/server/importer.py b/server/importer.py index fa6f58b785d..d6ca63a8092 100644 --- a/server/importer.py +++ b/server/importer.py @@ -881,21 +881,6 @@ def get_objs(self, host, obj_type, level, workspace): } documents = requests.post(host, json=data).json() - for doc in documents.get('rows', []): - doc['value']['deleted'] = False - doc_status_url = "http://{username}:{password}@{hostname}:{port}/{workspace_name}/_all_docs?keys=[\"{doc_id}\"]".format( - username=server.config.couchdb.user, - password=server.config.couchdb.password, - hostname=server.config.couchdb.host, - port=server.config.couchdb.port, - workspace_name=workspace.name, - doc_id=doc['value']['_id'] - ) - document_status = requests.get(doc_status_url).json() - assert len(document_status['rows']) == 1 - if any(map(lambda doc_hist: 'error' in doc_hist, document_status['rows'])): - doc['value']['deleted'] = True - return documents def verify_host_vulns_count_is_correct(self, couchdb_relational_map, couchdb_relational_map_by_type, workspace): @@ -975,9 +960,6 @@ def import_workspace_into_database(self, workspace_name): obj_importer = faraday_importer.get_importer_from_document(obj_type)() objs_dict = self.get_objs(couch_url, obj_type, level, workspace) for raw_obj in tqdm(objs_dict.get('rows', [])): - if raw_obj['value']['deleted']: - logger.warn('Skipping deleted object from couchdb {0}'.format(raw_obj['value']['_id'])) - continue # we use no_autoflush since some queries triggers flush and some relationship are missing in the middle with session.no_autoflush: raw_obj = raw_obj['value'] From 255e2f117e0c357409b8f84b2b76a23c05303c5f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 14 Sep 2017 14:18:33 -0300 Subject: [PATCH 0219/1506] Add missing import --- server/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/models.py b/server/models.py index 30381cc1170..b545bbde972 100644 --- a/server/models.py +++ b/server/models.py @@ -18,6 +18,7 @@ Text, UniqueConstraint, ) +from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import relationship, backref from sqlalchemy.schema import DDL from flask_sqlalchemy import SQLAlchemy From 4ebab020f02f759bea8a21e8aef81f548e4d4126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 14 Sep 2017 15:24:32 -0300 Subject: [PATCH 0220/1506] Fix bug making text/html content-type for json responses in API I was returning a Marshmallow structure instead of the current json data and this caused some problems. --- server/api/base.py | 6 +++--- server/api/modules/hosts.py | 2 +- test_cases/conftest.py | 9 +++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 36d57c675c8..36399069f8e 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -64,7 +64,7 @@ def _get_object(self, object_id, **kwargs): return obj def _dump(self, obj, **kwargs): - return self._get_schema_class()(**kwargs).dump(obj) + return self._get_schema_class()(**kwargs).dump(obj).data def _parse_data(self, schema, request, *args, **kwargs): return FlaskParser().parse(schema, request, locations=('json',), @@ -195,7 +195,7 @@ def post(self, **kwargs): flask.request) obj = self.model_class(**data) created = self._perform_create(obj, **kwargs) - return self._dump(created).data, 201 + return self._dump(created), 201 def _perform_create(self, obj): # assert not db.session.new @@ -226,7 +226,7 @@ def put(self, object_id, **kwargs): obj = self._get_object(object_id, **kwargs) self._update_object(obj, data) updated = self._perform_update(object_id, obj, **kwargs) - return self._dump(obj).data, 200 + return self._dump(obj), 200 def _update_object(self, obj, data): for (key, value) in data.items(): diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 598d169a2b7..c5001966f8c 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -44,7 +44,7 @@ class HostsView(ReadWriteWorkspacedView): @route('//services/') def service_list(self, workspace_name, host_id): services = self._get_object(host_id, workspace_name).services - return ServiceSchema(many=True).dump(services) + return ServiceSchema(many=True).dump(services).data def _get_base_query(self, workspace_name): """Get services_count in one query and not deferred, that doe diff --git a/test_cases/conftest.py b/test_cases/conftest.py index c7fc136d708..4d74162d565 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -39,10 +39,11 @@ def open(self, *args, **kwargs): ] ret = super(CustomClient, self).open(*args, **kwargs) - try: - ret.json = json.loads(ret.data) - except ValueError: - ret.json = None + if ret.headers.get('content-type') == 'application/json': + try: + ret.json = json.loads(ret.data) + except ValueError: + ret.json = None return ret From 1132ddd934a190f5811d92f152ba27153a2f8a20 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 14 Sep 2017 15:42:13 -0300 Subject: [PATCH 0221/1506] Fix model to filter byu polymorphic identity --- server/models.py | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/server/models.py b/server/models.py index 30381cc1170..1a3661d229a 100644 --- a/server/models.py +++ b/server/models.py @@ -260,6 +260,7 @@ class VulnerabilityGeneric(VulnerabilityABC): class Vulnerability(VulnerabilityGeneric): + __tablename__ = None host_id = Column(Integer, ForeignKey(Host.id), index=True) host = relationship( 'Host', @@ -267,15 +268,13 @@ class Vulnerability(VulnerabilityGeneric): foreign_keys=[host_id], ) - service_id = Column(Integer, ForeignKey(Service.id)) - service = relationship( - 'Service', - backref='vulnerabilities', - ) + @declared_attr + def service_id(cls): + return VulnerabilityGeneric.__table__.c.get('service_id', Column(Integer, db.ForeignKey('vulnerability.service_id'))) - __table_args__ = { - 'extend_existing': True - } + @declared_attr + def service(cls): + return relationship('VulnerabilityGeneric') __mapper_args__ = { 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[0] @@ -283,6 +282,7 @@ class Vulnerability(VulnerabilityGeneric): class VulnerabilityWeb(VulnerabilityGeneric): + __tablename__ = None method = Column(String(50), nullable=True) parameters = Column(String(500), nullable=True) parameter_name = Column(String(250), nullable=True) @@ -292,15 +292,13 @@ class VulnerabilityWeb(VulnerabilityGeneric): response = Column(Text(), nullable=True) website = Column(String(250), nullable=True) - service_id = Column(Integer, ForeignKey(Service.id)) - service = relationship( - 'Service', - backref='vulnerabilities_web', - ) + @declared_attr + def service_id(cls): + return VulnerabilityGeneric.__table__.c.get('service_id', Column(Integer, db.ForeignKey('vulnerability.service_id'))) - __table_args__ = { - 'extend_existing': True - } + @declared_attr + def service(cls): + return relationship('VulnerabilityGeneric') __mapper_args__ = { 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[1] @@ -308,6 +306,7 @@ class VulnerabilityWeb(VulnerabilityGeneric): class VulnerabilityCode(VulnerabilityGeneric): + __tablename__ = None line = Column(Integer, nullable=True) source_code_id = Column(Integer, ForeignKey(SourceCode.id), index=True) @@ -317,10 +316,6 @@ class VulnerabilityCode(VulnerabilityGeneric): foreign_keys=[source_code_id] ) - __table_args__ = { - 'extend_existing': True - } - __mapper_args__ = { 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[2] } From 7cf3dad1d7f2f0345185f4c29e1112e1d793b89a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 14 Sep 2017 16:14:33 -0300 Subject: [PATCH 0222/1506] Allow List APIs to specify how a list of objects is rendered Via _envelope_list method --- server/api/base.py | 9 +++++++-- test_cases/test_api_non_workspaced_various.py | 20 +++++++++++++++++-- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 36399069f8e..7a97a371769 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -147,9 +147,14 @@ def _validate_uniqueness(self, obj, object_id=None): class ListMixin(object): """Add GET / route""" + def _envelope_list(self, objects): + """Override this method to define how a list of objects is + rendered""" + return objects + def index(self, **kwargs): - return self._dump(self._get_base_query(**kwargs).all(), - many=True) + objects = self._get_base_query(**kwargs) + return self._envelope_list(self._dump(objects, many=True)) class ListWorkspacedMixin(ListMixin): diff --git a/test_cases/test_api_non_workspaced_various.py b/test_cases/test_api_non_workspaced_various.py index 37471b925e0..03563a487c4 100644 --- a/test_cases/test_api_non_workspaced_various.py +++ b/test_cases/test_api_non_workspaced_various.py @@ -1,11 +1,19 @@ #-*- coding: utf8 -*- - """Tests for many API endpoints that do not depend on workspace_name""" import pytest from test_cases import factories -from test_api_non_workspaced_base import ReadWriteAPITests +from test_api_non_workspaced_base import ReadWriteAPITests, API_PREFIX from server.models import License +from server.api.modules.licenses import LicenseView + +class LicenseEnvelopedView(LicenseView): + """A custom view to test that enveloping on generic views work ok""" + route_base = "test_envelope_list" + + def _envelope_list(self, objects): + return {"object_list": objects} + class TestLicensesAPI(ReadWriteAPITests): model = License @@ -14,4 +22,12 @@ class TestLicensesAPI(ReadWriteAPITests): # unique_fields = ['ip'] # update_fields = ['ip', 'description', 'os'] + def test_envelope_list(self, test_client, app): + LicenseEnvelopedView.register(app) + print app.url_map + original_res = test_client.get(self.url()) + assert original_res.status_code == 200 + new_res = test_client.get(API_PREFIX + 'test_envelope_list/') + assert new_res.status_code == 200 + assert new_res.json == {"object_list": original_res.json} From d02e5c7076f0329e60ad578de3d2dfa8024a7612 Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 14 Sep 2017 16:40:03 -0300 Subject: [PATCH 0223/1506] Fix typo, sort of --- server/importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index d6ca63a8092..e7d6dd34cff 100644 --- a/server/importer.py +++ b/server/importer.py @@ -325,7 +325,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation service.banner = document.get('banner') service.protocol = document.get('protocol') if not document.get('status'): - logger.warning('Service {0} with empty status. Using open as status'.format(document['_id'])) + logger.warning('Service {0} with empty status. Using \'open\' as status'.format(document['_id'])) document['status'] = 'open' status_mapper = { 'open': 'open', From d187dc5a68390cb1c476bc093c4f53a941ebe126 Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 14 Sep 2017 16:40:22 -0300 Subject: [PATCH 0224/1506] Fix bug for vuln templates references The references field in vuln templates is either an empty list or a comma separated string with multiple references. --- server/importer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index e7d6dd34cff..9124e90c36a 100644 --- a/server/importer.py +++ b/server/importer.py @@ -790,7 +790,8 @@ def run(self): severity=mapped_exploitation[document.get('exploitation', 'unclassified').lower()], description=document.get('description')) vuln_template.resolution = document.get('resolution') - for ref_doc in document['references']: + references = document['references'] if isinstance(document['references'], list) else [x.strip() for x in document['references'].split(',')] + for ref_doc in references: get_or_create(session, ReferenceTemplate, vulnerability=vuln_template, From 809a74bd92db3c584c440f0f2c611885401ae467 Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 14 Sep 2017 17:37:45 -0300 Subject: [PATCH 0225/1506] Fix Vuln templates import Now the importer normalizes the vuln template 'exploitation' to match one of the valid 'severity' values. --- server/importer.py | 57 +++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/server/importer.py b/server/importer.py index 9124e90c36a..b1c63c5807f 100644 --- a/server/importer.py +++ b/server/importer.py @@ -6,7 +6,7 @@ import sys import json import datetime -from collections import defaultdict +from collections import defaultdict, OrderedDict from binascii import unhexlify import requests @@ -56,6 +56,15 @@ logger = server.utils.logger.get_logger(__name__) session = db.session +MAPPED_VULN_SEVERITY = OrderedDict([ + ('critical', 'critical'), + ('high', 'high'), + ('med', 'medium'), + ('low', 'low'), + ('info', 'informational'), + ('unclassified', 'unclassified'), +]) + OBJ_TYPES = [ (1, 'Host'), (1, 'EntityMetadata'), @@ -348,14 +357,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation if not couch_parent_id: couch_parent_id = '.'.join(document['_id'].split('.')[:-1]) parent_ids = couchdb_relational_map[couch_parent_id] - mapped_severity = { - 'med': 'medium', - 'critical': 'critical', - 'high': 'high', - 'low': 'low', - 'info': 'informational', - 'unclassified': 'unclassified', - } + mapped_severity = MAPPED_VULN_SEVERITY severity = mapped_severity[document.get('severity')] for parent_id in parent_ids: if level == 2: @@ -769,25 +771,20 @@ def run(self): port=server.config.couchdb.port, path='cwe/_all_docs?include_docs=true' ) - cwes = requests.get(cwe_url).json()['rows'] + + try: + cwes = requests.get(cwe_url).json()['rows'] + except requests.exceptions.RequestException as e: + logger.warn(e) + return + for cwe in cwes: document = cwe['doc'] - mapped_exploitation = { - 'critical': 'critical', - 'med': 'medium', - 'high to very high': 'high', - 'high': 'high', - 'low': 'low', - 'info': 'informational', - 'unknown': 'unclassified', - 'unclassified': 'unclassified', - } - if document.get('exploitation') not in mapped_exploitation.values(): - logger.warn('Vuln template exploitation {0} not found. using unclassified'.format(document.get('exploitation'))) + new_severity = self.get_severity(document) vuln_template, created = get_or_create(session, VulnerabilityTemplate, name=document.get('name'), - severity=mapped_exploitation[document.get('exploitation', 'unclassified').lower()], + severity=new_severity, description=document.get('description')) vuln_template.resolution = document.get('resolution') references = document['references'] if isinstance(document['references'], list) else [x.strip() for x in document['references'].split(',')] @@ -797,6 +794,20 @@ def run(self): vulnerability=vuln_template, name=ref_doc) + def get_severity(self, document): + default = 'unclassified' + + mapped_exploitation = MAPPED_VULN_SEVERITY + + for key in mapped_exploitation.keys(): + if key in document.get('exploitation').lower(): + return mapped_exploitation[key] + + logger.warn( + 'Vuln template exploitation \'{0}\' not found. Using \'{1}\' instead.'.format(document.get('exploitation'), default) + ) + + return default class ImportLicense(FlaskScriptCommand): From 32e48d9b1535bc3688f7755c1a487e66304aa629 Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 14 Sep 2017 18:33:03 -0300 Subject: [PATCH 0226/1506] Fix vuln template duplication The vulnerability templates are queried by the user from the vulnerability creation dialog in the web UI. The only field the user has to pick one template over another is the name. Therefore, having more than one vuln template with the same name doesn't make sense. The new model doesn't allow duplication, but the old one did. The importer is now able to detect duplicated template names and appends "(n)" to the name with n being the number of repetition. --- server/importer.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/server/importer.py b/server/importer.py index b1c63c5807f..f84770077fb 100644 --- a/server/importer.py +++ b/server/importer.py @@ -6,7 +6,11 @@ import sys import json import datetime -from collections import defaultdict, OrderedDict +from collections import ( + Counter, + defaultdict, + OrderedDict +) from binascii import unhexlify import requests @@ -763,6 +767,9 @@ def run(self): class ImportVulnerabilityTemplates(FlaskScriptCommand): + def __init__(self): + self.names = Counter() + def run(self): cwe_url = "http://{username}:{password}@{hostname}:{port}/{path}".format( username=server.config.couchdb.user, @@ -781,12 +788,17 @@ def run(self): for cwe in cwes: document = cwe['doc'] new_severity = self.get_severity(document) + + new_name = self.get_name(document) + vuln_template, created = get_or_create(session, VulnerabilityTemplate, - name=document.get('name'), - severity=new_severity, - description=document.get('description')) + name=new_name) + + vuln_template.description = document.get('description') vuln_template.resolution = document.get('resolution') + vuln_template.severity = new_severity + references = document['references'] if isinstance(document['references'], list) else [x.strip() for x in document['references'].split(',')] for ref_doc in references: get_or_create(session, @@ -794,6 +806,19 @@ def run(self): vulnerability=vuln_template, name=ref_doc) + def get_name(self, document): + doc_name = document.get('name') + count = self.names[doc_name] + + if count > 0: + name = u'{0} ({1})'.format(doc_name, count) + else: + name = doc_name + + self.names[doc_name] += 1 + + return name + def get_severity(self, document): default = 'unclassified' From 14e9beb01bdf9cae17804f3aee32b0c6e69cdadb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 14 Sep 2017 19:23:14 -0300 Subject: [PATCH 0227/1506] First attempt on API pagination --- server/api/base.py | 21 +++- server/api/modules/hosts.py | 5 +- test_cases/test_api_hosts.py | 10 +- test_cases/test_api_non_workspaced_various.py | 2 +- test_cases/test_api_pagination.py | 107 ++++++++++++++++++ test_cases/test_api_workspaced_base.py | 9 ++ 6 files changed, 146 insertions(+), 8 deletions(-) create mode 100644 test_cases/test_api_pagination.py diff --git a/server/api/base.py b/server/api/base.py index 7a97a371769..d06bd7688dd 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -147,14 +147,29 @@ def _validate_uniqueness(self, obj, object_id=None): class ListMixin(object): """Add GET / route""" - def _envelope_list(self, objects): + def _envelope_list(self, objects, pagination_metadata=None): """Override this method to define how a list of objects is rendered""" return objects + def _paginate(self, query): + return query, None + def index(self, **kwargs): - objects = self._get_base_query(**kwargs) - return self._envelope_list(self._dump(objects, many=True)) + objects, pagination_metadata = self._paginate( + self._get_base_query(**kwargs)) + return self._envelope_list(self._dump(objects, many=True), + pagination_metadata) + + +class PaginatedMixin(object): + """Add pagination for list route""" + + def _paginate(self, query): + if 'per_page' in flask.request.args: + pagination_metadata = query.paginate() + return pagination_metadata.items, pagination_metadata + return super(PaginatedMixin, self)._paginate(query) class ListWorkspacedMixin(ListMixin): diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index c5001966f8c..4a7bd374c05 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -12,7 +12,7 @@ from server.utils.web import gzipped, validate_workspace,\ get_integer_parameter, filter_request_args from server.dao.host import HostDAO -from server.api.base import ReadWriteWorkspacedView +from server.api.base import ReadWriteWorkspacedView, PaginatedMixin from server.models import Host host_api = Blueprint('host_api', __name__) @@ -35,7 +35,8 @@ class ServiceSchema(Schema): status = fields.String(required=True) -class HostsView(ReadWriteWorkspacedView): +class HostsView(PaginatedMixin, + ReadWriteWorkspacedView): route_base = 'hosts' model_class = Host schema_class = HostSchema diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index c94a1cf79a0..c1e7feabb2c 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -2,8 +2,13 @@ from sqlalchemy.orm.util import was_deleted from test_cases import factories -from test_api_workspaced_base import API_PREFIX, ReadWriteAPITests +from test_api_workspaced_base import ( + API_PREFIX, + ReadWriteAPITests, + PaginationTestsMixin, +) from server.models import db, Host +from server.api.modules.hosts import HostsView HOSTS_COUNT = 5 SERVICE_COUNT = [10, 5] # 10 services to the first host, 5 to the second @@ -201,10 +206,11 @@ def test_index_shows_service_count(self, test_client, host_services): assert host['service_count'] == len(ids_map[host['id']]) -class TestHostAPIGeneric(ReadWriteAPITests): +class TestHostAPIGeneric(ReadWriteAPITests, PaginationTestsMixin): model = Host factory = factories.HostFactory api_endpoint = 'hosts' unique_fields = ['ip'] update_fields = ['ip', 'description', 'os'] + view_class = HostsView diff --git a/test_cases/test_api_non_workspaced_various.py b/test_cases/test_api_non_workspaced_various.py index 03563a487c4..30e299f4edb 100644 --- a/test_cases/test_api_non_workspaced_various.py +++ b/test_cases/test_api_non_workspaced_various.py @@ -11,7 +11,7 @@ class LicenseEnvelopedView(LicenseView): """A custom view to test that enveloping on generic views work ok""" route_base = "test_envelope_list" - def _envelope_list(self, objects): + def _envelope_list(self, objects, pagination_metadata=None): return {"object_list": objects} diff --git a/test_cases/test_api_pagination.py b/test_cases/test_api_pagination.py new file mode 100644 index 00000000000..134e70b13ee --- /dev/null +++ b/test_cases/test_api_pagination.py @@ -0,0 +1,107 @@ +#-*- coding: utf8 -*- + +"""Generic test mixins for APIs with pagination enabled when listing""" + +import pytest +from urllib import urlencode + +def with_0_and_n_objects(n=10): + return pytest.mark.parametrize('object_count', [0, n]) + +class PaginationTestsMixin: + per_page_parameter_name = 'per_page' + page_number_parameter_name = 'page' + view_class = None # Must be overriden + + @pytest.fixture + def delete_previously_created_objects(self, session): + from server.models import Host + for obj in self.objects: + session.delete(obj) + session.commit() + + @pytest.fixture + def custom_envelope(self, monkeypatch): + def _envelope_list(self, objects, pagination_metadata=None): + return {"data": objects} + monkeypatch.setattr(self.view_class, '_envelope_list', _envelope_list) + + @pytest.fixture + def pagination_test_logic(self, delete_previously_created_objects, + custom_envelope): + # Load this two fixtures + pass + + def create_many_objects(self, session, n): + objects = self.factory.create_batch(n) + session.commit() + return objects + + def page_url(self, page_number=None, per_page=None): + parameters = {} + if page_number is not None: + parameters[self.page_number_parameter_name] = page_number + if per_page is not None: + parameters[self.per_page_parameter_name] = per_page + return self.url() + '?' + urlencode(parameters) + + @pytest.mark.parametrize("page_number", [None, 1, 2]) + @pytest.mark.usefixtures('pagination_test_logic') + @pytest.mark.pagination + def test_returns_all_with_no_per_page(self, test_client, session, + page_number): + self.create_many_objects(session, 100) + res = test_client.get(self.page_url(page_number, + per_page=None)) + assert res.status_code == 200 + assert len(res.json['data']) == 100 + + @with_0_and_n_objects() + @pytest.mark.usefixtures('pagination_test_logic') + @pytest.mark.pagination + @pytest.mark.skip + def test_does_not_allow_negative_per_page(self, session, test_client, + object_count): + self.create_many_objects(session, object_count) + res = test_client.get(self.page_url(1, -1)) + assert res.status_code == 404 + + @with_0_and_n_objects() + @pytest.mark.usefixtures('pagination_test_logic') + @pytest.mark.pagination + @pytest.mark.skip + def test_does_not_allow_negative_page_number(self, session, test_client, + object_count): + self.create_many_objects(session, object_count) + res = test_client.get(self.page_url(-1, 10)) + assert res.status_code == 404 + + @pytest.mark.usefixtures('pagination_test_logic') + @pytest.mark.pagination + def test_pages_have_different_elements(self, session, test_client): + """Test correct page size, correct IDs and that there are + no duplicate items in different pages""" + ids = {getattr(obj, self.pk_field) + for obj in self.create_many_objects(session, 95)} + for page_number in range(1, 11): + res = test_client.get(self.page_url(page_number, 10)) + assert res.status_code == 200 + new_ids = {obj.get(self.pk_field) for obj in res.json['data']} + assert len(new_ids) == (5 if page_number == 10 else 10) + assert new_ids.issubset(ids) + ids.difference_update(new_ids) # Remove the new ids + assert not ids + + @pytest.mark.usefixtures('pagination_test_logic') + @pytest.mark.pagination + def test_404_on_page_with_no_elements(self, session, test_client): + self.create_many_objects(session, 5) + res = test_client.get(self.page_url(2, 5)) + assert res.status_code == 404 + + @pytest.mark.usefixtures('pagination_test_logic') + @pytest.mark.pagination + def test_succeed_on_first_page_with_no_elements(self, test_client): + res = test_client.get(self.page_url(1, 5)) + assert res.status_code == 200 + assert len(res.json['data']) == 0 diff --git a/test_cases/test_api_workspaced_base.py b/test_cases/test_api_workspaced_base.py index d7d757ffd4f..aaad56805f8 100644 --- a/test_cases/test_api_workspaced_base.py +++ b/test_cases/test_api_workspaced_base.py @@ -5,6 +5,8 @@ import pytest from sqlalchemy.orm.util import was_deleted from server.models import db, Workspace +from test_api_pagination import PaginationTestsMixin as \ + OriginalPaginationTestsMixin API_PREFIX = '/v2/ws/' OBJECT_COUNT = 5 @@ -162,6 +164,13 @@ def test_delete_from_other_workspace_fails(self, test_client, assert self.model.query.count() == OBJECT_COUNT +class PaginationTestsMixin(OriginalPaginationTestsMixin): + def create_many_objects(self, session, n): + objects = self.factory.create_batch(n, workspace=self.workspace) + session.commit() + return objects + + class ReadWriteTestsMixin(ListTestsMixin, RetrieveTestsMixin, CreateTestsMixin, From 93c8083759c5cbe2282cae28d1309cc984a5b4b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 15 Sep 2017 15:20:35 -0300 Subject: [PATCH 0228/1506] Unskip one pagination test, and add skip reason for the other --- test_cases/test_api_pagination.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test_cases/test_api_pagination.py b/test_cases/test_api_pagination.py index 134e70b13ee..9f614d3a7a9 100644 --- a/test_cases/test_api_pagination.py +++ b/test_cases/test_api_pagination.py @@ -56,10 +56,10 @@ def test_returns_all_with_no_per_page(self, test_client, session, assert res.status_code == 200 assert len(res.json['data']) == 100 + @pytest.mark.skip("TODO: Fix for sqlite and postgres") @with_0_and_n_objects() @pytest.mark.usefixtures('pagination_test_logic') @pytest.mark.pagination - @pytest.mark.skip def test_does_not_allow_negative_per_page(self, session, test_client, object_count): self.create_many_objects(session, object_count) @@ -69,7 +69,6 @@ def test_does_not_allow_negative_per_page(self, session, test_client, @with_0_and_n_objects() @pytest.mark.usefixtures('pagination_test_logic') @pytest.mark.pagination - @pytest.mark.skip def test_does_not_allow_negative_page_number(self, session, test_client, object_count): self.create_many_objects(session, object_count) From db2d0074b4d572d0545805785d6bad58feeeb96d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 15 Sep 2017 15:36:31 -0300 Subject: [PATCH 0229/1506] Customizable pagination parameter names Also remove unused import in test_api_pagination --- server/api/base.py | 19 +++++++++++++++++-- test_cases/test_api_pagination.py | 8 +++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index d06bd7688dd..9167be66b6f 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -164,10 +164,25 @@ def index(self, **kwargs): class PaginatedMixin(object): """Add pagination for list route""" + per_page_parameter_name = 'page_size' + page_number_parameter_name = 'page' def _paginate(self, query): - if 'per_page' in flask.request.args: - pagination_metadata = query.paginate() + if self.per_page_parameter_name in flask.request.args: + + try: + page = int(flask.request.args.get( + self.page_number_parameter_name, 1)) + except (TypeError, ValueError): + flask.abort(404, 'Invalid page number') + + try: + per_page = int(flask.request.args[ + self.per_page_parameter_name]) + except (TypeError, ValueError): + flask.abort(404, 'Invalid per_page value') + + pagination_metadata = query.paginate(page=page, per_page=per_page) return pagination_metadata.items, pagination_metadata return super(PaginatedMixin, self)._paginate(query) diff --git a/test_cases/test_api_pagination.py b/test_cases/test_api_pagination.py index 9f614d3a7a9..54c03a3ed97 100644 --- a/test_cases/test_api_pagination.py +++ b/test_cases/test_api_pagination.py @@ -9,13 +9,10 @@ def with_0_and_n_objects(n=10): return pytest.mark.parametrize('object_count', [0, n]) class PaginationTestsMixin: - per_page_parameter_name = 'per_page' - page_number_parameter_name = 'page' view_class = None # Must be overriden @pytest.fixture def delete_previously_created_objects(self, session): - from server.models import Host for obj in self.objects: session.delete(obj) session.commit() @@ -40,9 +37,10 @@ def create_many_objects(self, session, n): def page_url(self, page_number=None, per_page=None): parameters = {} if page_number is not None: - parameters[self.page_number_parameter_name] = page_number + parameters[ + self.view_class.page_number_parameter_name] = page_number if per_page is not None: - parameters[self.per_page_parameter_name] = per_page + parameters[self.view_class.per_page_parameter_name] = per_page return self.url() + '?' + urlencode(parameters) @pytest.mark.parametrize("page_number", [None, 1, 2]) From 87a521826bbc01ffd657caacc3ba63f987a6ba0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 18 Sep 2017 13:06:49 -0300 Subject: [PATCH 0230/1506] Automatic marshmallow schema generation Perform field instrospection based on the SQLAlchemy model fields. Code based on marshmallow_sqlalchemy --- requirements_server.txt | 1 + server/api/base.py | 13 +++++++++++++ server/api/modules/hosts.py | 32 ++++++++++++++++---------------- server/api/modules/licenses.py | 14 +++++++------- 4 files changed, 37 insertions(+), 23 deletions(-) diff --git a/requirements_server.txt b/requirements_server.txt index 7c9bacffd8a..31e614951be 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -15,3 +15,4 @@ Flask-Script==2.0.5 flask-classful marshmallow webargs +marshmallow-sqlalchemy diff --git a/server/api/base.py b/server/api/base.py index 9167be66b6f..abc855116b4 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -5,6 +5,9 @@ from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.inspection import inspect from werkzeug.routing import parse_rule +from marshmallow import Schema +from marshmallow.compat import with_metaclass +from marshmallow_sqlalchemy.schema import ModelSchemaMeta, ModelSchemaOpts from webargs.flaskparser import FlaskParser, abort from webargs.core import ValidationError from server.models import Workspace, db @@ -317,3 +320,13 @@ class ReadWriteWorkspacedView(CreateWorkspacedMixin, """A generic workspaced view with list, retrieve and create endpoints""" pass + + +class AutoSchema(with_metaclass(ModelSchemaMeta, Schema)): + """ + A Marshmallow schema that does field introspection based on + the SQLAlchemy model specified in Meta.model. + Unlike the marshmallow_sqlalchemy ModelSchema, it doesn't change + the serialization and deserialization proccess. + """ + OPTIONS_CLASS = ModelSchemaOpts diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 4a7bd374c05..ccad341eb7a 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -5,34 +5,34 @@ import flask from flask import Blueprint from flask_classful import route -from marshmallow import Schema, fields +from marshmallow import fields from sqlalchemy.orm import undefer from server.utils.logger import get_logger from server.utils.web import gzipped, validate_workspace,\ get_integer_parameter, filter_request_args from server.dao.host import HostDAO -from server.api.base import ReadWriteWorkspacedView, PaginatedMixin -from server.models import Host +from server.api.base import ReadWriteWorkspacedView, PaginatedMixin, AutoSchema +from server.models import Host, Service host_api = Blueprint('host_api', __name__) -class HostSchema(Schema): - id = fields.Integer(required=True, dump_only=True) - ip = fields.String(required=True) - description = fields.String(required=True) - os = fields.String() - service_count = fields.Integer() +class HostSchema(AutoSchema): + description = fields.String(required=True) # Explicitly set required=True + service_count = fields.Integer(dump_only=True) -class ServiceSchema(Schema): - id = fields.Integer(required=True, dump_only=True) - name = fields.String(required=True) - description = fields.String(required=False) - port = fields.Integer(required=True) - protocol = fields.String(required=True) - status = fields.String(required=True) + class Meta: + model = Host + fields = ('id', 'ip', 'description', 'os', 'service_count') + + +class ServiceSchema(AutoSchema): + + class Meta: + model = Service + fields = ('id', 'name', 'description', 'port', 'protocol', 'status') class HostsView(PaginatedMixin, diff --git a/server/api/modules/licenses.py b/server/api/modules/licenses.py index 914f3177aab..fbc59954b23 100644 --- a/server/api/modules/licenses.py +++ b/server/api/modules/licenses.py @@ -4,19 +4,19 @@ import flask from flask import Blueprint -from marshmallow import Schema, fields +from marshmallow import fields from server.models import License -from server.api.base import ReadWriteView +from server.api.base import ReadWriteView, AutoSchema license_api = Blueprint('license_api', __name__) -class LicenseSchema(Schema): - id = fields.Integer(required=True, dump_only=True) - product = fields.String(required=True) - start_date = fields.DateTime(required=True) - end_date = fields.DateTime(required=True) +class LicenseSchema(AutoSchema): + + class Meta: + model = License + fields = ('id', 'product', 'start_date', 'end_date') class LicenseView(ReadWriteView): From 82b06d6b5e97157efc85a444d482de6112f4e5e6 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 18 Sep 2017 15:20:16 -0300 Subject: [PATCH 0231/1506] Fix workspace scope The scope used to be a text field but was supposed to be a list. Replaced by a list and fixed the migration script to interpret every new line of the scope as an entry. --- server/dao/workspace.py | 6 +++--- server/importer.py | 34 ++++++++++++++++++---------------- server/models.py | 23 +++++++++++++++++++++-- 3 files changed, 42 insertions(+), 21 deletions(-) diff --git a/server/dao/workspace.py b/server/dao/workspace.py index d373e736fd2..799018aded5 100644 --- a/server/dao/workspace.py +++ b/server/dao/workspace.py @@ -25,7 +25,7 @@ class WorkspaceDAO(object): "end_date": [Workspace.end_date], "name": [Workspace.name], "public": [Workspace.public], - "scope": [Workspace.scope], + # "scope": [Workspace.scope], "start_date": [Workspace.start_date] } @@ -72,7 +72,7 @@ def __query_database(self, search=None, page=0, page_size=0, Workspace.end_date, Workspace.name, Workspace.public, - Workspace.scope, + # Workspace.scope, Workspace.start_date, ) @@ -100,6 +100,6 @@ def __get_workspace_data(self, workspace): "end_date": workspace.end_date, "name": workspace.name, "public": workspace.public, - "scope": workspace.scope, + # "scope": workspace.scope, "start_date": workspace.start_date } diff --git a/server/importer.py b/server/importer.py index f84770077fb..158ab9a2f5e 100644 --- a/server/importer.py +++ b/server/importer.py @@ -27,28 +27,29 @@ import server.utils.logger from server.models import ( db, - EntityMetadata, + Command, + Comment, + CommentObject, Credential, + EntityMetadata, + ExecutiveReport, Host, - Service, - Reference, - Command, - Workspace, Hostname, - Vulnerability, - VulnerabilityWeb, - User, + License, + Methodology, + MethodologyTemplate, PolicyViolation, + Reference, + ReferenceTemplate, + Service, + Scope, Task, TaskTemplate, - Methodology, - MethodologyTemplate, - ExecutiveReport, + User, + Vulnerability, VulnerabilityTemplate, - ReferenceTemplate, - License, - Comment, - CommentObject, + VulnerabilityWeb, + Workspace, ) from server.utils.database import get_or_create from server.web import app @@ -541,7 +542,8 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation workspace.start_date = datetime.datetime.fromtimestamp(float(document.get('duration')['start'])/1000) if document.get('duration') and document.get('duration')['end']: workspace.end_date = datetime.datetime.fromtimestamp(float(document.get('duration')['end'])/1000) - workspace.scope = document.get('scope') + for scope in [x.strip() for x in document.get('scope', '').split('\n') if x.strip()]: + scope_obj, created = get_or_create(session, Scope, name=scope, workspace=workspace) yield workspace diff --git a/server/models.py b/server/models.py index b545bbde972..a6d926d4c00 100644 --- a/server/models.py +++ b/server/models.py @@ -522,17 +522,36 @@ class Command(Metadata): class Workspace(Metadata): __tablename__ = 'workspace' id = Column(Integer, primary_key=True) - # TODO: change nullable=True for appropriate fields customer = Column(String(250), nullable=True) # TBI description = Column(Text(), nullable=True) active = Column(Boolean(), nullable=False, default=True) # TBI end_date = Column(DateTime(), nullable=True) name = Column(String(250), nullable=False, unique=True) public = Column(Boolean(), nullable=False, default=True) # TBI - scope = Column(Text(), nullable=True) start_date = Column(DateTime(), nullable=True) +class Scope(Metadata): + __tablename__ = 'scope' + id = Column(Integer, primary_key=True) + name = Column(Text(), nullable=False) + + workspace_id = Column( + Integer, + ForeignKey('workspace.id'), + index=True, + nullable=False + ) + workspace = relationship( + 'Workspace', + backref='scope', + foreign_keys=[workspace_id], + ) + + __table_args__ = ( + UniqueConstraint('name', 'workspace_id', name='uix_scope_name_workspace'), + ) + def is_valid_workspace(workspace_name): return db.session.query(server.models.Workspace).filter_by(name=workspace_name).first() is not None From c83b4864930bc0125ef6a44c015de2d599eccfd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 18 Sep 2017 15:30:33 -0300 Subject: [PATCH 0232/1506] Add integration with filteralchemy Allow to filter by os (exact) in hosts view --- requirements_server.txt | 1 + server/api/base.py | 22 ++++++++++++++++++++-- server/api/modules/hosts.py | 18 +++++++++++++++++- test_cases/test_api_hosts.py | 24 +++++++++++++++++++++++- 4 files changed, 61 insertions(+), 4 deletions(-) diff --git a/requirements_server.txt b/requirements_server.txt index 31e614951be..30ef8817d93 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -16,3 +16,4 @@ flask-classful marshmallow webargs marshmallow-sqlalchemy +filteralchemy diff --git a/server/api/base.py b/server/api/base.py index abc855116b4..b459f6c3a73 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -158,9 +158,13 @@ def _envelope_list(self, objects, pagination_metadata=None): def _paginate(self, query): return query, None + def _filter_query(self, query): + """Return a new SQLAlchemy query with some filters applied""" + return query + def index(self, **kwargs): - objects, pagination_metadata = self._paginate( - self._get_base_query(**kwargs)) + query = self._filter_query(self._get_base_query(**kwargs)) + objects, pagination_metadata = self._paginate(query) return self._envelope_list(self._dump(objects, many=True), pagination_metadata) @@ -190,6 +194,20 @@ def _paginate(self, query): return super(PaginatedMixin, self)._paginate(query) +class FilterAlchemyMixin(object): + """Add querystring parameter filtering to list route + + It is done by setting the ViewClass.filterset_class class + attribute + """ + + filterset_class = None + + def _filter_query(self, query): + assert self.filterset_class is not None, 'You must define a filterset' + return self.filterset_class(query).filter() + + class ListWorkspacedMixin(ListMixin): """Add GET // route""" # There are no differences with the non-workspaced implementations. The code diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index ccad341eb7a..f90443b3912 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -6,13 +6,20 @@ from flask import Blueprint from flask_classful import route from marshmallow import fields +from filteralchemy import FilterSet from sqlalchemy.orm import undefer +from webargs.flaskparser import parser from server.utils.logger import get_logger from server.utils.web import gzipped, validate_workspace,\ get_integer_parameter, filter_request_args from server.dao.host import HostDAO -from server.api.base import ReadWriteWorkspacedView, PaginatedMixin, AutoSchema +from server.api.base import ( + ReadWriteWorkspacedView, + PaginatedMixin, + AutoSchema, + FilterAlchemyMixin, +) from server.models import Host, Service host_api = Blueprint('host_api', __name__) @@ -28,6 +35,13 @@ class Meta: fields = ('id', 'ip', 'description', 'os', 'service_count') +class HostFilterSet(FilterSet): + class Meta: + model = Host + fields = ('os',) + parser = parser + + class ServiceSchema(AutoSchema): class Meta: @@ -36,11 +50,13 @@ class Meta: class HostsView(PaginatedMixin, + FilterAlchemyMixin, ReadWriteWorkspacedView): route_base = 'hosts' model_class = Host schema_class = HostSchema unique_fields = ['ip'] + filterset_class = HostFilterSet @route('//services/') def service_list(self, workspace_name, host_id): diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index c1e7feabb2c..3581fa32c56 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -51,6 +51,14 @@ def url(self, host=None, workspace=None): def services_url(self, host, workspace=None): return self.url(host, workspace) + 'services/' + def compare_results(self, hosts, response): + """ + Compare is the hosts in response are the same that in hosts. + It only compares the IDs of each one, not other fields""" + hosts_in_list = set(host.id for host in hosts) + hosts_in_response = set(host['id'] for host in response.json) + assert hosts_in_list == hosts_in_response + def test_list_retrieves_all_items_from_workspace(self, test_client, second_workspace, session, @@ -205,6 +213,21 @@ def test_index_shows_service_count(self, test_client, host_services): if host['id'] in ids_map: assert host['service_count'] == len(ids_map[host['id']]) + def test_filter_by_os_exact(self, test_client, session, workspace, + second_workspace, host_factory): + # The hosts that should be shown + hosts = host_factory.create_batch(10, workspace=workspace, os='Unix') + + # Search should be case sensitive so this shouln't be shown + host_factory.create_batch(1, workspace=workspace, os='UNIX') + + # This shouldn't be shown, they are from other workspace + host_factory.create_batch(5, workspace=second_workspace, os='Unix') + + res = test_client.get(self.url() + '?os=Unix') + assert res.status_code == 200 + self.compare_results(hosts, res) + class TestHostAPIGeneric(ReadWriteAPITests, PaginationTestsMixin): model = Host @@ -213,4 +236,3 @@ class TestHostAPIGeneric(ReadWriteAPITests, PaginationTestsMixin): unique_fields = ['ip'] update_fields = ['ip', 'description', 'os'] view_class = HostsView - From 680b2fb690e7b342a39099081c91c5b8b37cafe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 18 Sep 2017 16:23:37 -0300 Subject: [PATCH 0233/1506] Add like and ilike filters to Host.os Changed SQLite engine connector to enable case sensitive LIKEs --- server/api/modules/hosts.py | 3 +- server/models.py | 56 ++++++++++++++++++++---------------- test_cases/test_api_hosts.py | 27 ++++++++++++++++- 3 files changed, 59 insertions(+), 27 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index f90443b3912..ca6bc694a75 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -6,7 +6,7 @@ from flask import Blueprint from flask_classful import route from marshmallow import fields -from filteralchemy import FilterSet +from filteralchemy import FilterSet, operators from sqlalchemy.orm import undefer from webargs.flaskparser import parser @@ -40,6 +40,7 @@ class Meta: model = Host fields = ('os',) parser = parser + operators = (operators.Equal, operators.Like, operators.ILike) class ServiceSchema(AutoSchema): diff --git a/server/models.py b/server/models.py index efaeb1af98e..7f67245c606 100644 --- a/server/models.py +++ b/server/models.py @@ -49,31 +49,37 @@ def make_connector(self, app=None, bind=None): class CustomEngineConnector(_EngineConnector): - """Used by overrideb SQLAlchemy class to fix rollback issues""" - - def get_engine(self): - # Use an existent engine and don't register events if possible - uri = self.get_uri() - echo = self._app.config['SQLALCHEMY_ECHO'] - if (uri, echo) == self._connected_for: - return self._engine - - # Call original metohd and register events - rv = super(CustomEngineConnector, self).get_engine() - if uri.startswith('sqlite://'): - with self._lock: - @event.listens_for(rv, "connect") - def do_connect(dbapi_connection, connection_record): - # disable pysqlite's emitting of the BEGIN statement - # entirely. also stops it from emitting COMMIT before any - # DDL. - dbapi_connection.isolation_level = None - - @event.listens_for(rv, "begin") - def do_begin(conn): - # emit our own BEGIN - conn.execute("BEGIN") - return rv + """Used by overrided SQLAlchemy class to fix rollback issues. + + Also set case sensitive likes (in SQLite there are case + insensitive by default)""" + + def get_engine(self): + # Use an existent engine and don't register events if possible + uri = self.get_uri() + echo = self._app.config['SQLALCHEMY_ECHO'] + if (uri, echo) == self._connected_for: + return self._engine + + # Call original metohd and register events + rv = super(CustomEngineConnector, self).get_engine() + if uri.startswith('sqlite://'): + with self._lock: + @event.listens_for(rv, "connect") + def do_connect(dbapi_connection, connection_record): + # disable pysqlite's emitting of the BEGIN statement + # entirely. also stops it from emitting COMMIT before any + # DDL. + dbapi_connection.isolation_level = None + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA case_sensitive_like=true") + cursor.close() + + @event.listens_for(rv, "begin") + def do_begin(conn): + # emit our own BEGIN + conn.execute("BEGIN") + return rv db = SQLAlchemy() diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 3581fa32c56..ed1594d2664 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -224,10 +224,35 @@ def test_filter_by_os_exact(self, test_client, session, workspace, # This shouldn't be shown, they are from other workspace host_factory.create_batch(5, workspace=second_workspace, os='Unix') - res = test_client.get(self.url() + '?os=Unix') + url = self.url() + '?os=Unix' + res = test_client.get(url) assert res.status_code == 200 self.compare_results(hosts, res) + def test_filter_by_os_like_ilike(self, test_client, session, workspace, + second_workspace, host_factory): + # The hosts that should be shown + hosts = host_factory.create_batch(5, workspace=workspace, os='Unix 1') + hosts += host_factory.create_batch(5, workspace=workspace, os='Unix 2') + + # This should only be shown when using ilike, not when using like + case_insensitive_host = host_factory.create( + workspace=workspace, os='UNIX 3') + + # This doesn't match the like expression + host_factory.create(workspace=workspace, os="Test Unix 1") + + # This shouldn't be shown anywhere, they are from other workspace + host_factory.create_batch(5, workspace=second_workspace, os='Unix') + + res = test_client.get(self.url() + '?os__like=Unix %') + assert res.status_code == 200 + self.compare_results(hosts, res) + + res = test_client.get(self.url() + '?os__ilike=Unix %') + assert res.status_code == 200 + self.compare_results(hosts + [case_insensitive_host], res) + class TestHostAPIGeneric(ReadWriteAPITests, PaginationTestsMixin): model = Host From 60711dab2e630b4f9389ae1f9fac56828de69d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 18 Sep 2017 16:42:41 -0300 Subject: [PATCH 0234/1506] Move FilterSet parser declaration to base class To improve future refactors --- server/api/base.py | 7 ++++++- server/api/modules/hosts.py | 5 ++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index b459f6c3a73..2f9139b4f11 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -8,7 +8,7 @@ from marshmallow import Schema from marshmallow.compat import with_metaclass from marshmallow_sqlalchemy.schema import ModelSchemaMeta, ModelSchemaOpts -from webargs.flaskparser import FlaskParser, abort +from webargs.flaskparser import FlaskParser, parser, abort from webargs.core import ValidationError from server.models import Workspace, db @@ -348,3 +348,8 @@ class AutoSchema(with_metaclass(ModelSchemaMeta, Schema)): the serialization and deserialization proccess. """ OPTIONS_CLASS = ModelSchemaOpts + + +class FilterSetMeta: + """Base Meta class of FilterSet objects""" + parser = parser diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index ca6bc694a75..9e842b4c2d7 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -8,7 +8,6 @@ from marshmallow import fields from filteralchemy import FilterSet, operators from sqlalchemy.orm import undefer -from webargs.flaskparser import parser from server.utils.logger import get_logger from server.utils.web import gzipped, validate_workspace,\ @@ -19,6 +18,7 @@ PaginatedMixin, AutoSchema, FilterAlchemyMixin, + FilterSetMeta, ) from server.models import Host, Service @@ -36,10 +36,9 @@ class Meta: class HostFilterSet(FilterSet): - class Meta: + class Meta(FilterSetMeta): model = Host fields = ('os',) - parser = parser operators = (operators.Equal, operators.Like, operators.ILike) From b63d0e5c0fa20534d93d6dc0cb197c6eb679c2b0 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 18 Sep 2017 16:56:33 -0300 Subject: [PATCH 0235/1506] Add Risk to Vulnerabilities All vulns (including templates) need an additional field 'risk', a float between 1.0 and 10.0. --- server/models.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/models.py b/server/models.py index a6d926d4c00..ce74901b34c 100644 --- a/server/models.py +++ b/server/models.py @@ -213,6 +213,7 @@ class VulnerabilityABC(Metadata): name = Column(Text, nullable=False) resolution = Column(Text, nullable=True) severity = Column(Enum(*SEVERITIES, name='vulnerability_severity'), nullable=False) + risk = Column(Float(3,1), nullable=True) # TODO add evidence impact_accountability = Column(Boolean, default=False) @@ -220,6 +221,11 @@ class VulnerabilityABC(Metadata): impact_confidentiality = Column(Boolean, default=False) impact_integrity = Column(Boolean, default=False) + __table_args__ = ( + CheckConstraint('1.0 <= risk AND risk <= 10.0', + name='check_vulnerability_risk'), + ) + class VulnerabilityTemplate(VulnerabilityABC): __tablename__ = 'vulnerability_template' From 2568ce1f3c6df9cf20433dd2fe7fe8deada349a6 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 18 Sep 2017 17:14:14 -0300 Subject: [PATCH 0236/1506] Add missing fields in Vuln Code --- server/models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index ce74901b34c..9df4dea75f3 100644 --- a/server/models.py +++ b/server/models.py @@ -82,6 +82,8 @@ class SourceCode(Metadata): __tablename__ = 'source_code' id = Column(Integer, primary_key=True) filename = Column(Text, nullable=False) + function = Column(Text, nullable=True) + module = Column(Text, nullable=True) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship('Workspace', backref='source_codes') @@ -315,7 +317,9 @@ class VulnerabilityWeb(VulnerabilityGeneric): class VulnerabilityCode(VulnerabilityGeneric): - line = Column(Integer, nullable=True) + code = Column(Text, nullable=True) + start_line = Column(Integer, nullable=True) + end_line = Column(Integer, nullable=True) source_code_id = Column(Integer, ForeignKey(SourceCode.id), index=True) source_code = relationship( From f0a03031303d6cc1076cdef03b933ccccf03aa38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 18 Sep 2017 18:41:14 -0300 Subject: [PATCH 0237/1506] Move Host.service_count definition inside Host --- server/models.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/models.py b/server/models.py index 7f67245c606..9633f961c39 100644 --- a/server/models.py +++ b/server/models.py @@ -174,6 +174,12 @@ class Host(Metadata): backref='hosts', foreign_keys=[workspace_id] ) + + service_count = column_property((select([func.count('service.id')]). + select_from('service'). + where("service.host_id = host.id") + ), deferred=True) + __table_args__ = ( UniqueConstraint(ip, workspace_id, name='uix_host_ip_workspace'), ) @@ -238,12 +244,6 @@ class Service(Metadata): UniqueConstraint(port, protocol, host_id, workspace_id, name='uix_service_port_protocol_host_workspace'), ) -# TODO: Move this to Host definition. I need a way to reference Service before -# it is declared -Host.service_count = column_property((select([func.count(Service.id)]). - where(Service.host_id == Host.id)#.label('service_count') - ), deferred=True) - class VulnerabilityABC(Metadata): # revisar plugin nexpose, netspark para terminar de definir uniques. asegurar que se carguen bien From 4f216962e971cf34eb18d32eea63ba860be6882c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 19 Sep 2017 12:40:07 -0300 Subject: [PATCH 0238/1506] Fix a bug on foreign key column name --- server/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/models.py b/server/models.py index 7ee1140e6e9..580bd26547d 100644 --- a/server/models.py +++ b/server/models.py @@ -324,7 +324,7 @@ class Vulnerability(VulnerabilityGeneric): @declared_attr def service_id(cls): - return VulnerabilityGeneric.__table__.c.get('service_id', Column(Integer, db.ForeignKey('vulnerability.service_id'))) + return VulnerabilityGeneric.__table__.c.get('service_id', Column(Integer, db.ForeignKey('service.id'))) @declared_attr def service(cls): @@ -348,7 +348,7 @@ class VulnerabilityWeb(VulnerabilityGeneric): @declared_attr def service_id(cls): - return VulnerabilityGeneric.__table__.c.get('service_id', Column(Integer, db.ForeignKey('vulnerability.service_id'))) + return VulnerabilityGeneric.__table__.c.get('service_id', Column(Integer, db.ForeignKey('service.id'))) @declared_attr def service(cls): From fa5138606874fb9c57138e162e659bc6bda55094 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 19 Sep 2017 14:11:57 -0300 Subject: [PATCH 0239/1506] Add DB Schema files to Git Ignore file --- .gitignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index dfc6eca8658..9de77059c56 100644 --- a/.gitignore +++ b/.gitignore @@ -59,4 +59,8 @@ jsconfig.json .idea # vFeed DB -data/vfeed.db \ No newline at end of file +data/vfeed.db + +# Images based on DB Schema +entity_dbschema.png +uml_schema.png From 31604c649bf60fb06df281e6dadb8a2da7c91c32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 19 Sep 2017 14:33:35 -0300 Subject: [PATCH 0240/1506] Fix bugs in service relationship of vulns --- server/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/models.py b/server/models.py index 580bd26547d..a0cfae5e4b3 100644 --- a/server/models.py +++ b/server/models.py @@ -328,7 +328,7 @@ def service_id(cls): @declared_attr def service(cls): - return relationship('VulnerabilityGeneric') + return relationship('Service') __mapper_args__ = { 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[0] @@ -352,7 +352,7 @@ def service_id(cls): @declared_attr def service(cls): - return relationship('VulnerabilityGeneric') + return relationship('Service') __mapper_args__ = { 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[1] From 5083aadf32c729e7dc128a7b9c9cf02327e33254 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 19 Sep 2017 14:44:02 -0300 Subject: [PATCH 0241/1506] Fix warning when some DBs are missing The Licenses and Vuln Templates Databases may be missing, the user should receive a nice Warning instead of a stack trace and failure. --- server/importer.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/server/importer.py b/server/importer.py index 158ab9a2f5e..51d0e307f68 100644 --- a/server/importer.py +++ b/server/importer.py @@ -558,7 +558,6 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation methodology, created = get_or_create(session, Methodology, name=document.get('name')) methodology.workspace = workspace yield methodology -# methodology. class TaskImporter(object): @@ -782,12 +781,16 @@ def run(self): ) try: - cwes = requests.get(cwe_url).json()['rows'] + cwes = requests.get(cwe_url) + cwes.raise_for_status() + except requests.exceptions.HTTPError: + logger.warn('Unable to retrieve Vulnerability Templates Database. Moving on.') + return except requests.exceptions.RequestException as e: logger.warn(e) return - for cwe in cwes: + for cwe in cwes.json()['rows']: document = cwe['doc'] new_severity = self.get_severity(document) @@ -839,7 +842,7 @@ def get_severity(self, document): class ImportLicense(FlaskScriptCommand): def run(self): - cwe_url = "http://{username}:{password}@{hostname}:{port}/{path}".format( + licenses_url = "http://{username}:{password}@{hostname}:{port}/{path}".format( username=server.config.couchdb.user, password=server.config.couchdb.password, hostname=server.config.couchdb.host, @@ -847,17 +850,17 @@ def run(self): path='faraday_licenses/_all_docs?include_docs=true' ) - if not requests.head(cwe_url).ok: - logger.info('No Licenses database found, nothing to see here, move along!') - return - try: - licenses = requests.get(cwe_url).json()['rows'] + licenses = requests.get(licenses_url) + licenses.raise_for_status() + except requests.exceptions.HTTPError: + logger.warn('Unable to retrieve Licenses Database. Moving on.') + return except requests.exceptions.RequestException as e: logger.warn(e) return - for license in licenses: + for license in licenses.json()['rows']: document = license['doc'] license_obj, created = get_or_create(session, From 169b229de90a9f3a8a17fe0395ed0ab59f527fbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 19 Sep 2017 14:52:00 -0300 Subject: [PATCH 0242/1506] Add vuln web, vuln code and source code factories Also sort factories alphabically (when using or importing them, not in its declaration) --- test_cases/conftest.py | 9 ++++++--- test_cases/factories.py | 34 ++++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 4d74162d565..fa34ea28a2b 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -14,13 +14,16 @@ enabled_factories = [ - factories.WorkspaceFactory, + factories.CredentialFactory, + factories.LicenseFactory, factories.HostFactory, factories.ServiceFactory, + factories.SourceCodeFactory, factories.VulnerabilityFactory, - factories.CredentialFactory, - factories.LicenseFactory, + factories.VulnerabilityCodeFactory, + factories.VulnerabilityWebFactory, factories.UserFactory, + factories.WorkspaceFactory, ] for factory in enabled_factories: register(factory) diff --git a/test_cases/factories.py b/test_cases/factories.py index 33451deae62..b0240266311 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -14,8 +14,11 @@ Host, License, Service, + SourceCode, User, Vulnerability, + VulnerabilityCode, + VulnerabilityWeb, Workspace, ) @@ -113,12 +116,20 @@ class Meta: sqlalchemy_session = db.session +class SourceCodeFactory(WorkspaceObjectFactory): + filename = FuzzyText() + + class Meta: + model = SourceCode + sqlalchemy_session = db.session + + class VulnerabilityFactory(WorkspaceObjectFactory): name = FuzzyText() description = FuzzyText() - # host = factory.SubFactory(HostFactory) - # service = factory.SubFactory(ServiceFactory) + # host = factory.SubFactory(HostFactory) # TODO: Move to generic class + # service = factory.SubFactory(ServiceFactory) # TODO: Move to generic class workspace = factory.SubFactory(WorkspaceFactory) creator = factory.SubFactory(UserFactory) severity = FuzzyChoice(['critical', 'high']) @@ -128,6 +139,25 @@ class Meta: sqlalchemy_session = db.session +class VulnerabilityWebFactory(VulnerabilityFactory): + method = FuzzyChoice(['GET', 'POST', 'PUT', 'PATCH' 'DELETE']) + parameter_name = FuzzyText() + service = factory.SubFactory(ServiceFactory) + + class Meta: + model = VulnerabilityWeb + sqlalchemy_session = db.session + + +class VulnerabilityCodeFactory(VulnerabilityFactory): + line = FuzzyInteger(1, 5000) + source_code = factory.SubFactory(SourceCodeFactory) + + class Meta: + model = VulnerabilityCode + sqlalchemy_session = db.session + + class CredentialFactory(WorkspaceObjectFactory): username = FuzzyText() password = FuzzyText() From 914c1475868561faec42ad3d5260e71414b0da78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 19 Sep 2017 15:41:18 -0300 Subject: [PATCH 0243/1506] Add vuln count properties to workspaces --- server/models.py | 18 +++++++++++ test_cases/models/test_workspace.py | 47 +++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 test_cases/models/test_workspace.py diff --git a/server/models.py b/server/models.py index fa496ea4aba..8b37c5ab52d 100644 --- a/server/models.py +++ b/server/models.py @@ -573,6 +573,19 @@ class Command(Metadata): ) +def make_vuln_count_property(type_=None): + query = (select([func.count('vulnerability.id')]). + select_from('vulnerability'). + where('vulnerability.workspace_id = workspace.id') + ) + if type_: + # Don't do queries using this style! + # This can cause SQL injection vulnerabilities + # In this case type_ is supplied from a whitelist so this is safe + query = query.where("vulnerability.type = '%s'" % type_) + return column_property(query, deferred=True) + + class Workspace(Metadata): __tablename__ = 'workspace' id = Column(Integer, primary_key=True) @@ -586,6 +599,11 @@ class Workspace(Metadata): scope = Column(Text(), nullable=True) start_date = Column(DateTime(), nullable=True) + vulnerability_web_count = make_vuln_count_property('vulnerability_web') + vulnerability_code_count = make_vuln_count_property('vulnerability_code') + vulnerability_standard_count = make_vuln_count_property('vulnerability') + vulnerability_total_count = make_vuln_count_property() + def is_valid_workspace(workspace_name): return db.session.query(server.models.Workspace).filter_by(name=workspace_name).first() is not None diff --git a/test_cases/models/test_workspace.py b/test_cases/models/test_workspace.py new file mode 100644 index 00000000000..5c4c757926a --- /dev/null +++ b/test_cases/models/test_workspace.py @@ -0,0 +1,47 @@ +import pytest +from server.models import db +from test_cases.factories import ( + HostFactory, + ServiceFactory, + SourceCodeFactory, + VulnerabilityFactory, + VulnerabilityCodeFactory, + VulnerabilityWebFactory, +) + +SOURCE_CODE_VULN_COUNT = 3 +STANDARD_VULN_COUNT = [6, 2] # With host parent and with service parent +WEB_VULN_COUNT = 5 + + +def populate_workspace(workspace): + host = HostFactory.create(workspace=workspace) + service = ServiceFactory.create(workspace=workspace, host=host) + code = SourceCodeFactory.create(workspace=workspace) + + # Create standard vulns + host_vulns = VulnerabilityFactory.create_batch( + STANDARD_VULN_COUNT[0], workspace=workspace, host=host) + service_vulns = VulnerabilityFactory.create_batch( + STANDARD_VULN_COUNT[1], workspace=workspace, service=service) + + # Create web vulns + web_vulns = VulnerabilityWebFactory.create_batch( + WEB_VULN_COUNT, workspace=workspace, service=service) + + # Create source code vulns + code_vulns = VulnerabilityCodeFactory.create_batch( + SOURCE_CODE_VULN_COUNT, workspace=workspace, source_code=code) + + db.session.commit() + + +def test_vuln_count(workspace, second_workspace): + populate_workspace(workspace) + populate_workspace(second_workspace) + assert workspace.vulnerability_web_count == WEB_VULN_COUNT + assert workspace.vulnerability_code_count == SOURCE_CODE_VULN_COUNT + assert workspace.vulnerability_standard_count == sum(STANDARD_VULN_COUNT) + assert workspace.vulnerability_total_count == ( + sum(STANDARD_VULN_COUNT) + WEB_VULN_COUNT + SOURCE_CODE_VULN_COUNT + ) From 3ba2b9071cdc575427ea9474412d1cffde9d5d99 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 19 Sep 2017 18:27:23 -0300 Subject: [PATCH 0244/1506] Fix alphabetical order in requirements files --- requirements.txt | 12 ++++++------ requirements_dev.txt | 2 +- requirements_server.txt | 22 +++++++++++----------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/requirements.txt b/requirements.txt index 68fdda95530..b3d1479bca9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ -couchdbkit -mockito -whoosh argparse +colorama +couchdbkit +flask==0.12.2 IPy -restkit +mockito requests +restkit tornado -flask==0.12.2 -colorama tqdm==4.15.0 +whoosh diff --git a/requirements_dev.txt b/requirements_dev.txt index dbf901a1aab..a8b7439b29a 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,3 +1,3 @@ +factory-boy<=2.8.1 pytest pytest-factoryboy -factory-boy<=2.8.1 diff --git a/requirements_server.txt b/requirements_server.txt index 7c9bacffd8a..382d022aec0 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -1,17 +1,17 @@ -couchdbkit>=0.6.5 -restkit>=4.2.2 -requests>=2.10.0 -flask>=0.10.1 -tqdm==4.15.0 -twisted>=16.1.1 -sqlalchemy>=1.0.12 -pyopenssl>=16.0.0 -service_identity>=16.0.0 -pyasn1-modules -Flask-Security==3.0.0 bcrypt +couchdbkit>=0.6.5 Flask-SQLAlchemy==2.2 Flask-Script==2.0.5 flask-classful +Flask-Security==3.0.0 +flask>=0.10.1 marshmallow +pyasn1-modules +pyopenssl>=16.0.0 +requests>=2.10.0 +restkit>=4.2.2 +service_identity>=16.0.0 +sqlalchemy>=1.0.12 +tqdm==4.15.0 +twisted>=16.1.1 webargs From a3186c7cd97df697b9029aec8acf5d8a691b649f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 19 Sep 2017 18:28:19 -0300 Subject: [PATCH 0245/1506] Add SelfNestedField helper marshmallow field --- server/schemas.py | 20 ++++++++++++++++++++ test_cases/test_schemas.py | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 server/schemas.py create mode 100644 test_cases/test_schemas.py diff --git a/server/schemas.py b/server/schemas.py new file mode 100644 index 00000000000..f7b046a2047 --- /dev/null +++ b/server/schemas.py @@ -0,0 +1,20 @@ +from marshmallow import fields +from marshmallow.exceptions import ValidationError + +class SelfNestedField(fields.Field): + """A field to make namespaced schemas. It allows to have + a field whose contents are the dump of the same object with + other schema""" + + # Required because the target attribute will probably not exist + _CHECK_ATTRIBUTE = False + + def __init__(self, target_schema, *args, **kwargs): + self.target_schema = target_schema + super(SelfNestedField, self).__init__(*args, **kwargs) + + def _serialize(self, value, attr, obj): + ret, errors = self.target_schema.dump(obj) + if errors: + raise ValidationError(errors, data=ret) + return ret diff --git a/test_cases/test_schemas.py b/test_cases/test_schemas.py new file mode 100644 index 00000000000..684097dc8cd --- /dev/null +++ b/test_cases/test_schemas.py @@ -0,0 +1,20 @@ +from collections import namedtuple +from marshmallow import Schema, fields +from server.schemas import SelfNestedField + +Place = namedtuple('Place', ['name', 'x', 'y']) + +class PointSchema(Schema): + x = fields.Float() + y = fields.Float() + +class PlaceSchema(Schema): + name = fields.Str() + coords = SelfNestedField(PointSchema()) + +class TestSelfNestedField: + def test_field_serialization(self): + point = Place('home', 123, 456.1) + schema = PlaceSchema() + dumped = schema.dump(point).data + assert dumped == {"name": "home", "coords": {"x": 123.0, "y": 456.1}} From 37972599b48f74b3831dda8692cb1dbe39a4f4d9 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 19 Sep 2017 18:28:51 -0300 Subject: [PATCH 0246/1506] Add python-slugify as requirement for the server --- requirements_server.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_server.txt b/requirements_server.txt index 382d022aec0..baf43fc700e 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -8,6 +8,7 @@ flask>=0.10.1 marshmallow pyasn1-modules pyopenssl>=16.0.0 +python-slugify==1.2.4 requests>=2.10.0 restkit>=4.2.2 service_identity>=16.0.0 From 03d03ca9c9b5e971ef3a755eb60d609c22dac33b Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 19 Sep 2017 18:46:43 -0300 Subject: [PATCH 0247/1506] Add function to create tags --- server/importer.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index 51d0e307f68..0b4a76a1eb6 100644 --- a/server/importer.py +++ b/server/importer.py @@ -11,6 +11,7 @@ defaultdict, OrderedDict ) +from slugify import slugify from binascii import unhexlify import requests @@ -43,6 +44,7 @@ ReferenceTemplate, Service, Scope, + Tag, Task, TaskTemplate, User, @@ -580,7 +582,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation if not methodology: methodology = session.query(MethodologyTemplate).filter_by(id=methodology_id).first() task_class = TaskTemplate - task, created = get_or_create(session, task_class, name=document.get('name')) + task, task_created = get_or_create(session, task_class, name=document.get('name')) if task_class == TaskTemplate: task.template = methodology else: @@ -588,6 +590,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation task.workspace = workspace task.description = document.get('description') task.assigned_to = session.query(User).filter_by(username=document.get('username')).first() + create_tags([x.strip() for x in document.tags if x.strip()], 'Tag', task.id) mapped_status = { 'New': 'new', 'In Progress': 'in progress', @@ -600,6 +603,11 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation return [task] +def create_tags(tags, model, parent_id): + # tag, create = get_or_create(session, Tag, name=scope, workspace=workspace) + return True + + class ReportsImporter(object): def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): From 3294bf7b70bb046a6025611ed99e6180f9cb3abe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 19 Sep 2017 19:45:12 -0300 Subject: [PATCH 0248/1506] Refactor children count column properties and add to workspace --- server/models.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/server/models.py b/server/models.py index 8b37c5ab52d..74848202409 100644 --- a/server/models.py +++ b/server/models.py @@ -87,6 +87,19 @@ def do_begin(conn): SCHEMA_VERSION = 'W.3.0.0' +def _make_generic_count_property(parent_table, children_table): + """Make a deferred by default column property that counts the + amount of childrens of some parent object""" + # TODO: Fix SQLAlchemy warnings + children_id_field = '{}.id'.format(children_table) + parent_id_field = '{}.id'.format(parent_table) + children_rel_field = '{}.{}_id'.format(children_table, parent_table) + query = (select([func.count(children_id_field)]). + select_from(children_table). + where('{} = {}'.format(children_rel_field, parent_id_field))) + return column_property(query, deferred=True) + + class DatabaseMetadata(db.Model): __tablename__ = 'db_metadata' id = Column(Integer, primary_key=True) @@ -175,10 +188,7 @@ class Host(Metadata): foreign_keys=[workspace_id] ) - service_count = column_property((select([func.count('service.id')]). - select_from('service'). - where("service.host_id = host.id") - ), deferred=True) + service_count = _make_generic_count_property('host', 'service') __table_args__ = ( UniqueConstraint(ip, workspace_id, name='uix_host_ip_workspace'), @@ -573,7 +583,7 @@ class Command(Metadata): ) -def make_vuln_count_property(type_=None): +def _make_vuln_count_property(type_=None): query = (select([func.count('vulnerability.id')]). select_from('vulnerability'). where('vulnerability.workspace_id = workspace.id') @@ -599,10 +609,13 @@ class Workspace(Metadata): scope = Column(Text(), nullable=True) start_date = Column(DateTime(), nullable=True) - vulnerability_web_count = make_vuln_count_property('vulnerability_web') - vulnerability_code_count = make_vuln_count_property('vulnerability_code') - vulnerability_standard_count = make_vuln_count_property('vulnerability') - vulnerability_total_count = make_vuln_count_property() + credential_count = _make_generic_count_property('workspace', 'credential') + host_count = _make_generic_count_property('workspace', 'host') + service_count = _make_generic_count_property('workspace', 'service') + vulnerability_web_count = _make_vuln_count_property('vulnerability_web') + vulnerability_code_count = _make_vuln_count_property('vulnerability_code') + vulnerability_standard_count = _make_vuln_count_property('vulnerability') + vulnerability_total_count = _make_vuln_count_property() def is_valid_workspace(workspace_name): From b8a7949d897f119e331042945c1875c9caefa63e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 19 Sep 2017 19:46:12 -0300 Subject: [PATCH 0249/1506] Add workspaces API --- server/api/modules/workspaces.py | 20 +++++++++++++++++++ test_cases/test_api_non_workspaced_various.py | 19 +++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index 2befd1cde85..809e9647763 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -5,6 +5,7 @@ import flask from flask import Blueprint +from marshmallow import fields from server.models import db, Workspace from server.dao.host import HostDAO @@ -25,10 +26,29 @@ list_workspaces_as_user, get_workspace ) +from server.api.base import ReadWriteView, AutoSchema workspace_api = Blueprint('workspace_api', __name__) +class WorkspaceSchema(AutoSchema): + + host_count = fields.Integer(dump_only=True) + + class Meta: + model = Workspace + fields = ('id', 'customer', 'description', 'active', 'start_date', + 'end_date', 'name', 'public', 'scope', 'host_count') + + +class WorkspaceView(ReadWriteView): + route_base = 'workspaces' + model_class = Workspace + schema_class = WorkspaceSchema + +WorkspaceView.register(workspace_api) + + @workspace_api.route('/ws', methods=['GET']) @gzipped def workspace_list(): diff --git a/test_cases/test_api_non_workspaced_various.py b/test_cases/test_api_non_workspaced_various.py index 30e299f4edb..2330dff7fe5 100644 --- a/test_cases/test_api_non_workspaced_various.py +++ b/test_cases/test_api_non_workspaced_various.py @@ -4,8 +4,12 @@ import pytest from test_cases import factories from test_api_non_workspaced_base import ReadWriteAPITests, API_PREFIX -from server.models import License +from server.models import ( + License, + Workspace, +) from server.api.modules.licenses import LicenseView +from server.api.modules.workspaces import WorkspaceView class LicenseEnvelopedView(LicenseView): """A custom view to test that enveloping on generic views work ok""" @@ -31,3 +35,16 @@ def test_envelope_list(self, test_client, app): assert new_res.status_code == 200 assert new_res.json == {"object_list": original_res.json} + + +class TestWorkspaceAPI(ReadWriteAPITests): + model = Workspace + factory = factories.WorkspaceFactory + api_endpoint = 'workspaces' + + def test_host_count(self, host_factory, test_client, session): + host_factory.create(workspace=self.first_object) + session.commit() + res = test_client.get(self.url(self.first_object)) + assert res.status_code == 200 + assert res.json['host_count'] == 1 From 569f8d6dbbcd4d610c757373f521ea1193011800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 19 Sep 2017 19:56:10 -0300 Subject: [PATCH 0250/1506] Move workspace children count to nested field in API --- server/api/modules/workspaces.py | 22 +++++++++++++++---- test_cases/test_api_non_workspaced_various.py | 2 +- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index 809e9647763..c13b2c29d48 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -5,7 +5,7 @@ import flask from flask import Blueprint -from marshmallow import fields +from marshmallow import Schema, fields from server.models import db, Workspace from server.dao.host import HostDAO @@ -13,6 +13,7 @@ from server.dao.service import ServiceDAO from server.dao.workspace import WorkspaceDAO from server.utils.logger import get_logger +from server.schemas import SelfNestedField from server.utils.web import ( build_bad_request_response, filter_request_args, @@ -31,14 +32,27 @@ workspace_api = Blueprint('workspace_api', __name__) -class WorkspaceSchema(AutoSchema): +class WorkspaceSummarySchema(Schema): + credentials = fields.Integer(dump_only=True, attribute='credential_count') + hosts = fields.Integer(dump_only=True, attribute='host_count') + services = fields.Integer(dump_only=True, attribute='service_count') + web_vulns = fields.Integer(dump_only=True, + attribute='vulnerability_web_count') + code_vulns = fields.Integer(dump_only=True, + attribute='vulnerability_code_count') + std_vulns = fields.Integer(dump_only=True, + attribute='vulnerability_standard_count') + total_vulns = fields.Integer(dump_only=True, + attribute='vulnerability_total_count') + - host_count = fields.Integer(dump_only=True) +class WorkspaceSchema(AutoSchema): + stats = SelfNestedField(WorkspaceSummarySchema()) class Meta: model = Workspace fields = ('id', 'customer', 'description', 'active', 'start_date', - 'end_date', 'name', 'public', 'scope', 'host_count') + 'end_date', 'name', 'public', 'scope', 'stats') class WorkspaceView(ReadWriteView): diff --git a/test_cases/test_api_non_workspaced_various.py b/test_cases/test_api_non_workspaced_various.py index 2330dff7fe5..3e274791320 100644 --- a/test_cases/test_api_non_workspaced_various.py +++ b/test_cases/test_api_non_workspaced_various.py @@ -47,4 +47,4 @@ def test_host_count(self, host_factory, test_client, session): session.commit() res = test_client.get(self.url(self.first_object)) assert res.status_code == 200 - assert res.json['host_count'] == 1 + assert res.json['stats']['hosts'] == 1 From 03493126656a31a919e3360e4658a303c1f2d854 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 20 Sep 2017 12:58:00 -0300 Subject: [PATCH 0251/1506] Fix unicode error when importing users with special characters --- server/importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index 51d0e307f68..f05d07d355b 100644 --- a/server/importer.py +++ b/server/importer.py @@ -749,7 +749,7 @@ def import_users(self, all_users, admins): if user['name'] in admins.keys(): # This is an already imported admin user, skip continue - logger.info('Importing {0}'.format(user['name'])) + logger.info(u'Importing {0}'.format(user['name'])) if not db.session.query(User).filter_by(username=user['name']).first(): app.user_datastore.create_user( username=user['name'], From b6bbd3f4537f1eed01faa4f8eb71280e4aa8e6e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 20 Sep 2017 13:01:20 -0300 Subject: [PATCH 0252/1506] Fix factories for new vuln code schema --- test_cases/factories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_cases/factories.py b/test_cases/factories.py index b0240266311..e339a69cd88 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -150,7 +150,7 @@ class Meta: class VulnerabilityCodeFactory(VulnerabilityFactory): - line = FuzzyInteger(1, 5000) + start_line = FuzzyInteger(1, 5000) source_code = factory.SubFactory(SourceCodeFactory) class Meta: From 4aae8b3d7e0b4e18ee88fdb373f27ba1753fbc6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 20 Sep 2017 13:12:42 -0300 Subject: [PATCH 0253/1506] Add legacy _id field to workspaces API --- server/api/modules/workspaces.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index c13b2c29d48..6f22959cde4 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -48,11 +48,12 @@ class WorkspaceSummarySchema(Schema): class WorkspaceSchema(AutoSchema): stats = SelfNestedField(WorkspaceSummarySchema()) + _id = fields.Integer(dump_only=True, attribute='id') class Meta: model = Workspace - fields = ('id', 'customer', 'description', 'active', 'start_date', - 'end_date', 'name', 'public', 'scope', 'stats') + fields = ('_id', 'id', 'customer', 'description', 'active', + 'start_date', 'end_date', 'name', 'public', 'scope', 'stats') class WorkspaceView(ReadWriteView): From f5473486aa11302d6b6eeed699bfc26369c02545 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 20 Sep 2017 14:38:26 -0300 Subject: [PATCH 0254/1506] Change workspaces api URL and lookup field Also remove unnecessary print and add logic for customizing the lookup_field in the test cases --- server/api/modules/workspaces.py | 4 +++- test_cases/test_api_non_workspaced_base.py | 4 ++-- test_cases/test_api_non_workspaced_various.py | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index 6f22959cde4..2fe71786120 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -57,7 +57,9 @@ class Meta: class WorkspaceView(ReadWriteView): - route_base = 'workspaces' + route_base = 'ws' + lookup_field = 'name' + lookup_field_type = unicode model_class = Workspace schema_class = WorkspaceSchema diff --git a/test_cases/test_api_non_workspaced_base.py b/test_cases/test_api_non_workspaced_base.py index f7ea1b54676..9f47c6db5a2 100644 --- a/test_cases/test_api_non_workspaced_base.py +++ b/test_cases/test_api_non_workspaced_base.py @@ -15,7 +15,7 @@ class GenericAPITest: model = None factory = None api_endpoint = None - pk_field = 'id' + lookup_field = 'id' unique_fields = [] update_fields = [] @@ -37,7 +37,7 @@ def object_instance(self, session): def url(self, obj=None): url = API_PREFIX + self.api_endpoint + '/' if obj is not None: - id_ = unicode(obj.id) if isinstance( + id_ = unicode(getattr(obj, self.lookup_field)) if isinstance( obj, self.model) else unicode(obj) url += id_ + u'/' return url diff --git a/test_cases/test_api_non_workspaced_various.py b/test_cases/test_api_non_workspaced_various.py index 3e274791320..5611aa6906f 100644 --- a/test_cases/test_api_non_workspaced_various.py +++ b/test_cases/test_api_non_workspaced_various.py @@ -28,7 +28,6 @@ class TestLicensesAPI(ReadWriteAPITests): def test_envelope_list(self, test_client, app): LicenseEnvelopedView.register(app) - print app.url_map original_res = test_client.get(self.url()) assert original_res.status_code == 200 new_res = test_client.get(API_PREFIX + 'test_envelope_list/') @@ -40,7 +39,8 @@ def test_envelope_list(self, test_client, app): class TestWorkspaceAPI(ReadWriteAPITests): model = Workspace factory = factories.WorkspaceFactory - api_endpoint = 'workspaces' + api_endpoint = 'ws' + lookup_field = 'name' def test_host_count(self, host_factory, test_client, session): host_factory.create(workspace=self.first_object) From 6e52ec3d89f8e24dd7985e8cd523dcc8d451412e Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 20 Sep 2017 19:12:46 -0300 Subject: [PATCH 0255/1506] Fix tag importer fucntion This commit introduces a bug. TODO: fix attempt to create tags without task ID. --- server/importer.py | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/server/importer.py b/server/importer.py index 0b4a76a1eb6..812393ddbc5 100644 --- a/server/importer.py +++ b/server/importer.py @@ -6,6 +6,7 @@ import sys import json import datetime +import requests from collections import ( Counter, defaultdict, @@ -14,14 +15,13 @@ from slugify import slugify from binascii import unhexlify -import requests from IPy import IP from flask_script import Command as FlaskScriptCommand from passlib.utils.binary import ab64_encode from restkit.errors import RequestError, Unauthorized from tqdm import tqdm - import server.config + import server.couchdb import server.database import server.models @@ -45,6 +45,7 @@ Service, Scope, Tag, + TagObject, Task, TaskTemplate, User, @@ -59,8 +60,8 @@ COUCHDB_USER_PREFIX = 'org.couchdb.user:' COUCHDB_PASSWORD_PREFIX = '-pbkdf2-' - logger = server.utils.logger.get_logger(__name__) + session = db.session MAPPED_VULN_SEVERITY = OrderedDict([ @@ -158,6 +159,19 @@ def get_children_from_couch(workspace, parent_couchdb_id, child_type): return r.json()['rows'] +def create_tags(raw_tags, parent_id, parent_type): + for tag_name in [x.strip() for x in raw_tags if x.strip()]: + tag, tag_created = get_or_create(session, Tag, name=tag_name, slug=slugify(tag_name)) + relation, relation_created = get_or_create( + session, + TagObject, + object_id=parent_id, + object_type=parent_type, + tag_id=tag.id, + ) + session.commit() + + class EntityNotFound(Exception): def __init__(self, entity_id): super(EntityNotFound, self).__init__("Entity (%s) wasn't found" % entity_id) @@ -461,16 +475,16 @@ def add_references(self, document, vulnerability, workspace): class CommandImporter(object): - DOC_TYPE = 'CommandRunInformation' + DOC_TYPE = 'CommandRunInformation' def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): start_date = datetime.datetime.fromtimestamp(document.get('itime')) command, instance = get_or_create( - session, - Command, - command=document.get('command', None), - start_date=start_date, + session, + Command, + command=document.get('command', None), + start_date=start_date, ) if document.get('duration'): command.end_date = start_date + datetime.timedelta(seconds=document.get('duration')) @@ -507,8 +521,8 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation class CredentialImporter(object): - DOC_TYPE = 'Cred' + DOC_TYPE = 'Cred' def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): parents = [] if level == 2: @@ -536,8 +550,8 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation class WorkspaceImporter(object): - DOC_TYPE = 'Workspace' + DOC_TYPE = 'Workspace' def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): workspace.description = document.get('description') if document.get('duration') and document.get('duration')['start']: @@ -590,7 +604,9 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation task.workspace = workspace task.description = document.get('description') task.assigned_to = session.query(User).filter_by(username=document.get('username')).first() - create_tags([x.strip() for x in document.tags if x.strip()], 'Tag', task.id) + tags = document.get('tags', []) + if len(tags): + create_tags(tags, task.id, 'task') mapped_status = { 'New': 'new', 'In Progress': 'in progress', @@ -603,10 +619,6 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation return [task] -def create_tags(tags, model, parent_id): - # tag, create = get_or_create(session, Tag, name=scope, workspace=workspace) - return True - class ReportsImporter(object): From 000e01b95f9d8cf79bc6e378b243b771437529e4 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 20 Sep 2017 20:58:45 -0300 Subject: [PATCH 0256/1506] Add attachment model --- server/fields.py | 107 ++++++++++++++++++++++++++++++++ server/models.py | 39 +++++++++++- test_cases/data/faraday.png | Bin 0 -> 25541 bytes test_cases/data/test.html | 10 +++ test_cases/test_model_fields.py | 40 ++++++++++++ 5 files changed, 193 insertions(+), 3 deletions(-) create mode 100644 server/fields.py create mode 100644 test_cases/data/faraday.png create mode 100644 test_cases/data/test.html create mode 100644 test_cases/test_model_fields.py diff --git a/server/fields.py b/server/fields.py new file mode 100644 index 00000000000..895fbb1268c --- /dev/null +++ b/server/fields.py @@ -0,0 +1,107 @@ +import imghdr +from tempfile import SpooledTemporaryFile + +from PIL import Image + +from depot.fields.upload import UploadedFile +from depot.io.utils import file_from_content +from depot.io.utils import INMEMORY_FILESIZE +from depot.manager import DepotManager + + +class FaradayUploadedFile(UploadedFile): + """Simple :class:`depot.fields.interfaces.DepotFileInfo` implementation that stores files. + + Takes a file as content and uploads it to the depot while saving around + most file information. Pay attention that if the file gets replaced + through depot manually the ``UploadedFile`` will continue to have the old data. + + Also provides support for encoding/decoding using JSON for storage inside + databases as a plain string. + + Default attributes provided for all ``UploadedFile`` include: + - filename - This is the name of the uploaded file + - file_id - This is the ID of the uploaded file + - path - This is a depot_name/file_id path which can + be used with :meth:`DepotManager.get_file` to retrieve the file + - content_type - This is the content type of the uploaded file + - uploaded_at - This is the upload date in YYYY-MM-DD HH:MM:SS format + - url - Public url of the uploaded file + - file - The :class:`depot.io.interfaces.StoredFile` instance of the stored file + """ + max_size = 1024 + thumbnail_format = 'PNG' + thumbnail_size = (128, 128) + forbidden_content_types = [ + 'text/html', + 'application/javascript', + ] + + def process_content(self, content, filename=None, content_type=None): + """Standard implementation of :meth:`.DepotFileInfo.process_content` + + This is the standard depot implementation of files upload, it will + store the file on the default depot and will provide the standard + attributes. + + Subclasses will need to call this method to ensure the standard + set of attributes is provided. + """ + + file_path, file_id = self.store_content(content, filename, content_type) + + self['file_id'] = file_id + + saved_file = self.file + self['content_type'] = saved_file.content_type + if any(map(lambda forbidden_content_type: self['content_type'] in forbidden_content_type, self.forbidden_content_types)): + raise UserWarning('Content not allowed') + self['filename'] = saved_file.filename + + self['path'] = file_path + self['uploaded_at'] = saved_file.last_modified.strftime('%Y-%m-%d %H:%M:%S') + self['_public_url'] = saved_file.public_url + + image_format = imghdr.what(None, h=content[:32]) + if image_format: + self['content_type'] = 'image/{0}'.format(image_format) + self.generate_thumbnail(content) + + def generate_thumbnail(self, content): + content = file_from_content(content) + uploaded_image = Image.open(content) + if max(uploaded_image.size) >= self.max_size: + uploaded_image.thumbnail((self.max_size, self.max_size), Image.BILINEAR) + content = SpooledTemporaryFile(INMEMORY_FILESIZE) + uploaded_image.save(content, uploaded_image.format) + + content.seek(0) + + thumbnail = uploaded_image.copy() + thumbnail.thumbnail(self.thumbnail_size, Image.ANTIALIAS) + thumbnail = thumbnail.convert('RGBA') + thumbnail.format = self.thumbnail_format + + output = SpooledTemporaryFile(INMEMORY_FILESIZE) + thumbnail.save(output, self.thumbnail_format) + output.seek(0) + + thumb_path, thumb_id = self.store_content(output, + 'thumb.%s' % self.thumbnail_format.lower()) + self['thumb_id'] = thumb_id + self['thumb_path'] = thumb_path + + thumbnail_file = self.thumb_file + self['_thumb_public_url'] = thumbnail_file.public_url + content.close() + + @property + def thumb_file(self): + return self.depot.get(self.thumb_id) + + @property + def thumb_url(self): + public_url = self['_thumb_public_url'] + if public_url: + return public_url + return DepotManager.get_middleware().url_for(self['thumb_path']) diff --git a/server/models.py b/server/models.py index 580bd26547d..b8cc5e5b5e0 100644 --- a/server/models.py +++ b/server/models.py @@ -3,7 +3,6 @@ # See the file 'doc/LICENSE' for the license information from datetime import datetime -import pytz from sqlalchemy import ( Boolean, CheckConstraint, @@ -17,6 +16,7 @@ String, Text, UniqueConstraint, + Unicode, event ) from sqlalchemy.orm import relationship, backref @@ -29,6 +29,8 @@ SQLAlchemy as OriginalSQLAlchemy, _EngineConnector ) +from depot.fields.sqlalchemy import UploadedFileField +from server.fields import FaradayUploadedFile from flask_security import ( RoleMixin, UserMixin, @@ -328,7 +330,7 @@ def service_id(cls): @declared_attr def service(cls): - return relationship('VulnerabilityGeneric') + return relationship('Service') __mapper_args__ = { 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[0] @@ -352,7 +354,7 @@ def service_id(cls): @declared_attr def service(cls): - return relationship('VulnerabilityGeneric') + return relationship('Service') __mapper_args__ = { 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[1] @@ -645,6 +647,37 @@ def __repr__(self): self.username) +class Evidence(Metadata): + __tablename__ = 'evidence' + + uid = Column(Integer, autoincrement=True, primary_key=True) + name = Column(Unicode(16), unique=True) + content = Column('content_col', UploadedFileField) # plain attached file + file = Column(UploadedFileField(upload_type=FaradayUploadedFile)) + + vulnerability_id = Column( + Integer, + ForeignKey(VulnerabilityGeneric.id), + index=True + ) + vulnerability = relationship( + 'VulnerabilityGeneric', + backref='evidence', + foreign_keys=[vulnerability_id], + ) + + +class UserAvatar(Metadata): + __tablename_ = 'user_avatar' + + uid = Column(Integer, autoincrement=True, primary_key=True) + name = Column(Unicode(16), unique=True) + # photo field will automatically generate thumbnail + photo = Column(UploadedFileField(upload_type=FaradayUploadedFile)) + user_id = Column('user_id', Integer(), ForeignKey('user.id')) + user = relationship('User') + + class MethodologyTemplate(Metadata): # TODO: reset template_id in methodologies when deleting meth template __tablename__ = 'methodology_template' diff --git a/test_cases/data/faraday.png b/test_cases/data/faraday.png new file mode 100644 index 0000000000000000000000000000000000000000..36ccf51fb7f877e74048db7b2ea5daa6afad18cf GIT binary patch literal 25541 zcmZU)V~{98kS;p5ZTpOE+qP}nwr$(CZJTFo+xINq?nb=&(NUe5Un-)bkl7LPvSP4M zSWo}}0I(9`!ioR@fSUj23J^g5o;X48ssI2OcosrJ@)ANq1oDn{rWV#F008QcH5xAJ z$~$elr_u~CLNGi3M$RHBgb@79JOyQ8aCv?)hCc*^VFD3UV1zhDU<46Rq^~~_kqf-< zhfm+kYzGvVsaMFbD?-@x_(C>g> z8z^OjvbA7D3>-Q6ujl)3w@C09@z00LJ)v9YIz747(M0GQ%|xE{MX){`2zn@p=H^%G zn#dR?$4{!8(gAf2;&h;WZt$>H``4(`y#Z|~#EHNWwt>Pbm-nz7y;yPH4lPI$0`&pD zRrH&08Lc`E>F~xi+ZY+a{4mOZI9=dQg+i>~8({;atltqOCE=*6-&=ukRgU`-@fSBE zszBdWz!cEXjN1jO;E~LN!wZ+|%`yuLrtNDN&D|$KxCL<}dAOlzg#IvVvLM9DLEJV0=a*WZ=wxAb#eFQ7FM@!*yngXiEWF#1 z2cU5h?j8}ZKn{qhKTJ6R!drkiJv@j%EIk-m-z`u$H&ViT5CuZ)S2+-jU#~u|z9{`I zy&Qv5`UD?i@sS-~zCQrSZ_Op&B(WoTJPuOG^LJ{v~hTSRz}Ujqtl`E<4Q zH>ICFG$WY?GIi)`OjRTo2p3cr^p&X2xC_x1V8CS=WpGOb7bEcB)=N1WW8ucb6kPZ* zAgkf10D_-2t)sk;pRBZlUxTkzemaplD4Ybv1Ok|M(f2a2H9b|OpRiQ-;Qg>=j7a;M z--t20u6e_7H@KrIx{7CWNm;*n%7;1=*G%VO+OPu@c4odu)n=^e%vhkc0S{O7zu$i`~9P#Wc$H` z13(TCN%O&)0|4nkRrS&B!SM9q7sG`3K@1&;*1;JC;Ksql1|a1DWrMKf;WGzN=ixd9 z=-Pqt0uAm^x&f5*!MuT@{&J~@5kP<=5}Jl1AA|`LT8Kj~`acmUje|J`a}(%MU_=HV z6!^^hQs7esQsp14g{hAii9N%PZY7pk5=fC&n+iFK}`vr6iUgj zR6tkI5!b^Jk|{uww=$>B#H8_06(O<+a25D0GMcwI19ArK#M=qI5sNF>D@0$=!XOF- zCyrSdR5#pa7)TeEhARb83bPo2Avp8^d&G?zX*3vVxDdB{?GH74HH5BLSx38~xq@)T z>Ixj%XLnd+*V(4U!9)+68b~?Jv}bcDYRBA$#{z=oi$RLPj3JjHkwKGzlp&UZpP`tc zp5e?zq>Ee|yd4Dpn{+RB*YZaDW%LE(o0A%$nys3)n!%dPn$#NoLUavqEwxz(0wfTg zFJVt04}lrXI7m9kGpIF)g&QFoIh#HkMVnh2f5W{)(<#weCrqReZO;EhGKvTt0Xw`k zyf%#gH2viGl==ktbpDj`Wb;(?q=#__QwS3ZC{1Q-**=HQT(3$;p(_zAM;3K|M-yQH0Ffnv}dNPe{9sOAAQm-yDrQwH!njk zE^p8;`UNJ@O6oJ`G72(kB8V z!ZLz9QC$Ldf_{Q(!hlj;QiV#mQt_gJnYxBdCZRSFFJWNZd{TH)d5Sx!B#9+iCccN@ zQ#Dp*R+Lt%R_7+FJG{H>i{ZWNK?|x9Y9FcviVj60QZ-T|QYUgsQc?0pGDT7@Ibzau z@_Z6^Qi^JkDuAj?nM^rD#YP3b@}30DysamGEdpBbfD0nP9fhx4!Ovp zIHPu>rlYi^6t&DnxXT;TSC>uAKz48@^7^7m~W|XK2Qu09Qbtj zWmsycY}iFuDyU58RajcMT=)feN=S`-3?(WGsRFcuG9k4g*N`0eEjTx5yNLXVyO^_R z5|L1m7qNR0Qjr*8DX~*wxe@b`>~M`@O{Gi20`=SVSavkyBrXg(L_dNbn?PMblfgLQ zwc#oWX^8}hpb5o^j)^8pQ_9nq!q<=6>n-wjI;XuY<)P&!3;9evO}$QgPuCCWP!CZb zG8cVQf&z%=HlSflCV>x_!K6}D2-muKatNfRNrLg%him*_}6sIYtwg;dG zyrVxyX-9S_Z|L0Y)$Z+AcdiEV`}2q`#AUIDaiVy*e5HQXz2OxUCFY&$t?k$C{rBC6 zd$MoY?RtfZj-AP^%)QQ))1}o#+Qnby@1i2&BTJG+lc`DEC_sqBF zUrT&*^0U&}EPPw4G^#p^K{XQP@A8+LUa#4s9ig1HonM_Zj|rz`7r2+7*vr^hSRPn$ z+WvIIdg*o#mRPo|7B}q_Uia3}T2eox_LM)1V^h6pV|D6!=UtT!&j&0ER^(b4T92Fg ztw*gRFYzxGaB;B-vQ@LawAZz(myT<5ZqTl3yZSv2#bK@FU1p!AlQSt=3vFngL0)B_ z-XFT#u6^Iddj)>x{hoh!ahtf>Zn1WIJmg+=UwvvmbKsfrn)#Hy)>+Fr*FVIc6)xj9 z<+qHU$8d)-hjfTJiFt}Xih7L5jNEe7c|G2iZ_dZtr{0E{>dcgo0g;`O_LHRMkaO$$ z#@!=4GF~I^RVJ)**H-LE_L`+DWOm%G9$fEomb%cFewOlf$9nhDCw}<9y=@>bCud6|58-$f3lTKZsqu(X}^vNZ~6ZRpUwkH0*BvL@SlQ9o1G z_K17y>Q27@ekt^oHn_f)ZOv^w55sPYZj>IGuBAS&PF;szUz;uzrm2w{{^63eZC#B=-0X{s81c| zU_iTviO`_rI!Dv+x!$YBi>k@o7SV_XOX80=S$}RL2s0ONBUsBl6ccGR@M!#jO4{hk|rI)8g&@;iMW-a zkcpOTlth!=mG)?DGubOp-eZ1t_B6KEnWLVB+NjH#$>wp(Cog-`A1BPwZrz^ob`IFx zKOx2vtHar%KH;PF5v&z1P;QX!&@K^dQG71eOj)IDS_&|moRwX!U&YFOtc~K%Q_Nek zuWoo4>?ln=nT=}G5~cI$C30u^*tE|w_Hq}~FwGY} zaDKh+83A7pD-1e`0}3_74}bCZ9xpH~K zE~1>RIBpxd@~9rr}#3(d6ZX;>3GG_OO13b_WZI0E!zN z3ke^EG%P)WOmaaYR@^&5KPfq0F>Xa=PDS-cg6gq?zEZGqZ`m|{TWwD?+oMsZKs2^9);Efr1mdb^0)6FnJQ zF8)>&E|Je=^KB{cXg0aZHqKsp`}$Z7J&x9p4VW%P5z>UwwdnmgsyuH7tR}b`hS`#r zRDFF6-CX0XWbL0Z%_1Xx!>-7zlRMA9E;@&X~boP=EWF}Dd)a%&*0Vh ztn%IOb6i8er_I(S>g9PlxmiBv956qBuVfr+WOV#}&y2G}Yv=3o<9J^?P#jd-KwYG| z#`=0(8hxgjynR~&awPz!j%QM#bO$gXbUMRehP@m#QxIMTWQNlTMjYB61p6!A{>dD6 z=i2<$Y!yuIj63Y~bPOXBEtVcBjyip+zNvoO&||81Na8Tzm~|iT@p_-~poemX>`lZ> z{H0;6;f%P!A|w7Jk(;zhcTGu52R#uxxiU#krA`S&QA-6&xmHnJ(OEuGK6HVMHIu2K zS-YvYnZ-HO$>JIDp$hf|_9rw*^fY=&T3hCLn%kTEXB~^7nXu zD;u?~s*A^tMY>tTzT_>nr@auN_$Ts$-TUVYJ!OiSE zPs3*gd@hV6&m>kRo+~aU{-a+bqpx|ORpWcez|{R3rc&!|_AdT+J*S_yf&PStizE4u z-|4vPDo8aJ!09xAJrA6J5?LTLAVQ7UDm*U0RRjRL6;OH;4nJEV;1TKl+{XYoOhSth zUpXl7_*H<&3A_{q3@1zrfmx3oJy$jCXaDEEr#r+?5op5+LShzK$*a63~yOLmO;k?HZtUrdlMI$cItpC zd9{=ck~OS#+qI_~>UtQbJWf*%Ek~c*v7?hC-4n*P%#-6C?FQ5S6u;^V)2kaK1cWqv zCjz&a0?Z5yc2qpS=k6nhi>1U>oO@ev>G0gd6(y8c%!@~HX_<@ZUnZeOJd-Nia^7)! z3K}=oy?TOAd#n9rg%2ZFk+j&VYz&H)KVHfP3La&X`^#}mua#bwb;~KMWz341u6oSf z!=CTgD<`W@FFZN9Yp*-zeI2W18~6_7x4Q2^n**r@x%&wV8)eLM&gAfYZ!eopl`fg| zLb=x;;!OlUpkFF2!)@9xF=iz;EQT`Ar=y}!~RGldUtSz0Y?O86rh!& zvL90P+1knq<7XDyR!eAVICWvYIDM(VtyN7cW%^?9z zOwDLgS69E#c-A1_IM~ow-(u(HFqkEyg}3E)W0Rvsy5#76{6Ycb2)GUG5flsd5l$Ad z5ZQ}*j#!gCk{p$Mm*g5xpS&2an5v>Gr?jNRq}-~!QjJlbQm9*UukI+RDMT*uQn1_L zU!vbKpBCO`-nj}1X*Ee%DUIo|Y1j$U!G?2;1(Yqz&Ld`capb$acWA3z^Sx!jSJ+c` zY}-@yx7D|lw}Z$%!1AQEW*H=FB(o$Q4cbH;!|bEm_ed0c*iFXv@+-A0F)4*NfifYz z#plY8S8~7mcLdZm`uQ*j8uSs1)V1xk{4NHdqh#)o0X*ktzYv0(eylsTVkSU$n=I-=`s)@KM352~5 zf{U#Knvkb1R>!|Lqd#;9wqZw4&yrv*AvJwWO-4Oc&0<}91@fXBiz3T1d#jzZea@ZQ zeI;IPcbA8dPsD|QYCwy~-1s6LB@-DZ$t@~-B3Gg}<(;?_Pfj^bp-D*Gsa_)X+|=nG znq``>S`JrL+ooIe(^=4?Pz1TlJb0B6l5&h|g`EY-70F{??!4&Wd#ZU>d)v_X*~qpc zHsiKY$H>oF&!J9LOsh;?O(@9Q$h66AO2kTpmnENLoc?8Mr%1(GLmx!iJ5BHz3%56Cxt|fU~^(d}1wb!2Ao`Po}DJAVk z_o$N+B{h0A;yRhuj5aq{DrXkgsO|D@3BNxruZK2mKR;ZOZgacjJmbFY9~9na-}SHK zN9kKhW{0zLH*!=wbe{L;DZ7`0Y<0Vny;50txrjgU%%A4EZ$^VTGVY*l4>w+GmZR`H_)%6Fd=h+f)rV_FtK@9atdndftSfBr*1Q`?yM@*)b~LtC z*5o#Ndl7!t@r{eK7k~tu#3bKuo?f3-7y#`=0IYE~c6DlYc6O@-VbN&-+fcuserdRj zXcH@WIO_s#ilU6JxxXIY==$YtT~2#idjCoX*%r#`&gwGKoJMvwv<}x?wtRoHYUyn1nxH0woaVxJVgH~!TE3gA7nZrg8vk8w&EdDmystBvU4;cV4-E8 zWhCN-A|N2(b~HBSR1_BdZ}`7A9wKvRXM0XMIyW~rS~n(IJ4Z7*1`ZAmI(kMrMn;-{ z5;RU8w$29bG`3E}|E1*r(<5x+WaMaJ?`&abOYk3h4GisEoOy_d{=?A!jQ{e}#NFb5 zE!jH#cUb=dr27vJ9Rn>r-T&zR2j%_`DyO`KyNR{Bu!W6@tnde5mCb#4Zx?ZkRXNnNt8tmFRXW{uy4;`F(siV2Aq?cg)k5WVnRw!@R!JEb&5|D1bs(N zEA%%jpbV`nm7fe0_CdxsKrQ!gu9BxdN(mz{c%a)3kkN-2`Brcod}pQ;Q4IL&ynq#^ ziG>u9r^=PLi8GqF(#9x6hHo|IPeE!~hAkv&k5&Tdo!ZP{s855r$omY*05ZZN1;CGO zU)5F$oQYDbMuLk0hYAQu)VR0llEnd2wL_*xW80QJa zLQ%DyjtuE9Q|`g^5{c$uQ)&b$e-cS|vO1vrBfN}cAQ7roz9QHWkY%7f)M5&~1nz$D zCe*AygDtbHsR^O7pbTE7SW1mTX?ZnkR-sa)$Yug(A{O~CrMOtrz!B|2O64WXKzhpB z1;|5!w1Kjql7RnLc?nk`i}2Z;1heoF7@@)lR9c`#ul=BTy7c?>n>W2YDzbzMv*Od@ zDlA4xa^0Qp%@KZ?z9hsJh%bJW>NYd^8Qr9vpa=kW8cSYsiy&!%VY2cy7IR0wzXCpQ ztvV#0On#h$#4k87C&{&PfU5@yEvj)6R)T6o&v+x06ebmciaq7AWXZ5Nd5KF&iAdz8 z%4C3S3Yp;J4y;O=;I3^M6t! z*}q9-swwH>m1Lzd0a#1 z`ku-Y%yco{&dM^uJS(Z>k>@{aa~?C|Qt1c{#o_p#F0Uq2jsYbVk?njRud;sP&PwcR zEcmT@kmjk7_=BeFKPco~w{G^OXsZh4J>s2DzdkswUB>V)b3P_Jjbbgi4J>vBJ0?cK zFjbH!ZSEpYDB}uAP+yt}0yfd-u-h-u-{<^lLFict?<3@9KIXkZ<^(#1H^pg2yWF07 zPvb0C(2x}jRMm#IXOBGUq*~ipaGL6E_rH|Qs$M$jqe6nCiPV{q&RJO^Wy3V|u2`U8 zL9<*A)sQ9SwG|W;zE4pToo5Aqj;r`of49m8yH`=fARhgGik7oCJRDesBY>Gmgg9Tg zGH~}^C+JXqpY^N+Fa4o~hDMId=DtVdh8(MAaQy$Yxp)oD(lop*DOB+!W-^i2BJ_09 zb0>a|(3K0}NR#xmpO1Wvi&xfYz!qu@Y_5P7TSmNvfV%7;h3IrunGp}7w!tbJl%?rC za^2Q;VTy*L;jVpUsue!foMvmZ>Gx8pTzeZnd?Pa7k}4{5`@GI6s!T!ED$em-dfZYT zyLwVa>uVy`rO)a^TcV~#K!jnxAKi9hA_bp0ndQRNe;E_Q5Rlsk)U^#%$On zo#A?F5<^};tE5Azrl*g$&X9vVSANjjkzCj8UK9;W6wc0=4Y)*=xDB^Ub+veOFavZw=E=M^zR)?DviQ{(h2>TqM+?NVn{$ zu1=|EZ>91G_5|l;(8#?jil;gf`~4Yv_aPS1QD@LX5LJ}S7N}t7g#*K_C|&^$4lrX1 z*ju5G3}qGFD)v`Ld0y2o?c;-D+wn>&&ae9usq?l7`&s*aI9)k`S1K-0yg#fUDZ_tM zC<4Wyg3=WSjhG`bC*sJt3RqWO-k^x`>J=&Vw?(=?sDbKIvY5Ru-_b}$fH)ae&_TA( z$?4?6>ZRt(+VuM*P@VsrUO){oH>N!qdL3#%J-l;T@YmNf%|v;E3MzG@2OY*4>>U0J zem)|~e`sZO`!3*>0}>c17Q1HBU}$q?5uvdW*!4n$B}Uue?!~WG-@F>CmPm9>LF`gd zO`G=<86Pc`%t?M1N8CiX8%UZWrKoXGk+;DsEM~5lzd!VhzQ^)Fb@g71q|aI7Q1Bk$ zdup|+zlv@#w^YcN z;&okLMov-kx1V4tsmm=sMA}Hg%B+fv%`8zx(-h16Z~u_L#qa9$kjHmfgNSyhXqkZH0O)cCS66D1k5CUvHw zJPK6)xXl{1)-l3cAb$8>$*0o*Sy^ak!|x|8VX6-2YYYDrJggeLl3k>v<(I_5`kY50 zhkT@uM5$O4!}m&JtqVyJk|C|c&p#<#a3ci^dx4jTD_@+yS>z}m=1xnR9#Fd5kptoz z=tC*8)f%e4q9%)}y)aT8HOmyrOjy|BkMM)wqp2c_N9MW@C_?SM08K##ZPzNN{ zJi#OVMX{hvhSV9_=n#1C*TA5!8f&?rB%<(-t)!|XGyqaq-*?Y_M^3_arxdM)*AfTg zPEkj5hO?6Rrh>~je2ud2W$Zv!FZuxe%W@MWO5k86Si&}yoepZ_h4(x&(P%e$cG1}% zHMl)z-~}BXN1Zz z$ysMs_SdM$1sfj2m0}fKf_ORV% zxijREgH4f*DC^}I{tW^vj_OVO5OLCBPdf`tXO<1r=U z1ekM*8oY-{G1q}52I_Kmd+r%Y=FLxu%Wx?3q}jEUr(dcicrhyiYGq27w(_}(`g!LPQM z4fGUGLxOm6m~9XT4XwHif#8n9xPrqWeX73^atoz_aW!rkLH!l#*b&1X;Fi$c-oL#A ze7~IEVFE$(P|eT?-3Vuu3hPK>3 zk8S}I=!vi{y0hN0bSilyq^r!DYb_r`iWrQ)hHea5Qp;@OVW)h7^l1_0+monYo*ma+ z?JU-rMdfgbve<(VzG>;}ttL-(KehW>dNRxJGRKXQ#0QEEJ3REvu9w#r;_io#d%ej( zT2Lv1ytUWn5Rv2A;e;Wm5_ED!^Jj)8!^)#~1cFbL1XVD)q5-P^Dx1laK;}S9rv!)U zl`Oc^)rOcI%S~46zqOwOZdh`YgZEV&^tm?M|Sm-WU9yW2NX$bk{O}P0Y(EjGKh~ z?rp3HJcT7cB(G4(!0n9+u-Wyd9}S=VdKrs(e_j= z1UUcP$WiOyC<(|@tdf#_p~pv)cb^30-p_1d%W0COE~E)!>ZIW~e|MOjqt=h%3hS^8 zIW)o@F*0bL-ARc(C_?OQyX+5}v4kd1qeKSr_2_wvgGLZUQ=X6uK}YTr70=fo)485{ zd2akf)6{BL5G!KrbT?J>5-_P;|Dq`(C^LgZN|B(+U1Rs-al6KA?$VEiA}ZRjUF4xaeAks~4>OF&qQ!5%ja7v*hnCbau<-PV zy^^ma_Q@1TWtguQCDtB-uUHQKuZOHMFdL%nh}8`I!}rBX zDotJ^vmJJXKBqdS$ufQxiYkmru;B@9T)F{IdKADRRtT~}5K*JSJ(f1~EIE#diS_!@ zwL6@Ko;w(gA(9y+7{-PYzNHicPb~a^fIJ!VfH)5JYvsDpIyzBGNm7Su9q5Y^MS8oC zA1EI*C8Q)8Go1y;VF{MprjjGKE{Ue5mlp=m*#tud1xr?Y)AbyCAk=*ZE!ohOGQRaf zoxRo1`YEEUKqPJY)&uhv9)=!39Vh8%(RDP10p?*;!$!JvxN%Bd3V13))EFe_N(;a= ztuADs<}+ooo;tcuAYHm?x?=pSS4Qowy!FIC8Ns1WVdvhg}Y>>SvTjiZ_mIr=3$shEWH1Y4u9^;R3@F_ z<*(rTLb*Vv=SIyR+2ABWGG+yOEVXoTM(EM0KdsHgiZ{64D4$1O{nF7_#tlE7 zP8M`3IND??UMf80ABU!p4YV9~y$;*|NjZ*fHai z8MB(IdM>@c7Db39UhMLojYi8|)$_miPbb40A9cmUNzWPXwLNh%mP z)G#m&B6sp*sYTmLk$fpp7J_%P5lM`pPF@H5w-M zEh zt2+w@(V~VW8v$GlOMn4F>-*mb1tldSgf2GH#Ah7KeP?0TRM*` zLBXxdvGa^lq(pR3=o8w|$1 z!Yqp;co?JA&H^yj5fmX@eQ-C8FF54@GE3XCS?;_ga>|y|z1&Y^vQwk7VGLyPBVHkg z!Y%;ZhJj@i?QD^m-mp=hAT@`T+asO|6Pry}W14v#o)R_e3666zXNG+f@npl?vzOo|1gL5Y|&XAiQt z19i7fQB}N)StbbGiM;Y)#QAcNYrShBnC2N$+n62v&@=1Smm`+w-VQ9MCAx;*)kJ5@ z1faGmG6<@}EvT@V!Uhw-0pWQu{t}j0jpdKuQ(y_8sX-3j!(t||xljwWO!LQl-niDw ztJN=mfN%d09X>)DQ|R#XGp8CS1;F!QEgo4C{YVUlf=})COABAxW#RytqB2|o1iY)r z4nGdK5L+Xe+ znr!+qCWjJwRGLYD03zfVd41Y2Lb0DaD>ZRn-+SdOuGBH{Kp>xWa7EZ5;!|@fu6Hvb zE<9hC3pn_XsPB0j#hq$55K>r^z*@lkp+0R8V33c3Dk6poUUC!Ib|x7jgDNhh?bss@ zVgiwNY`OucL#11aVgD5>1&gU33iZXiYA4pu&CbzKpWA?@`@ptCv0OAp4NR zq@$K4^dQOLy}3rlJ6f0;-`@Ldl(m2uY=UN?{yjjKJcdjYF?4SI4rEcx@Pd9&u=Z!9 z62~Z4qHOcCo!Mkevhh5eMw|G?WZNj8frE82PlBipZ$Iu|=icBak_KoB*Yhab) z-jeY$knlf&^PIaL5zN_|MEwLay!d8lOM8)cUTh?>sumjHt|-9%qB6iPS?oB0K#%qVumcI4L$6A5+SwKY$ivGp;PBpc#|) z(a)SX?D4XqPXRZG(KPm|Z4(OX+@5*D~NVu|uH>CWCtSkpY z3B0G_P@RcCt>#9=Wj4}v4bF|PNwr&D70|+e7 z^un&@EVt=*67lTT9{!w&JJ$@IY%nwV7XZ1i>c5t_G^AJzFewJxh^7q<3|zSu`&gwb z#70LnO`l*xUpu}=O9*3ZQ6y1bM7LT=!T}~f=PVBDgaS&&8*H0uZdT?*pA?IZ3U|gig2ap?IXl9O-gg$wUVjkEFKsP zTsJFHzlshFX~@;MYu%5>_}fXFpNR<+utST`W3Y=Q2!x=%+L9QkHQO)b_%g190Ox{+ zlQlEmGLS>EkydVb$wX}Je(jUXlPh}=NhaROp>Y0b>&3C`f~HooR+4wvu${PL8Va;1 z(gr46$-qZJ@ZX%;gYN&jPbv8g9h&SsvD;#|p!&grJ{I@yJt%5h`5^~ER<(JJMT-gai`Rp`hgaCET(z7)3zOWkuh73;VG!cLT;>#C&V?a54K4K@3NnGO0XNaT>93e;8{rm^ zL1XN-A#gP6iMBwjd8!D=pyXl&q&N%WVXhJ^xucHg{BC0>>VI^CX&)L<6e|ARy=#uCVEv#mqT`XcvU_q`bX#!?HSHw>3IWEuW zjWfHB?ng+u!Q}U|vz(|~5FJ`HyeY=fw6NDx5Ay~P zQz`3kCkL~g+SFlZH6he1D#v0@Ikv&EXDecrMKExM(l zIL(EQO+_3%K^0*|UvEIZST$X&Kv=vs<2R-89EE<3&0wX{3br6jnNC9wpEmY)+4xhF z2sJtC-pp96E(S+wb$p#3tI@k&AMZ=tu7QJ?eFjfy$v6;Pj;pql+i444TT3nie;N+V z%^Ge|u_9sxD)&v;oEzn&KRRP^9?;&q+;;*+(5c^mI~&EE=TW&NE`1oGyzQ)@mRa}~ zm#+xrPQ|r>-V~j!GnO+j)9K%xCcaM|a5)QOLE$W#KaiTnvKm)`0rTpVA$%3Feve%3 zW3G0ZDwpA1BLSv%P(TD>fyqYUhm&5=#Bf@^99p1=V`Fuw0T}f{h%oi8{lH}+Z1!tB zQRYcmm-_}pdRCWS{joz;`;#_Lm+f6Oi)eY;2(pB=E%hBE-#JBOrgmkj??FYb-QGnE zjMMt|K8c`h`Y8mJZ9v9%rqXnKIO|koIsu9Cia%xffU@X8dAK-T2g%0vGUw(NexeHH z&LWB*Uaf58v^ISEDD5fW7<7S^h^@S?1x}I(?0vAr7I3`&cP4V#Y+Re~*exeSYK;yAkX=(;#%RT#$gY zk2@O#)EXo_NZG*908d)4@Jqg*PGXO)WUT{a_e2b43~WhKvantIsD>Z%c@t{q8K^((r4Ucv6#kEn=^TyVP&Vt{kFy;DwMt~zJB z>^gfsgZxxGos}^=REjH#M6AO0gl`eDEOLSo)&R2oRYrlgWDRr3X)QS}nOPmgG-K@} zR+goXIu1oMgnHuknRRJvO@yi!^+Z>uY*KuTMj#9WCHnXgonU_zbyigh*K6$a;^pp zVhqJ+MUD}&)L&uK8A=5PEcI_E@`7Wkvy_*{#u6objo6*4!(B?m=-|GYI}}fq-g=oT zp@$J~X(GC)Yt=);blg~gs<}$;!T`L^i=h*s$_Ys1FD-LzERH)$64 zB!$pUr!v#NI49noOUg`?LDUrCHYxVZmMnWJ)OX`hg!mb!(>A>jaND1K1IYHQoAhjL?3m4df(E(1of zvtYi@R!w{0;k?GC$pbvOTKwSWW7e3mcpes`wLL^+cE$bzjdVcZzN?l`zqIWc=H~QT zD@n5qrKyDdXnV0CiJZFWhp>QP?Ha?E(12G^5)7I^$HAiAe@KyvHcDJgi*>=cisXTw z0-iT4L{*bD`{1wua@U>A0gX`|$#Hxr9`%5zIYx^TRb0Bz(IRJy&O-?IQL+aHasg_6 z@tc<4}&aXH{Gut~q=A?&wQhQ*Zx*No+G&8%W;|Zk-4f%9Z*oc|fw3 zq)x}NHb6)u5A$?@vIRO+EFc8)XIwDT#*T9iN#~4!MiC3>TwgR-gB;WX@7}8eeWk*jKbg4_79!F1WzTC z&bvFbYKF7f5Hv_blu#@Mnu%P@gBrXGe=Im55%}7g0UgjBPJEs$4>kOOy|6|(Ott~e zAhMl|Kx6i|RW!s`ePtNIRlC_)L-U0E4p+}q;#LR5h+HoPjb~s~Sp`)fk2HK-V8z{< z0a{C4DEGAv8d0}5Vi)~`UK_)lbP^WB3rKZ{s`8Rk_usn56*428e;(Td8Z_Y-$ozMdi z98{5~R);63VDM|D*zrOQE)p)xp{xDhunen?#5!RW(4?R;toQ7D4pR?kYx7Nj!~{p3 zgvJ}HKJg~6HZL0|QL?f@Nc@^;*`v~miuBu+8)sD-C}BGW*aabEdT<&b?gAqhKzjM| zD4KQbSz%$1C)3HBR2|ngS#P7(*r++M&L^bKizj;zRe?BN0yBX(H9*M$PkX?_I67Eh z(Q;4*jcFKgG-<6)JkRp`u`TRjD;s#>_z=-#W8L?-$~$Oz+@EV=+}Gd;ruj+Wq5}{a zd+4IeEUe?hSI|h47t&RulQUmzOi~9shOg03IZi(PZ1s$h4oT3($Vo>-2mk6#MQ{c+ z%L7t@{sdNG@LXh6-+5%QQzIlQ@k|CYL7_P`I{sTFhg35KSFf02<(Q{DXex?kF%x>J zf8Ke+VNkQ(q;0(kwYhQJ-gJhKO)oUl`LG<}%-3mQ+TaL~+%U9@Z3nt*qkgbI^vRDg z>_n54>ej{Xrnam^-QPKXmv=WN#I%@4hMi|@d!7m@IT%t0({Yo8Ie(8jXCLQ=(bA2o zb_%93@M#qZYoSdS;SRDOYYV%P*kWBlg;PyH8AuIa(A+46u`Y~v#bFWHu%U|N`0#0U zQzh4LkjxBH-DDS%LfkwZ(aiw0l!P4ZDFk_1?SvMscu(sTa%<<#K;NM%er$SM%zp8O!@I0F`@GaJ=?qpfo;4nCx6`GpCIX)0nK8!9tzGY`>HlIS|v{g_+V~MKC&{On|`0p2B z-y>~WiqB+d?2+sEY=8HqM}CA?q3U)fBx>WXl#X&~$*L@3!#JN)2C(WRlW!b2H6aUi zWZsB4!W#;fRiWUxe*FmmBC;@x;g&Iz*QgbF(LL#gp#q+87V(k2!*-%h8%LUli@6df z2pug+I^Zk97`HA3x5(lRoh=Lh_y0WcM_<#fYa_Ati-0;GePEPxWi8pt1|VgAX%3bo z%@&AzK+ij3PU~Fpq+D4SDN{a3Vr`XGL%UNiLG*;kxmtx%@He@<(N5 zTR}TM#vHY?MFdc(JFQKT@c1fc7$Lb+vI+oa%sLm%YFf-2U*)lmh-(nR;PAVMQ9;3U zbc8#pUtL=>4;(&gY_)E;~)m+7aWjXn-v0S4FFsBb;LnNv09Fk?O^wOq%2Sc_|M&hbT zCqXBe`!F2IDI=ZxQ8jumea=QFlK1}AKaTF$YqA9u*OYPwhN*`fHxNAXNFNVY)wRf} zEv`6zEH@I44~(*E=)HEi%E}i{e*X|2RyBC${jX~le(XK+oJeq)0&qu7amaPN#IDm8 z)D5RkzBSbI(9?W@qOt19uyKM|(x;v!56*aA&+R{B?4=h~BP-wy11-YaN~G^f=M@EW zLJQcR`WHiwyP945eB!msrB$`2+PES;sU$zeqL{_nFRF9Id*ANQD7A4#iPfE)G=|iO z**6D%m%uM7=F=~;kX9QSm@_%LutItH6vqjta2$<;I;6;Th-J<}7}Dwl)XB2z?c;oatsZB+CZUL!^%syH7o=HRVgJlTB=yzlE%#b2rh) z*iG47)JGQWIoCK0)uc~&2Vx6R#(jf>-Fus-)fr6UQ&x;#J^$pR854Lj?7pYg)dixe zhFxILt+q>E&lZ$dkJ*;P0wnTU`=)uF>8ZpT;f>q;mMv^^%9|P)qL4r`RoJ*W0#+qs z6~GLkLu-*)?FEjMibo)|lC>MgM4e>%Rv<7CqB?tic+cSWK4fL800vXs3Pj1mC!cra zn`u`r6_k1$ouWpimt!w3t6{_1L?-QVBpifl5|C9QE$&#jtnp?cV|!hqz^uFRgr!nM z)mQPe#vtoH`p!qeM`hJBqKAkDXDCgNwXPmR7+YN2un_`4HC=>pb^AxYu=uvGr{pbKxB&!W=jj;g_=tmGIyCZ| z=Wq>7M99GqT2G53O1cs1{oy_KAelj0oLakrhRxkph-`gse^>uNasjXT5rc)qhIj8X z_y`hfETb^gXdJq64(S&_k=%ef%a`lPXh962t0YXyf>V zlg5iKWKN!0w;9&}FyqaI5p0fglP9;V5g@cu{9&P$=9W#NN1&FAIQ&WziwPAA7``|c zTR+sB4%#%FRT_6qPuAFylj`Smnu!+~>RE4jyv-M*2yGfy$UD%|@=mnC{i_w-(VzPx ze|Kl9lI*xLscYW)f^L&eYV6XYrV7?<#E1IRV?Vynz+4+QrV0%@U{=kShY$+UeYZBhzwl5R=FuD}2VS-}r}y^ca1~nLNjNS@v9_fpJz9 zQ^i>W^QXeRSv-s_oC_XqDK+?=9}x+I5!YNts;UTeM`-S4lgLQBtM6M;VO|x1l?fB7 zfHw@*UY`*QD;IAG5#BJb=FVv1Gy^dQ(l)x4MT~8Qygb-bvvtX&T01}Z-c(noj%mRh z$w#i9uySa94%dMF6*K(RGj2@+yo1VcP+gB8sz4C^=h01R@um`-@N0FUpmL9BJMJo zj-2JWTNf!)rX5#jKapxo-VYdC%uV{b!8~9+6Z776G#l;TO3^8Qi|mFd2*bpaZg%NAdVREgPv$&>Gm{t;|L7>pz!tGV+ED|(4JkhWpU z#*4`?SD~jBe30sX<{s6=dY<@@f=;~{RcqrF25EBnCq80w_jiAc4c7Hu6H9e84jz;f zD{uW|C#9I>`vhWJ`Fsf08pfE~qUci+ba*F&s~qr z`-8X0s4Eh4&sqC31svDpzP;`ts5m5tG~HD}- zwVL2f{6HZ=$Wm6q)_ZA7tfkSK9>oa)wZt4Pi*O8rA+9hwYMf$? zlvin(pqDni;&>j6M;)`PpS47)=K64XJW2vg5MovJj4LnIljpM9grb;1z%?lL_4;e> zvSeFxyVNYBas`|y*mYv>DakTBFO1r`b9_Vu#bu0{NJT6vIznHU#xfDk4WKcZVR$eJ zxKdyv*E&+9l#-GFv`!K}3ARyM8;)D~F19mwede2cU);wlj{q`4uHfO0XmbtzW_^Bm zjJJ|V%kocGtfgaQvn;;2I=V<*z(!3OHX^MJg=$^3YD}Q3T?!qS zlH*wDvPCjKw1E*^f>NCVA}n?w0ue?rx2I> z;2LlXSOp|f;1FL%9i_w_7~8bNwqSTEMuaSUT8&(2x@5OVJ}i97TDpsHYXC%?7$YR? z*3d@cv)}Qi&MPiq7Y3o#DIx+#qAZ@nkSIb+4~{*zg_hOj5?Mi2E5Q-(44x@G7@t=6 zvMgCG41jpHN#3$!NXo)bJ5kL+fS84i-o32;PUn(C5mI6Q*oH$PMnMKV|EWP9$V+V4 zK*h#o5;f^6O_`K^kZw$qBBHl**$hcoYexqg=i-q_qsc~QF36gSE%;2LE2$^^JFk_c~y;?}BwBH|8k%nlWS%?hVi!=i(E z{z##k5m6YJ0}fZyid3jexD;TC6HZ+G^-mzTo?Cu`KSc@vked=u1ri4(%Y-)gq<&mm zTNmHidI@hNW$^_LtJHiaJ&v(AE}VN%*oz>Uyl%61Z^PJrC+G-hh1>eMT^x!K02?^8 z!f1?4ov;tAkQb9UfNUde|G@*)iuz-}A!L6QFdy73Qw~!UtEPmF&7#>%X-3n`m7r)@ zHo}xL5G~G&e>+?+ zebAYgwF4Qz<*&dg)QrC7zzA1QtiG#D%$0EKqq7Rj`nottvTDFTTm!sfr6y|Rl*W;B zk?Kj;1SwrqJDU;m$z^24gpb7XIPLhLzxrL76Ia}-(7N;}UVjc^H1g<+&)B|^_bg6W z1UQ@$o8e&^HUFBoV^C;|z%?+LdbkhcQ|e1ivY0OL&~Mfkv&CXI%aj;#Mrs$Xsf)Wv z1UOCs3_e_{lv&F~EZNr1({5hbdHxDoHY_v%O7WNCIn*h}1FMM(M$;V3=baPU7lbSa zUcAg$l}rLg5vw%^aC!Iqq2N654#aLH!>)2=V#b_<=~yy1(IQSVdQ7NFB9G{+xhGY= zts<_$x9)I7h*WOxFyz6C$59+z*Ud*Vu-0B*UmQ$X9{lE+(=Lzh>v#M(K%H;(PXSkt z${?|oI0zVkk>9+q_3wTWrf4H)IORVjPP@x$;^3fs)|rhDzk!99aAm{s3Vc=xRsjPH z9Q(QVvt$5_9nc5^Pl-RG#^JIDD{ed>H1=f~5$PKA5u1?P6tx1AVfMmc>@rF>I|P@Q z1N2Wi@bHMaf0e0Z_0VXTIp8r@Ni_#jRvZES#Y2$Q1e_#UDek%HcAE)n=A^)Ix}?ah zOOJ~w z^fCb+8mtvb1rAHI9L3VC3Mta=Sj?CEc+hKlvZ{c*%B%^_8=Fu4-X|Ace>DZ!_}umF zzaQP>ADVWkCV*X1C}L`CFqI)gg`&}F3Nk5>_a;hDlddKsJ8#^7ep9$?X*|-)96-k_ zmgu>bgKrabqft#3Ukbv8MeAB}U!N*4*^VUZp$OxUg(Kev3Rba)Vjf1HS!avnR7o%d zm)(*QP7ebJU}H0r^PohT--)k?(3vx1(c)!Flq6-wSnOeUfZ{GM%#uqi8)27Gr$yq= zF<)SRVZ)Qp)x?IA0DANMhrY_Qy+$a^LpUC0V^aE5vT_9z>8!IjrL-JaX%;q}jhMpr zU13>8;19*EtELH-<*Ji!Udi_z8EN^jO`%_`=aFwcJ4`y=Kg5}iNT&(|SiHWa5e1vj zqEd(;C6~^Q@8@l86ADaD#LWuSP(`lPxs4^ z8@Tr2yaS7?IneQ3UX|O|XB8lfL)-F-WhZ~(4|!Kq7=Y*1Xcc?_Mo3vTmFlv5D3(wR z0^6i5O~-svthiltJXWN3{ms8*_v|xT$RZK}(gbwsUw*XVjMKO+X8G!yFcyw*Yl%d7 zz2|eE_+EBYzn;K-Eti~4?CL13cxj5mYCIFR9?g3nU9!h~q5j5yw1O3iy8^E6q`>_C zC!Zt2+|ZRJVxn*Pz3fbG_Q- z3anxaA_^8KF=(M+zy0uE8aBsR_%>403Gt3*ID9#abJz9QF6j}4fgv;c;o_w@6YU+x zfByY^mqA_|N4-)>IRh5Fu_8k5z;D$YC>mU4^R+6-4#0V@Ue@-`OSDs}t&_4COH6?r zE|5po_4JWU5@fC*7d@r%k$$n|vbN~e|NENmZ~e&0%Mm3KPNd;GuX%^Q@!@R zgkORZW(;5_oN&Q#BCClghrN^biZsVTO%c5GTthVuV-#!MQAgHywunZOqt#Ef zwjOoGd7G*>2Vsn5)glZ~Aow8xM-cry3g*)Hmcwg~c9URC3r;K$R=Q z9Ex1VhU32ak&g4u;LNsoqhk4FS&AW3{Zf&?{9}jPOQwJG9DGh$sUxqr0EtOwLy2{3 zJf%iGj(=P*<_mGT71EiBGar{XtSO7~T+?`Y8BXH4s|y>%hP^N;$HVDTNpAHu5M}yK z1qQ0g5{%Y`?X4_Rmf?YU-ZNtsBn&LVR76DlIJXk>FYJXWDBcd9(vAn7336h8TaKK~ z=WfGDBA~XBAuoPVLUn9*$`^sBb77mN*Ub17>@o@ioPm=-9zE)K{R+mh>F_b#GRTUz zD#3<%DD;v_a!Vu{=Fi5EO^+I05%Vd|yBgxvA>O zFJ@V`w18kXRJyV##X=WC#MGoH z+Lhw%Ra!Bfi3_J<8VEw2Yf_m5xbY_o=rPVrHF$V-LRte>*qijNlo)AsUC6LkcL*i{ z(X$n>E^dO_Hm!>{{L9Bjs~9{x&OE7Q`N`T=g0S2rlN9!H1%{r`(1`e^vo}x!V9%k0Lwbm;n7#W$=>{{YuD06eD|^(VVXw1aqxS8u~5b2 zwCfKqWxpob(IHQrs>?}pi@i$}3t}5|cItc1{_rLz${u@q%K9O+`b7%`O!XPX7NPmW zR+-WWZQ2TkZQ}Vw4|Aqhg)^OD!Ym>t=g4_q{O(}>Z82S_3xghv_{7>l{}cX~@G7I_ zFtYY@fs!FnkD1dSt)EM(#O2c83GYLeK&S!|&*gXB{-D8B4v)I#BGUkevlQN@ESk>r zr5IU*ErmbmDa@Y4a}$Fob;fNKv2?aF%{iRrfb~ml4(g|FuW&Na7Li&@)6~^XTeBnB z4&p7208cF)hB zl$44WB>3c;fQU&^H-mTuxwSPPh=R##qCOAG=xgU+D{?29vWG42N|wv2>>^U>gMInt z0EK3^ye>GC7e@sJ;ybG{Pd2$5oWGS2S4h&u5a@-h`M@zL82(BHctA~Q(YiEOg_=*a z9r{sBw4ZrWHQS|?N!OGsFM5;5`lXNr#%nRjd|}ovSG!}5HQ}rkm}6j!pQ6+D&O8-L z1@8dglc&+~YYq>4)l`Qnp!#gtww`e^FWqO}muzg{_@nlL$?G5$v_Z8+mKD-tQz{q~ zIaE$3^OX~!afw}sl^oCP^7Ujp6#xJUu1Q2eROIfo6^w~Cr#Zcbq}RW{O5q24*wu*e zAi(IppbCIa&V^mDo?D6=DRZeURh_34PrA_$Y+x)cFgXRGlOU~YrX=Xc@*8RaOEy|E zE8z+t?JgRU3y42m5XN*?;unnqAEbNO)3qE~vLYg(vY~R36$EiRb#kP_B+tZl5XhVE zMi4;}=UE*B>p#+%%3B05bUPtcfP=N_Q2ybKiVTe+lB6bFXa$1-m$YTj%91W=ahJY^ z%Wn%20#cEsB}L#0nudtb44bK6K$lVuvJ4r(c<^1dO)zF)K#}^Y9&l)y zLl(SB*HaHg^alWWwRExq@^SKcA`uSNlMzm)LMwe+dbY1gQ^GrZzBVfjiR+WR zT6w?4Bu7}0OBzUg!YZx+p?OqQ6gV)DNzRd?5-TTiGv1|jP00?7PL-w202v3$MyySr zKnQ!j6}2@WU;YyShcV{N84|R*Pz%*9RRFD$33h62P&ny(s-o5qR7F%x(nN11QIdiN zqwO@EG;6^MRIIPc?idz9jeR9K8RxNJ+DyY_@l0?`I!;Rl{bSN^IMP%WHV#jON!QEo2PDv5}{=58b*da34#w3K-( zZbNYt^03^)fV4J@>xeB9qf)A7)|{l!IRv7bOW~-UA+chliH7KnD1G31m@1MEfRyvq zFqVwSN}y#+T}i}3l3^kR+}8nq{Fqoh>gqQEs5D(X&7j?!yChs4u4}?OI9wEG=*&o9 zMgqU55(vD5-&6Kx*qV{Rj06s!1ZK>^;gj_Yr5OqQo=afH9Q>X)&)!wA%@40B%W9RB4+Lz+92|)B zX38E+2@I~iH~Yn>z35ZsQ;Hy{xX+Ks3E|)BF`% z*Z6O#Doe~ImgTB!qUY_}{MHrw-uC&zF_*P`@zeFktRKJP>h2pJt<@?z(gzZmcxK^1 zu*{U5k-&jVV07&h-g`gIhtnB>tJlkcjgNWWInDFSKR=r5S%anMxa%8>VysLJMAKV~ zURU!89o5v-xN4frRoTRy-@kf%mDlj_1It#(x0<{QFW$5Ky8aK`)c2=v;5&zh`6_EM z4>J!l4!S9)IG;QR}^w`E4Rt={lfBXq&0hbQr4yeU@~MBb-UCNaHw z=ui6+U%S<}1zLF4Sr_q9``ke6^v6gXig3mp915w<;H+H&>D}2?p11ai)b+P^t-iJ1 zyKivuO?2U=c}u+?-8!VMbMM377{7Z-+qp|a*msTJ;w9dDVoQ2=`ljyUO?M`*p!oNb z-V-B_Y-l^{veapP-sM-dK73t!)B5%EFB^YkdHd>nJMaHo@usir`P!;n)kItV;Hr9F z(0A@tE%$$;y=is#@@w;_eY$hay&bomlfB}yEgK_USGe-7IBD`5KFcKbuDTbGqy4@9 zQ_ohfrp1??GVs{q$(}{1%7-voy;F7wnV3O6BY||!2F~d=Jicn#a`DF7zjjskc~6X_ zz4^z``Mbw42TfahuXr$d+r5jJPPmKy%zGtITbS;7f>X>v@yzJC-dJNp+u{?^@50-z zT)blG&hKAP@Xnd@@K=t+13zJ@cjGMsw{;JGpsz}zOHNz3rPE7%;L0WEE!}Yg?`XVe z_GjPK?q!m1y>`sIZ+!Ju?}U>aQ=V~k){==7V&xLEJ!`zvuALRG(WEkm`)0-*9Bzrv z&?_r};k6HX-c@te76?Go5e8P*w3I{pc+NeKe7xXYb>tOc*T5UzzQOalnvUrhefqIb z@!c6_<6m3or8kVP^%_p&1t*!zkNUhDuM=CKSF%ywO*~dT(e!#|`1LpPuC2k0yjSJH zeTyG-Z*flqXbrh?<-wKRR!EeBYx!>Nq^p19>aoTXk6Jvb_#}u!b~7_sIAo%oL0pRj zMpm-~aFSaO+*h?xIlT5!Sx21ZjXeIKxAGi& zEopAs`hwYr?dyx)l9po>t9%NlAt=3T_#@rk8<%9OCYtV91FBYfc)Bl#U3h%eDFbWx zJE7;^_WN&L@Ya(8%<`*TUDY~y%rBdmXk_{D=KVdP()(BKS#j;)(|w_&!xx<~2ZvAA zGnC>b&^!K!=dJqu&d1jeqHDAd z-g{1)*PFh{OP;sXeBHtAcnTM_&GH5r{7<{GU54()h6_FK`kS|H+%>#^>z2=6oMpy- z{YlxXiSB9cX1jWnkAm_|X77nsNXV6VY z0_okkwM!cBymI_)SK9l$>+k&O@#pEr8(n!GW}&O~YZptuo^WpA?S zcz=E|x^}g<^7Cz8W%n>MVgnrO#a@9OtH zGqU39zMjVxefEwmtF9hg{-B&dIQOc~ZJ(L!2lQs;HG>Zht-yl#SMshOrphEfaO8zc zmu$Or$5q#ktawlv^H})oym6(Q)C33))9=g!&WB0RGi0iiz-T&?YzkfvC}ny#b-p2c z;~l4bfCan*z0&CnD~>HqZc^eSoCZ-v)p)QzfHImM98ZrYn%X-{cstNK*HV7QJE%ok zGq8tV0-?T|*>UF`qpNPtdza2T7`6c3n_@cKG>FEbnx2GOnmP`aWtkoWR%*r^9EkL0 z${tz?B)qNOX + + + + $Title$ + + +$END$ + + \ No newline at end of file diff --git a/test_cases/test_model_fields.py b/test_cases/test_model_fields.py new file mode 100644 index 00000000000..e940ea88e72 --- /dev/null +++ b/test_cases/test_model_fields.py @@ -0,0 +1,40 @@ +import os + +import pytest +from depot.manager import DepotManager + +from server.fields import FaradayUploadedFile + +CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) + +@pytest.fixture(scope='module') +def setup_depot(request): + DepotManager.configure('default', { + 'depot.storage_path': '/home/lcubo/tmp_images' + }) + + +def test_html_content_type_is_not_html(setup_depot): + with open(os.path.join(CURRENT_PATH, 'data', 'test.html'))as image_data: + field = FaradayUploadedFile(image_data.read()) + assert field['content_type'] == 'application/octet-stream' + assert len(field['files']) == 1 + + +def test_image_is_detected_correctly(setup_depot): + + with open(os.path.join(CURRENT_PATH, 'data', 'faraday.png'))as image_data: + field = FaradayUploadedFile(image_data.read()) + + print(field) + assert field['content_type'] == 'image/png' + assert 'thumb_id' in field.keys() + assert 'thumb_path' in field.keys() + assert len(field['files']) == 2 + + +def test_normal_attach_is_not_detected_as_image(setup_depot): + with open(os.path.join(CURRENT_PATH, 'data', 'report_w3af.xml'))as image_data: + field = FaradayUploadedFile(image_data.read()) + assert field['content_type'] == 'application/octet-stream' + assert len(field['files']) == 1 \ No newline at end of file From a3e858408cdba083ccdc4de0a8746799dd17604a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 21 Sep 2017 12:59:13 -0300 Subject: [PATCH 0257/1506] Fix importing services with unknown status value Use "open" by default. Also add mapper for "up" and "down" values --- server/importer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index f05d07d355b..0e65250ba1b 100644 --- a/server/importer.py +++ b/server/importer.py @@ -343,11 +343,13 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation document['status'] = 'open' status_mapper = { 'open': 'open', + 'up': 'open', 'closed': 'closed', + 'down': 'closed', 'filtered': 'filtered', 'open|filtered': 'filtered' } - service.status = status_mapper[document.get('status')] + service.status = status_mapper[document.get('status', 'open')] service.version = document.get('version') service.workspace = workspace From 778ad64123ddc710547dee6c9202d6a152bf1489 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 21 Sep 2017 14:07:09 -0300 Subject: [PATCH 0258/1506] Adapt workspace functions to new API getWorkspaceSummary and getworkspacesNames --- server/www/scripts/commons/providers/server.js | 4 ++-- server/www/scripts/vulndb/controllers/importFromWs.js | 1 + server/www/scripts/workspaces/providers/workspaces.js | 8 +++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index 145a3661aaa..abcea8361eb 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -9,7 +9,7 @@ angular.module("faradayApp") .factory("ServerAPI", ["BASEURL", "$http", "$q", function(BASEURL, $http, $q) { var ServerAPI = {}; - var APIURL = BASEURL + "_api/"; + var APIURL = BASEURL + "_api/v2/"; var createGetUrl = function(wsName, objectName) { var objectName = ((objectName) ? "/" + objectName : ""); @@ -235,7 +235,7 @@ angular.module("faradayApp") ServerAPI.getWorkspaceSummary = function(wsName, confirmed) { - var getUrl = createGetUrl(wsName, "summary"); + var getUrl = createGetUrl(wsName); var payload = {}; if (confirmed !== undefined) { diff --git a/server/www/scripts/vulndb/controllers/importFromWs.js b/server/www/scripts/vulndb/controllers/importFromWs.js index b4fd4ee4ac7..e07ca0e9e1a 100644 --- a/server/www/scripts/vulndb/controllers/importFromWs.js +++ b/server/www/scripts/vulndb/controllers/importFromWs.js @@ -7,6 +7,7 @@ angular.module('faradayApp') $scope.data; var init = function() { + // TODO migration: use workspacesFact that is fixed ServerAPI.getWorkspacesNames().then( function(ws_data) { $scope.workspaces = ws_data.data.workspaces; diff --git a/server/www/scripts/workspaces/providers/workspaces.js b/server/www/scripts/workspaces/providers/workspaces.js index 2d847863fa6..c98f029cc3f 100644 --- a/server/www/scripts/workspaces/providers/workspaces.js +++ b/server/www/scripts/workspaces/providers/workspaces.js @@ -9,7 +9,13 @@ angular.module('faradayApp') workspacesFact.list = function() { var deferred = $q.defer(); ServerAPI.getWorkspacesNames(). - then(function(response) { deferred.resolve(response.data.workspaces) }, errorHandler); + then(function(response) { + var names = []; + response.data.forEach(function(workspace){ + names.push(workspace.name); + }); + deferred.resolve(names); + }, errorHandler); return deferred.promise; }; From bdc3073fb88487ece3d640e5cc3549a6cea0be76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 21 Sep 2017 14:15:20 -0300 Subject: [PATCH 0259/1506] Add trailing slashes to API endpoints to avoid double requests 301 + real request --- server/www/scripts/commons/providers/server.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index abcea8361eb..d2bc39e90ac 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -13,7 +13,7 @@ angular.module("faradayApp") var createGetUrl = function(wsName, objectName) { var objectName = ((objectName) ? "/" + objectName : ""); - var get_url = APIURL + "ws/" + wsName + objectName; + var get_url = APIURL + "ws/" + wsName + objectName + '/'; return get_url; }; @@ -31,7 +31,7 @@ angular.module("faradayApp") }; var createDbUrl = function(wsName) { - return APIURL + "ws/" + wsName; + return APIURL + "ws/" + wsName + "/"; } var createDeleteUrl = createPostUrl; @@ -225,7 +225,7 @@ angular.module("faradayApp") } ServerAPI.getWorkspacesNames = function() { - return get(APIURL + "ws"); + return get(APIURL + "ws/"); } ServerAPI.getWorkspace = function(wsName) { @@ -270,7 +270,7 @@ angular.module("faradayApp") } ServerAPI.getServicesBy = function(wsName, what) { - var url = createGetUrl(wsName, 'services') + '/count'; + var url = createGetUrl(wsName, 'services') + 'count/'; return get(url, {"group_by": what}) } @@ -285,7 +285,7 @@ angular.module("faradayApp") ServerAPI.getVulnsBySeverity = function(wsName, confirmed) { - var url = createGetUrl(wsName, 'vulns') + '/count'; + var url = createGetUrl(wsName, 'vulns') + 'count/'; var payload = {'group_by': 'severity'} if (confirmed !== undefined) { From 6701c9f8e1b2657c711b60763740ed4ff8fda34c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 21 Sep 2017 15:29:45 -0300 Subject: [PATCH 0260/1506] Refactor workspace controller to do only one request The workspace list was doing 2*n+1 requests (before and after couchdb migration). I changed the code so now it does only one unique request to /_api/ws/v2/ and fetches all the data at once --- .../www/scripts/commons/providers/server.js | 4 ++ .../workspaces/controllers/workspaces.js | 41 ++++++++----------- .../workspaces/providers/workspaces.js | 8 ++++ 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index d2bc39e90ac..d50ecf62c19 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -228,6 +228,10 @@ angular.module("faradayApp") return get(APIURL + "ws/"); } + ServerAPI.getWorkspaces = function() { + return get(APIURL + "ws/"); + } + ServerAPI.getWorkspace = function(wsName) { var getUrl = createDbUrl(wsName); return get(getUrl); diff --git a/server/www/scripts/workspaces/controllers/workspaces.js b/server/www/scripts/workspaces/controllers/workspaces.js index 4dd5bd8b81f..a43799e4a6b 100644 --- a/server/www/scripts/workspaces/controllers/workspaces.js +++ b/server/www/scripts/workspaces/controllers/workspaces.js @@ -32,32 +32,23 @@ angular.module('faradayApp') $scope.hash = ""; } - // todo: refactor the following code - workspacesFact.list().then(function(wss) { - $scope.wss = wss; - var objects = {}; - $scope.wss.forEach(function(ws){ - workspacesFact.get(ws).then(function(resp) { - $scope.onSuccessGet(resp); - }); - objects[ws] = dashboardSrv.getObjectsCount(ws); - }); - $q.all(objects).then(function(os) { - for(var workspace in os) { - if(os.hasOwnProperty(workspace)) { - $scope.objects[workspace] = { - "total_vulns": "-", - "hosts": "-", - "services": "-" - }; - for (var stat in os[workspace]) { - if (os[workspace].hasOwnProperty(stat)) { - if ($scope.objects[workspace].hasOwnProperty(stat)) - $scope.objects[workspace][stat] = os[workspace][stat]; - } - }; + workspacesFact.getWorkspaces().then(function(wss) { + + $scope.wss = []; // Store workspace names + wss.forEach(function(ws){ + $scope.wss.push(ws.name); + $scope.onSuccessGet(ws); + $scope.objects[ws.name] = { + "total_vulns": "-", + "hosts": "-", + "services": "-" + }; + for (var stat in ws.stats) { + if (ws.stats.hasOwnProperty(stat)) { + if ($scope.objects[ws.name].hasOwnProperty(stat)) + $scope.objects[ws.name][stat] = ws.stats[stat]; } - } + }; }); }); }; diff --git a/server/www/scripts/workspaces/providers/workspaces.js b/server/www/scripts/workspaces/providers/workspaces.js index c98f029cc3f..d0b0fcfe8cd 100644 --- a/server/www/scripts/workspaces/providers/workspaces.js +++ b/server/www/scripts/workspaces/providers/workspaces.js @@ -19,6 +19,14 @@ angular.module('faradayApp') return deferred.promise; }; + workspacesFact.getWorkspaces = function() { + var deferred = $q.defer(); + ServerAPI.getWorkspaces(). + then(function(response) + { deferred.resolve(response.data) }, errorHandler); + return deferred.promise; + }; + returnStatus = function(data) { return $q.when(data.status); }; From 64f66059cda586b443a69c3e1d42b4da9e3d09c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 21 Sep 2017 17:13:52 -0300 Subject: [PATCH 0261/1506] Change format of start date and end date in workspaces API. Make them available inside a "duration" field. Use JS integer timestamps instead of ISO-formatted datetimes --- server/api/modules/workspaces.py | 10 ++++++++-- server/schemas.py | 11 +++++++++++ test_cases/test_schemas.py | 19 ++++++++++++++++++- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index 2fe71786120..445bac07782 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -13,7 +13,7 @@ from server.dao.service import ServiceDAO from server.dao.workspace import WorkspaceDAO from server.utils.logger import get_logger -from server.schemas import SelfNestedField +from server.schemas import SelfNestedField, JSTimestampField from server.utils.web import ( build_bad_request_response, filter_request_args, @@ -46,14 +46,20 @@ class WorkspaceSummarySchema(Schema): attribute='vulnerability_total_count') +class WorkspaceDurationSchema(Schema): + start = JSTimestampField(attribute='start_date') + end = JSTimestampField(attribute='end_date') + + class WorkspaceSchema(AutoSchema): stats = SelfNestedField(WorkspaceSummarySchema()) + duration = SelfNestedField(WorkspaceDurationSchema()) _id = fields.Integer(dump_only=True, attribute='id') class Meta: model = Workspace fields = ('_id', 'id', 'customer', 'description', 'active', - 'start_date', 'end_date', 'name', 'public', 'scope', 'stats') + 'duration', 'name', 'public', 'scope', 'stats') class WorkspaceView(ReadWriteView): diff --git a/server/schemas.py b/server/schemas.py index f7b046a2047..63d4fcc49cd 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -1,6 +1,8 @@ +import time from marshmallow import fields from marshmallow.exceptions import ValidationError + class SelfNestedField(fields.Field): """A field to make namespaced schemas. It allows to have a field whose contents are the dump of the same object with @@ -18,3 +20,12 @@ def _serialize(self, value, attr, obj): if errors: raise ValidationError(errors, data=ret) return ret + + +class JSTimestampField(fields.Field): + """A field to serialize datetime objects into javascript + compatible timestamps (like time.time()) * 1000""" + + def _serialize(self, value, attr, obj): + if value is not None: + return int(time.mktime(value.timetuple()) * 1000) diff --git a/test_cases/test_schemas.py b/test_cases/test_schemas.py index 684097dc8cd..1c96b330098 100644 --- a/test_cases/test_schemas.py +++ b/test_cases/test_schemas.py @@ -1,20 +1,37 @@ +import time +import datetime from collections import namedtuple from marshmallow import Schema, fields -from server.schemas import SelfNestedField +from server.schemas import SelfNestedField, JSTimestampField Place = namedtuple('Place', ['name', 'x', 'y']) + class PointSchema(Schema): x = fields.Float() y = fields.Float() + class PlaceSchema(Schema): name = fields.Str() coords = SelfNestedField(PointSchema()) + class TestSelfNestedField: def test_field_serialization(self): point = Place('home', 123, 456.1) schema = PlaceSchema() dumped = schema.dump(point).data assert dumped == {"name": "home", "coords": {"x": 123.0, "y": 456.1}} + + +class TestJSTimestampField: + def test_parses_current_datetime(self): + ts = time.time() + dt = datetime.datetime.fromtimestamp(ts) + parsed = JSTimestampField()._serialize(dt, None, None) + assert parsed == int(ts) * 1000 + assert isinstance(parsed, int) + + def test_parses_null_datetime(self): + assert JSTimestampField()._serialize(None, None, None) is None From f0b88c222f01f03653d26f970e3f4fd43a116b9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 22 Sep 2017 13:01:16 -0300 Subject: [PATCH 0262/1506] Remove SQLAlchemy warnings on count properties Use text(), column() and table() functions to encapsulate parts of a SQL query, as indicated on the warnings --- server/models.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/server/models.py b/server/models.py index 6c4399bf205..4df6d706d9c 100644 --- a/server/models.py +++ b/server/models.py @@ -21,7 +21,7 @@ ) from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import relationship, backref -from sqlalchemy.sql import select +from sqlalchemy.sql import select, text, table, column from sqlalchemy import func from sqlalchemy.orm import column_property from sqlalchemy.schema import DDL @@ -95,9 +95,10 @@ def _make_generic_count_property(parent_table, children_table): children_id_field = '{}.id'.format(children_table) parent_id_field = '{}.id'.format(parent_table) children_rel_field = '{}.{}_id'.format(children_table, parent_table) - query = (select([func.count(children_id_field)]). - select_from(children_table). - where('{} = {}'.format(children_rel_field, parent_id_field))) + query = (select([func.count(column(children_id_field))]). + select_from(table(children_table)). + where(text('{} = {}'.format( + children_rel_field, parent_id_field)))) return column_property(query, deferred=True) @@ -595,15 +596,15 @@ class Command(Metadata): def _make_vuln_count_property(type_=None): - query = (select([func.count('vulnerability.id')]). - select_from('vulnerability'). - where('vulnerability.workspace_id = workspace.id') + query = (select([func.count(column('vulnerability.id'))]). + select_from(table('vulnerability')). + where(text('vulnerability.workspace_id = workspace.id')) ) if type_: # Don't do queries using this style! # This can cause SQL injection vulnerabilities # In this case type_ is supplied from a whitelist so this is safe - query = query.where("vulnerability.type = '%s'" % type_) + query = query.where(text("vulnerability.type = '%s'" % type_)) return column_property(query, deferred=True) From 987bf1f9417d119dd23023c480dd91ee9fb049df Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 2 Oct 2017 12:47:12 -0300 Subject: [PATCH 0263/1506] Add comment and cleanup --- server/models.py | 1 + test_cases/test_model_fields.py | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/server/models.py b/server/models.py index b8cc5e5b5e0..a84b559b65c 100644 --- a/server/models.py +++ b/server/models.py @@ -673,6 +673,7 @@ class UserAvatar(Metadata): uid = Column(Integer, autoincrement=True, primary_key=True) name = Column(Unicode(16), unique=True) # photo field will automatically generate thumbnail + # if the file is a valid image photo = Column(UploadedFileField(upload_type=FaradayUploadedFile)) user_id = Column('user_id', Integer(), ForeignKey('user.id')) user = relationship('User') diff --git a/test_cases/test_model_fields.py b/test_cases/test_model_fields.py index e940ea88e72..0dc257d1268 100644 --- a/test_cases/test_model_fields.py +++ b/test_cases/test_model_fields.py @@ -3,7 +3,7 @@ import pytest from depot.manager import DepotManager -from server.fields import FaradayUploadedFile +from server.fields import FaradayUploadedFile CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) @@ -25,8 +25,6 @@ def test_image_is_detected_correctly(setup_depot): with open(os.path.join(CURRENT_PATH, 'data', 'faraday.png'))as image_data: field = FaradayUploadedFile(image_data.read()) - - print(field) assert field['content_type'] == 'image/png' assert 'thumb_id' in field.keys() assert 'thumb_path' in field.keys() From 5b8d02e11f05b09b3777bd1e84f3fc54f4e7f426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 2 Oct 2017 17:49:17 -0300 Subject: [PATCH 0264/1506] Fix count fields for postgresql I broke it in a previous commit --- server/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/models.py b/server/models.py index 4df6d706d9c..90f736d5f82 100644 --- a/server/models.py +++ b/server/models.py @@ -21,7 +21,7 @@ ) from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import relationship, backref -from sqlalchemy.sql import select, text, table, column +from sqlalchemy.sql import select, text, table from sqlalchemy import func from sqlalchemy.orm import column_property from sqlalchemy.schema import DDL @@ -95,7 +95,7 @@ def _make_generic_count_property(parent_table, children_table): children_id_field = '{}.id'.format(children_table) parent_id_field = '{}.id'.format(parent_table) children_rel_field = '{}.{}_id'.format(children_table, parent_table) - query = (select([func.count(column(children_id_field))]). + query = (select([func.count(text(children_id_field))]). select_from(table(children_table)). where(text('{} = {}'.format( children_rel_field, parent_id_field)))) @@ -596,7 +596,7 @@ class Command(Metadata): def _make_vuln_count_property(type_=None): - query = (select([func.count(column('vulnerability.id'))]). + query = (select([func.count(text('vulnerability.id'))]). select_from(table('vulnerability')). where(text('vulnerability.workspace_id = workspace.id')) ) From 1dd0d087cb4feb23d468267117ec63cfe3613d3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 2 Oct 2017 17:58:19 -0300 Subject: [PATCH 0265/1506] Remove fixed TODO of models.py --- server/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/models.py b/server/models.py index 90f736d5f82..e128be9e92d 100644 --- a/server/models.py +++ b/server/models.py @@ -91,7 +91,6 @@ def do_begin(conn): def _make_generic_count_property(parent_table, children_table): """Make a deferred by default column property that counts the amount of childrens of some parent object""" - # TODO: Fix SQLAlchemy warnings children_id_field = '{}.id'.format(children_table) parent_id_field = '{}.id'.format(parent_table) children_rel_field = '{}.{}_id'.format(children_table, parent_table) From 85d3dc17088719b68f5c510df0b66a3090f3c8ae Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 2 Oct 2017 18:01:18 -0300 Subject: [PATCH 0266/1506] Add file model with importer. Update config to use storage from server.ini The code checks if the storage was configured, if not it will create the directory and save it on server.ini The model allows to use file with any object. In this commit the importer for vuln evidence was added --- server/app.py | 37 ++++++++++++++++++++++++++++++++++++- server/importer.py | 30 +++++++++++++++++++++++++++++- server/models.py | 34 +++++++++++++--------------------- 3 files changed, 78 insertions(+), 23 deletions(-) diff --git a/server/app.py b/server/app.py index c9bc560bfe8..14fa97ad12c 100644 --- a/server/app.py +++ b/server/app.py @@ -1,7 +1,16 @@ # Faraday Penetration Test IDE # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information -from ConfigParser import NoOptionError +import logging +import os +from os.path import join, expanduser + +try: + # py2.7 + from configparser import ConfigParser, NoSectionError, NoOptionError +except ImportError: + # py3 + from ConfigParser import ConfigParser, NoSectionError, NoOptionError import flask from flask import Flask @@ -10,10 +19,28 @@ Security, SQLAlchemyUserDatastore, ) +from depot.manager import DepotManager import server.config from server.utils.logger import LOGGING_HANDLERS +logger = logging.getLogger(__name__) + + +def setup_storage_path(): + default_path = join(expanduser("~"), '.faraday/storage') + if not os.path.exists(default_path): + logger.info('Creating directory {0}'.format(default_path)) + os.mkdir(default_path) + config = ConfigParser() + config.read(server.config.LOCAL_CONFIG_FILE) + config.add_section('storage') + config.set('storage', 'path', default_path) + with open(server.config.LOCAL_CONFIG_FILE, 'w') as configfile: + config.write(configfile) + + return default_path + def create_app(db_connection_string=None, testing=None): app = Flask(__name__) @@ -36,6 +63,14 @@ def create_app(db_connection_string=None, testing=None): # 'sha512_crypt', 'plaintext', # TODO: remove it ] + try: + storage_path = server.config.storage.path + except AttributeError: + logger.warn('No storage section or path in the .faraday/server.ini. Setting the default value to .faraday/storage') + storage_path = setup_storage_path() + DepotManager.configure('default', { + 'depot.storage_path': storage_path + }) if testing: app.config['TESTING'] = testing try: diff --git a/server/importer.py b/server/importer.py index f05d07d355b..0eba3776417 100644 --- a/server/importer.py +++ b/server/importer.py @@ -6,6 +6,7 @@ import sys import json import datetime +from tempfile import NamedTemporaryFile from collections import ( Counter, defaultdict, @@ -50,6 +51,7 @@ VulnerabilityTemplate, VulnerabilityWeb, Workspace, + File, ) from server.utils.database import get_or_create from server.web import app @@ -298,7 +300,7 @@ def merge_with_host(self, host, interface, workspace): if type(interface['hostnames']) in (str, unicode): interface['hostnames'] = [interface['hostnames']] - for hostname_str in interface['hostnames']: + for hostname_str in interface['hostnames'] or []: if not hostname_str: # skip empty hostnames continue @@ -435,6 +437,32 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation self.add_references(document, vulnerability, workspace) self.add_policy_violations(document, vulnerability, workspace) + attachments_data = document.get('_attachments') or {} + for attachment_name, attachment_data in attachments_data.items(): + #http://localhost:5984/evidence/334389048b872a533002b34d73f8c29fd09efc50.c7b0f6cba2fae8e446b7ffedfdb18026bb9ba41d/forbidden.png + attachment_url = "http://{username}:{password}@{hostname}:{port}/{path}".format( + username=server.config.couchdb.user, + password=server.config.couchdb.password, + hostname=server.config.couchdb.host, + port=server.config.couchdb.port, + path='{0}/{1}/{2}'.format(workspace.name, document.get('_id'), attachment_name) + ) + response = requests.get(attachment_url) + response.raw.decode_content = True + attachment_file = NamedTemporaryFile() + attachment_file.write(response.content) + attachment_file.seek(0) + session.commit() + file, created = get_or_create( + session, + File, + filename=attachment_name, + object_id=vulnerability.id, + object_type=vulnerability.__class__.__name__) + file.content = attachment_file.read() + + attachment_file.close() + yield vulnerability def add_policy_violations(self, document, vulnerability, workspace): diff --git a/server/models.py b/server/models.py index 8ac09b1f8d7..25a1070be03 100644 --- a/server/models.py +++ b/server/models.py @@ -677,36 +677,28 @@ def __repr__(self): self.username) -class Evidence(Metadata): - __tablename__ = 'evidence' - - uid = Column(Integer, autoincrement=True, primary_key=True) - name = Column(Unicode(16), unique=True) - content = Column('content_col', UploadedFileField) # plain attached file - file = Column(UploadedFileField(upload_type=FaradayUploadedFile)) - - vulnerability_id = Column( - Integer, - ForeignKey(VulnerabilityGeneric.id), - index=True - ) - vulnerability = relationship( - 'VulnerabilityGeneric', - backref='evidence', - foreign_keys=[vulnerability_id], - ) +class File(Metadata): + __tablename__ = 'file' + + id = Column(Integer, autoincrement=True, primary_key=True) + name = Column(Text, unique=True) + filename = Column(Text, unique=True) + description = Column(Text, unique=True) + content = Column(UploadedFileField(upload_type=FaradayUploadedFile)) # plain attached file + object_id = Column(Integer, nullable=False) + object_type = Column(Text, nullable=False) class UserAvatar(Metadata): __tablename_ = 'user_avatar' - uid = Column(Integer, autoincrement=True, primary_key=True) - name = Column(Unicode(16), unique=True) + id = Column(Integer, autoincrement=True, primary_key=True) + name = Column(Text, unique=True) # photo field will automatically generate thumbnail # if the file is a valid image photo = Column(UploadedFileField(upload_type=FaradayUploadedFile)) user_id = Column('user_id', Integer(), ForeignKey('user.id')) - user = relationship('User') + user = relationship('User', foreign_keys=[user_id]) class MethodologyTemplate(Metadata): From 25f8c4ad9be5080d25d5110fa3d5800be264d88c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 3 Oct 2017 19:17:44 -0300 Subject: [PATCH 0267/1506] Override port and bindaddress using parameters. Also log port and address used for debugging --- faraday-server.py | 10 +++++++++- server/web.py | 4 ++++ test_cases/test_api_commands.py | 0 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 test_cases/test_api_commands.py diff --git a/faraday-server.py b/faraday-server.py index 7cfac7ff035..68bdc2af36d 100755 --- a/faraday-server.py +++ b/faraday-server.py @@ -89,6 +89,8 @@ def main(): parser.add_argument('--stop', action='store_true', help='stop Faraday Server') parser.add_argument('--nodeps', action='store_true', help='Skip dependency check') parser.add_argument('--no-setup', action='store_true', help=argparse.SUPPRESS) + parser.add_argument('--port', help='Overides server.ini port configuration') + parser.add_argument('--bind_address', help='Overides server.ini bind_address configuration') f = open(server.config.VERSION_FILE) f_version = f.read().strip() @@ -109,11 +111,17 @@ def main(): # Overwrites config option if SSL is set by argument if args.ssl: - server.config.ssl.set('enabled', 'true') + server.config.ssl.enabled = 'true' if not args.no_setup: setup_environment(not args.nodeps) + if args.port: + server.config.faraday_server.port = args.port + + if args.bind_address: + server.config.faraday_server.bind_address = args.bind_address + if args.start: # Starts a new process on background with --ignore-setup # and without --start nor --stop diff --git a/server/web.py b/server/web.py index 939e408bb7c..a278055a491 100644 --- a/server/web.py +++ b/server/web.py @@ -85,6 +85,10 @@ class WebServer(object): WEB_UI_LOCAL_PATH = os.path.join(server.config.FARADAY_BASE, 'server/www') def __init__(self, enable_ssl=False): + logger.get_logger(__name__).info('Starting server at port {0} with bind address {1}. SSL {2}'.format( + server.config.faraday_server.port, + server.config.faraday_server.bind_address, + enable_ssl)) self.__ssl_enabled = enable_ssl self.__config_server() self.__config_couchdb_conn() diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py new file mode 100644 index 00000000000..e69de29bb2d From 0eaaef7744131a74f2c05d34f7cf88c2ca219cb1 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 4 Oct 2017 11:07:48 -0300 Subject: [PATCH 0268/1506] Add new commands api --- server/api/modules/commandsrun.py | 50 +++++++++++++++++++++++++++++++ test_cases/conftest.py | 1 + test_cases/factories.py | 9 ++++++ test_cases/test_api_commands.py | 43 ++++++++++++++++++++++++++ 4 files changed, 103 insertions(+) diff --git a/server/api/modules/commandsrun.py b/server/api/modules/commandsrun.py index 7485f9522ed..177790fb96d 100644 --- a/server/api/modules/commandsrun.py +++ b/server/api/modules/commandsrun.py @@ -1,10 +1,13 @@ # Faraday Penetration Test IDE # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information +import time import flask from flask import Blueprint +from marshmallow import fields +from server.api.base import AutoSchema, ReadWriteWorkspacedView from server.utils.logger import get_logger from server.utils.web import ( gzipped, @@ -12,9 +15,56 @@ filter_request_args, get_integer_parameter ) from server.dao.command import CommandDAO +from server.models import Command commandsrun_api = Blueprint('commandsrun_api', __name__) +class Titlecased(fields.Field): + def _serialize(self, value, attr, obj): + if value is None: + return '' + return value.title() + +class CommandSchema(AutoSchema): + _id = fields.Integer(dump_only=True, attribute='id') + itime = fields.Method('get_itime') + duration = fields.Method('get_duration') + workspace = fields.Method('get_workspace_name') + + def get_workspace_name(self, obj): + return obj.workspace.name + + def get_itime(self, obj): + return time.mktime(obj.start_date.utctimetuple()) + + def get_duration(self, obj): + return (obj.end_date - obj.start_date).seconds + + class Meta: + model = Command + fields = ('_id', 'command', 'duration', 'itime', 'ip', 'hostname', + 'params', 'user', 'workspace') + + +class CommandView(ReadWriteWorkspacedView): + route_base = 'commands' + model_class = Command + schema_class = CommandSchema + + def _envelope_list(self, objects, pagination_metadata=None): + commands = [] + for command in objects: + commands.append({ + 'id': command['_id'], + 'key': command['_id'], + 'value': command + }) + return { + 'commands': commands, + } + +CommandView.register(commandsrun_api) + @gzipped @commandsrun_api.route('/ws//commands', methods=['GET']) diff --git a/test_cases/conftest.py b/test_cases/conftest.py index fa34ea28a2b..8b1bff45501 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -24,6 +24,7 @@ factories.VulnerabilityWebFactory, factories.UserFactory, factories.WorkspaceFactory, + factories.CommandFactory, ] for factory in enabled_factories: register(factory) diff --git a/test_cases/factories.py b/test_cases/factories.py index e339a69cd88..001ecdd19b7 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -1,10 +1,13 @@ import factory import datetime + +import pytz from factory.fuzzy import ( FuzzyChoice, FuzzyNaiveDateTime, FuzzyInteger, FuzzyText, + FuzzyDateTime, ) from server.models import ( db, @@ -75,6 +78,7 @@ class Meta: class WorkspaceObjectFactory(FaradayFactory): workspace = factory.SubFactory(WorkspaceFactory) + creator = factory.SubFactory(UserFactory) @classmethod def build_dict(cls, **kwargs): @@ -169,6 +173,11 @@ class Meta: class CommandFactory(WorkspaceObjectFactory): command = FuzzyText() + end_date = FuzzyDateTime(datetime.datetime.utcnow().replace(tzinfo=pytz.utc) + datetime.timedelta(20), datetime.datetime.utcnow().replace(tzinfo=pytz.utc) + datetime.timedelta(30)) + start_date = FuzzyDateTime(datetime.datetime.utcnow().replace(tzinfo=pytz.utc) - datetime.timedelta(30), datetime.datetime.utcnow().replace(tzinfo=pytz.utc) - datetime.timedelta(20)) + ip = FuzzyText() + user = FuzzyText() + hostname = FuzzyText() class Meta: model = Command diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index e69de29bb2d..8db93a3eba3 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -0,0 +1,43 @@ +#-*- coding: utf8 -*- +"""Tests for many API endpoints that do not depend on workspace_name""" + +import pytest +from test_cases import factories +from test_api_workspaced_base import ListTestsMixin, API_PREFIX, GenericAPITest +from server.models import ( + Command, + Workspace, +) +from server.api.modules.commandsrun import CommandView +from server.api.modules.workspaces import WorkspaceView + + +@pytest.mark.usefixtures('logged_user') +class TestListCommandView(GenericAPITest): + model = Command + factory = factories.CommandFactory + api_endpoint = 'commands' + unique_fields = ['ip'] + update_fields = ['ip', 'description', 'os'] + view_class = CommandView + + def test_(self, test_client, second_workspace, session): + self.factory.create(workspace=second_workspace) + session.commit() + res = test_client.get(self.url()) + assert res.status_code == 200 + assert 'commands' in res.json + for command in res.json['commands']: + assert set([u'id', u'key', u'value']) == set(command.keys()) + object_properties = [ + u'_id', + u'command', + u'duration', + u'hostname', + u'ip', + u'itime', + u'params', + u'user', + u'workspace' + ] + assert set(object_properties) == set(command['value'].keys()) \ No newline at end of file From 55150f3bcac019a46b0e9716e34821201a95b879 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 4 Oct 2017 11:08:02 -0300 Subject: [PATCH 0269/1506] Fix web ui url --- server/www/scripts/commons/providers/server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index d50ecf62c19..57933640a90 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -13,7 +13,7 @@ angular.module("faradayApp") var createGetUrl = function(wsName, objectName) { var objectName = ((objectName) ? "/" + objectName : ""); - var get_url = APIURL + "ws/" + wsName + objectName + '/'; + var get_url = APIURL + "ws/" + wsName + objectName; return get_url; }; @@ -31,7 +31,7 @@ angular.module("faradayApp") }; var createDbUrl = function(wsName) { - return APIURL + "ws/" + wsName + "/"; + return APIURL + "ws/" + wsName; } var createDeleteUrl = createPostUrl; From f8ad3e6b22a6e9dbd3f88bbba764c6beec100f00 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 4 Oct 2017 11:14:27 -0300 Subject: [PATCH 0270/1506] Remove unused code --- server/api/modules/commandsrun.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/server/api/modules/commandsrun.py b/server/api/modules/commandsrun.py index 177790fb96d..70262211be9 100644 --- a/server/api/modules/commandsrun.py +++ b/server/api/modules/commandsrun.py @@ -19,11 +19,6 @@ commandsrun_api = Blueprint('commandsrun_api', __name__) -class Titlecased(fields.Field): - def _serialize(self, value, attr, obj): - if value is None: - return '' - return value.title() class CommandSchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') From 95b23cf4392d35a066ce04023c046c28737066bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 4 Oct 2017 17:14:03 -0300 Subject: [PATCH 0271/1506] Fix test_common and test_import_users_from_couch It were failing because test_common was trying to import Interface, and because user import logic was moved to the importer. I temporally skipped a test that checks that the import fails if user has an unknown password scheme. Now it sets the password to "changeme" instead --- test_cases/test_common.py | 6 ++---- test_cases/test_import_users_from_couch.py | 5 ++++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/test_cases/test_common.py b/test_cases/test_common.py index aa183554fb6..6bc07265f88 100644 --- a/test_cases/test_common.py +++ b/test_cases/test_common.py @@ -5,7 +5,7 @@ See the file 'doc/LICENSE' for the license information ''' -from server.models import Host, Interface, Service, Vulnerability +from server.models import Host, Service, Vulnerability import random def new_random_workspace_name(): return ("aworkspace" + "".join(random.sample([chr(i) for i in range(65, 90) @@ -17,9 +17,7 @@ def create_host(self, host_name="pepito", os="linux"): return host def create_interface(self, host, iname="coqiuto", mac="00:03:00:03:04:04"): - interface = Interface(name=iname, mac=mac) - self.model_controller.addInterfaceSYNC(host.getName(), interface) - return interface + raise NotImplementedError() def create_service(self, host, interface, service_name = "coquito"): service = Service(service_name) diff --git a/test_cases/test_import_users_from_couch.py b/test_cases/test_import_users_from_couch.py index 91385fcb688..cb7f9ca6761 100644 --- a/test_cases/test_import_users_from_couch.py +++ b/test_cases/test_import_users_from_couch.py @@ -1,6 +1,6 @@ import pytest from passlib.hash import pbkdf2_sha1 -import import_users_from_couch +from server.importer import ImportCouchDBUsers NON_ADMIN_DOC = { "_id": "org.couchdb.user:removeme2", @@ -25,6 +25,8 @@ "type": "user" } +import_users_from_couch = ImportCouchDBUsers() + def test_import_encrypted_password_from_admin_user(): original_hash = ('-pbkdf2-eeea435c505e74d33a8c1b55c39d8dd355db4c2d,' 'aedeef5a01f96a84360d2719fc521b9f,10') @@ -41,6 +43,7 @@ def test_import_non_admin_from_document(): assert pbkdf2_sha1.verify('12345', new_hash) +@pytest.mark.skip(reason="Now it use default password 'changeme'") def test_import_admin_from_document_fails(): with pytest.raises(ValueError): import_users_from_couch.get_hash_from_document(ADMIN_DOC) From 944fceed970c22b916cd1df51e103cd165b13aa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 4 Oct 2017 17:20:23 -0300 Subject: [PATCH 0272/1506] Remove interface usage in test_models Now all test_cases/test_*.py work! --- test_cases/test_models.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/test_cases/test_models.py b/test_cases/test_models.py index aabe9e3ee82..8253c1720c4 100644 --- a/test_cases/test_models.py +++ b/test_cases/test_models.py @@ -4,7 +4,7 @@ from persistence.server import server_io_exceptions from mock import MagicMock, patch, create_autospec -HOST_JSON_STRING = '{"_id":1,"id":"08d3b6545ec70897daf05cd471f4166a8e605c00","key":"08d3b6545ec70897daf05cd471f4166a8e605c00","value":{"_id":"08d3b6545ec70897daf05cd471f4166a8e605c00","_rev":"1-a12368dc03d557c337e833f8090db568","default_gateway":["192.168.20.1","00:1d:aa:c9:83:e8"],"description":"","interfaces":[1],"metadata":{"create_time":1475852074.455225,"creator":"","owner":"","update_action":0,"update_controller_action":"ModelControler._processAction ModelControler.newHost","update_time":1475852074.455226,"update_user":""},"name":"10.31.112.29","os":"Microsoft Windows Server 2008 R2 Standard Service Pack 1","owned":"false","owner":"","services":12,"vulns":43}}' +HOST_JSON_STRING = '{"_id":1,"id":"08d3b6545ec70897daf05cd471f4166a8e605c00","key":"08d3b6545ec70897daf05cd471f4166a8e605c00","value":{"_id":"08d3b6545ec70897daf05cd471f4166a8e605c00","_rev":"1-a12368dc03d557c337e833f8090db568","default_gateway":["192.168.20.1","00:1d:aa:c9:83:e8"],"description":"","metadata":{"create_time":1475852074.455225,"creator":"","owner":"","update_action":0,"update_controller_action":"ModelControler._processAction ModelControler.newHost","update_time":1475852074.455226,"update_user":""},"name":"10.31.112.29","os":"Microsoft Windows Server 2008 R2 Standard Service Pack 1","owned":"false","owner":"","services":12,"vulns":43}}' INTERFACE_JSON_STRING = '{"_id":1,"id":"08d3b6545ec70897daf05cd471f4166a8e605c00.02946afc59c50a4d76c1adbb082c2d5439baf50a","key":"08d3b6545ec70897daf05cd471f4166a8e605c00.02946afc59c50a4d76c1adbb082c2d5439baf50a","value":{"_id":"08d3b6545ec70897daf05cd471f4166a8e605c00.02946afc59c50a4d76c1adbb082c2d5439baf50a","_rev":"1-c279e0906d2b1f02b832a99d5f58f99c","description":"","host_id":1,"hostnames":["qa3app09"],"ipv4":{"DNS":[],"address":"10.31.112.29","gateway":"0.0.0.0","mask":"0.0.0.0"},"ipv6":{"DNS":[],"address":"0000:0000:0000:0000:0000:0000:0000:0000","gateway":"0000:0000:0000:0000:0000:0000:0000:0000","prefix":"00"},"mac":"00:50:56:81:01:e3","metadata":{"create_time":1475852074.456803,"creator":"","owner":"","update_action":0,"update_controller_action":"ModelControler._processAction ModelControler.newInterface","update_time":1475852074.456803,"update_user":""},"name":"10.31.112.29","network_segment":"","owned":false,"owner":"","ports":{"closed":null,"filtered":null,"opened":null}}}' @@ -47,7 +47,6 @@ def test_flatten_dictionary(self): u"_rev":u"1-a12368dc03d557c337e833f8090db568", u"default_gateway":[u"192.168.20.1",u"00:1d:aa:c9:83:e8"], u"description":u"", - u"interfaces":[1], u"metadata":{u"create_time":1475852074.455225, u"creator":u"", u"owner":u"", @@ -66,13 +65,11 @@ def test_flatten_dictionary(self): def test_faraday_ready_objects_getter(self): hosts = models._get_faraday_ready_objects(self.ws, [self.a_host_dictionary], 'hosts') - interfaces = models._get_faraday_ready_objects(self.ws, [self.an_interface_dictionary], 'interfaces') services = models._get_faraday_ready_objects(self.ws, [self.a_service_dictionary], 'services') vulns = models._get_faraday_ready_objects(self.ws, [self.a_vuln_dictionary], 'vulns') vulns_web = models._get_faraday_ready_objects(self.ws, [self.a_vuln_dictionary], 'vulns_web') self.assertTrue(all([isinstance(h, models.Host) for h in hosts])) - self.assertTrue(all([isinstance(i, models.Interface) for i in interfaces])) self.assertTrue(all([isinstance(s, models.Service) for s in services])) self.assertTrue(all([isinstance(v, models.Vuln) for v in vulns])) self.assertTrue(all([isinstance(v, models.VulnWeb) for v in vulns_web])) @@ -81,8 +78,8 @@ def test_id_creation(self): # ideally, the flatten dictionary should be provided and shouldnt depend upon # our implementation. # ideally. - classes = [models.Host, models.Interface, models.Service, models.Vuln, models.VulnWeb, models.Note] - dicts = [self.a_host_dictionary, self.an_interface_dictionary, self.a_service_dictionary, + classes = [models.Host, models.Service, models.Vuln, models.VulnWeb, models.Note] + dicts = [self.a_host_dictionary, self.a_service_dictionary, self.a_vuln_dictionary, self.a_vuln_web_dictionary, self.a_note_dictionary] dicts = map(models._flatten_dictionary, dicts) for class_, dictionary in zip(classes, dicts): From 51a8606b107773d20f3b918a2fe094dc22899286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 4 Oct 2017 17:29:23 -0300 Subject: [PATCH 0273/1506] Fix plugin tests Don't use interfaces --- plugins/plugin.py | 21 +++------------------ test_cases/plugins/test_acunetix.py | 8 +++----- test_cases/plugins/test_burp.py | 8 +++----- test_cases/plugins/test_nessus.py | 8 +++----- test_cases/plugins/test_nexpose_full.py | 8 +++----- test_cases/plugins/test_nmap.py | 5 ----- test_cases/plugins/test_ping.py | 5 ----- test_cases/plugins/test_telnet.py | 5 ----- test_cases/plugins/test_whois.py | 5 ----- 9 files changed, 15 insertions(+), 58 deletions(-) diff --git a/plugins/plugin.py b/plugins/plugin.py index fb49ad7637e..f1253b14551 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -18,7 +18,6 @@ import model.common from model.common import factory from persistence.server.models import (Host, - Interface, Service, Vuln, VulnWeb, @@ -160,23 +159,9 @@ def createAndAddInterface( ipv6_gateway="0000:0000:0000:0000:0000:0000:0000:0000", ipv6_dns=[], network_segment="", hostname_resolution=[]): - # hostname_resolution must be a list. Many plugins are passing a string - # as argument causing errors in the WEB UI. - if isinstance(hostname_resolution, str): - hostname_resolution = [hostname_resolution] - - int_obj = model.common.factory.createModelObject( - Interface.class_signature, - name, mac=mac, ipv4_address=ipv4_address, - ipv4_mask=ipv4_mask, ipv4_gateway=ipv4_gateway, ipv4_dns=ipv4_dns, - ipv6_address=ipv6_address, ipv6_prefix=ipv6_prefix, - ipv6_gateway=ipv6_gateway, ipv6_dns=ipv6_dns, - network_segment=network_segment, - hostnames=hostname_resolution, parent_id=host_id) - - int_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDINTERFACE, host_id, int_obj) - return int_obj.getID() + # We don't use interface anymore, so return a host id to maintain + # backwards compatibility + return host_id def createAndAddServiceToInterface(self, host_id, interface_id, name, protocol="tcp?", ports=[], diff --git a/test_cases/plugins/test_acunetix.py b/test_cases/plugins/test_acunetix.py index defb895f4bc..0b151906788 100644 --- a/test_cases/plugins/test_acunetix.py +++ b/test_cases/plugins/test_acunetix.py @@ -21,7 +21,6 @@ Note, Host, Service, - Interface ) from plugins.modelactions import modelactions @@ -33,7 +32,6 @@ class AcunetixParserTest(unittest.TestCase): def setUp(self): self.plugin = AcunetixPlugin() factory.register(Host) - factory.register(Interface) factory.register(Service) factory.register(Vuln) factory.register(VulnWeb) @@ -45,9 +43,9 @@ def test_Plugin_creates_apropiate_objects(self): action = self.plugin._pending_actions.get(block=True) self.assertEqual(action[0], modelactions.ADDHOST) self.assertEqual(action[1].name, "5.175.17.140") - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDINTERFACE) - self.assertEqual(action[2].name, "5.175.17.140") + # action = self.plugin._pending_actions.get(block=True) + # self.assertEqual(action[0], modelactions.ADDINTERFACE) + # self.assertEqual(action[2].name, "5.175.17.140") action = self.plugin._pending_actions.get(block=True) self.assertEqual(action[0], modelactions.ADDSERVICEINT) self.assertEqual(action[3].ports, [80]) diff --git a/test_cases/plugins/test_burp.py b/test_cases/plugins/test_burp.py index 5bbb846bc71..62d14ce854d 100644 --- a/test_cases/plugins/test_burp.py +++ b/test_cases/plugins/test_burp.py @@ -21,7 +21,6 @@ Note, Host, Service, - Interface ) from plugins.modelactions import modelactions import test_common @@ -34,7 +33,6 @@ class BurpTest(unittest.TestCase): def setUp(self): self.plugin = BurpPlugin() factory.register(Host) - factory.register(Interface) factory.register(Service) factory.register(Vuln) factory.register(VulnWeb) @@ -46,9 +44,9 @@ def test_Plugin_creates_adecuate_objects(self): action = self.plugin._pending_actions.get(block=True) self.assertEqual(action[0], modelactions.ADDHOST) self.assertEqual(action[1].name, "200.20.20.201") - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDINTERFACE) - self.assertEqual(action[2].name, "200.20.20.201") + # action = self.plugin._pending_actions.get(block=True) + # self.assertEqual(action[0], modelactions.ADDINTERFACE) + # self.assertEqual(action[2].name, "200.20.20.201") action = self.plugin._pending_actions.get(block=True) self.assertEqual(action[0], modelactions.ADDSERVICEINT) self.assertEqual(action[3].name, 'http') diff --git a/test_cases/plugins/test_nessus.py b/test_cases/plugins/test_nessus.py index 8be7b33c7c4..ae3daad50fa 100644 --- a/test_cases/plugins/test_nessus.py +++ b/test_cases/plugins/test_nessus.py @@ -21,7 +21,6 @@ Note, Host, Service, - Interface ) from plugins.modelactions import modelactions import test_common @@ -33,7 +32,6 @@ class NessusParserTest(unittest.TestCase): def setUp(self): self.plugin = NessusPlugin() factory.register(Host) - factory.register(Interface) factory.register(Service) factory.register(Vuln) factory.register(VulnWeb) @@ -45,9 +43,9 @@ def test_Plugin_Calls_createAndAddHost(self): action = self.plugin._pending_actions.get(block=True) self.assertEqual(action[0], modelactions.ADDHOST) self.assertEqual(action[1].name, "12.233.108.201") - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDINTERFACE) - self.assertEqual(action[2].name, "12.233.108.201") + # action = self.plugin._pending_actions.get(block=True) + # self.assertEqual(action[0], modelactions.ADDINTERFACE) + # self.assertEqual(action[2].name, "12.233.108.201") action = self.plugin._pending_actions.get(block=True) self.assertEqual(action[0], modelactions.ADDVULNHOST) self.assertEqual(action[2].name, "Nessus Scan Information") diff --git a/test_cases/plugins/test_nexpose_full.py b/test_cases/plugins/test_nexpose_full.py index 30588f034f4..0c5435e0387 100644 --- a/test_cases/plugins/test_nexpose_full.py +++ b/test_cases/plugins/test_nexpose_full.py @@ -24,7 +24,6 @@ Note, Host, Service, - Interface ) from plugins.modelactions import modelactions @@ -35,7 +34,6 @@ class NexposeTest(unittest.TestCase): def setUp(self): self.plugin = NexposeFullPlugin() factory.register(Host) - factory.register(Interface) factory.register(Service) factory.register(Vuln) factory.register(VulnWeb) @@ -47,9 +45,9 @@ def test_Plugin_creates_apropiate_objects(self): action = self.plugin._pending_actions.get(block=True) self.assertEqual(action[0], modelactions.ADDHOST) self.assertEqual(action[1].name, "192.168.1.1") - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDINTERFACE) - self.assertEqual(action[2].name, "192.168.1.1") + # action = self.plugin._pending_actions.get(block=True) + # self.assertEqual(action[0], modelactions.ADDINTERFACE) + # self.assertEqual(action[2].name, "192.168.1.1") for i in range(131): action = self.plugin._pending_actions.get(block=True) if type(action[2]) in [Vuln, VulnWeb]: diff --git a/test_cases/plugins/test_nmap.py b/test_cases/plugins/test_nmap.py index 2901efc0780..4cc0dbcbe6f 100644 --- a/test_cases/plugins/test_nmap.py +++ b/test_cases/plugins/test_nmap.py @@ -21,7 +21,6 @@ Note, Host, Service, - Interface ) from plugins.modelactions import modelactions @@ -56,7 +55,6 @@ class NmapXMLParserTest(unittest.TestCase): def setUp(self): factory.register(Host) - factory.register(Interface) factory.register(Service) factory.register(Vuln) factory.register(VulnWeb) @@ -68,9 +66,6 @@ def test_Plugin_Calls_createAndAddHost(self): action = self.plugin._pending_actions.get(block=True) self.assertEqual(action[0], modelactions.ADDHOST) self.assertEqual(action[1].name, "198.38.82.159") - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDINTERFACE) - self.assertEqual(action[2].name, "198.38.82.159") def test_Plugin_Calls_createAndAddService(self): self.plugin.parseOutputString(self.xml_output) diff --git a/test_cases/plugins/test_ping.py b/test_cases/plugins/test_ping.py index 3a5a12def79..7729884eeb3 100644 --- a/test_cases/plugins/test_ping.py +++ b/test_cases/plugins/test_ping.py @@ -21,7 +21,6 @@ Note, Host, Service, - Interface ) from plugins.modelactions import modelactions @@ -33,7 +32,6 @@ class CmdPingPluginTest(unittest.TestCase): "(216.58.222.142): icmp_seq=1 ttl=53 time=28.9 ms") def setUp(self): factory.register(Host) - factory.register(Interface) factory.register(Service) factory.register(Vuln) factory.register(VulnWeb) @@ -45,9 +43,6 @@ def test_Plugin_Calls_createAndAddHost(self): action = self.plugin._pending_actions.get(block=True) self.assertEqual(action[0], modelactions.ADDHOST) self.assertEqual(action[1].name, "216.58.222.142") - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDINTERFACE) - self.assertEqual(action[2].name, "216.58.222.142") if __name__ == '__main__': diff --git a/test_cases/plugins/test_telnet.py b/test_cases/plugins/test_telnet.py index 65885c7a7f9..340c48b011e 100644 --- a/test_cases/plugins/test_telnet.py +++ b/test_cases/plugins/test_telnet.py @@ -21,7 +21,6 @@ Note, Host, Service, - Interface ) from plugins.modelactions import modelactions @@ -41,7 +40,6 @@ class CmdPingPluginTest(unittest.TestCase): "Connection closed by foreign host.\n") def setUp(self): factory.register(Host) - factory.register(Interface) factory.register(Service) factory.register(Vuln) factory.register(VulnWeb) @@ -53,9 +51,6 @@ def test_Plugin_Calls_createAndAddHost(self): action = self.plugin._pending_actions.get(block=True) self.assertEqual(action[0], modelactions.ADDHOST) self.assertEqual(action[1].name, "127.0.0.1") - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDINTERFACE) - self.assertEqual(action[2].name, "127.0.0.1") if __name__ == '__main__': diff --git a/test_cases/plugins/test_whois.py b/test_cases/plugins/test_whois.py index efbe874caf1..f73eee3ba85 100644 --- a/test_cases/plugins/test_whois.py +++ b/test_cases/plugins/test_whois.py @@ -21,7 +21,6 @@ Note, Host, Service, - Interface ) from plugins.modelactions import modelactions @@ -34,7 +33,6 @@ class CmdPingPluginTest(unittest.TestCase): def setUp(self): factory.register(Host) - factory.register(Interface) factory.register(Service) factory.register(Vuln) factory.register(VulnWeb) @@ -46,9 +44,6 @@ def test_Plugin_Calls_createAndAddHost(self): action = self.plugin._pending_actions.get(block=True) self.assertEqual(action[0], modelactions.ADDHOST) self.assertEqual(action[1].name, "205.251.196.172") - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDINTERFACE) - self.assertEqual(action[2].name, "205.251.196.172") if __name__ == '__main__': From ac72912bbb653e4fb6a669e04833b6f5a4655b2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 4 Oct 2017 17:39:42 -0300 Subject: [PATCH 0274/1506] Remove DAO tests and run plugin controller one DAO isn't going to be used anymore. I had to rename plugins_controller_unittests.py because it has to start with "test_" so pytest discovers it --- test_cases/dao/__init__.py | 1 - test_cases/dao/test_command.py | 37 ----------- test_cases/dao/test_credential.py | 38 ------------ test_cases/dao/test_host.py | 62 ------------------- test_cases/dao/test_services.py | 57 ----------------- test_cases/dao/test_vuln.py | 55 ---------------- ...nittests.py => test_plugins_controller.py} | 0 7 files changed, 250 deletions(-) delete mode 100644 test_cases/dao/__init__.py delete mode 100644 test_cases/dao/test_command.py delete mode 100644 test_cases/dao/test_credential.py delete mode 100644 test_cases/dao/test_host.py delete mode 100644 test_cases/dao/test_services.py delete mode 100644 test_cases/dao/test_vuln.py rename test_cases/{plugins_controller_unittests.py => test_plugins_controller.py} (100%) diff --git a/test_cases/dao/__init__.py b/test_cases/dao/__init__.py deleted file mode 100644 index 8b137891791..00000000000 --- a/test_cases/dao/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/test_cases/dao/test_command.py b/test_cases/dao/test_command.py deleted file mode 100644 index 9ddab8fdde0..00000000000 --- a/test_cases/dao/test_command.py +++ /dev/null @@ -1,37 +0,0 @@ -import os -import sys -sys.path.append(os.path.abspath(os.getcwd())) -import string -import random -import unittest - -from server.dao.command import CommandDAO -from test_cases.factories import WorkspaceFactory, CommandFactory - - -def test_list_with_multiple_workspace(app, session): - with app.app_context(): - workspace = WorkspaceFactory.build() - - commands_dao = CommandDAO(workspace) - expected = {'commands': []} - - res = commands_dao.list() - assert expected == res - - command = CommandFactory.build(workspace=workspace) - session.add(command) - session.commit() - expected = {'commands': [{'value': {'itime': None, 'command': command.command, 'user': None, 'workspace': 0, 'params': None, 'duration': None, 'ip': None, '_id': None, 'hostname': None}, 'id': None, 'key': None}]} - res = commands_dao.list() - assert expected == res - - another_workspace = WorkspaceFactory.build() - - another_command = CommandFactory.build(workspace=another_workspace) - session.add(another_command) - session.commit() - - res = commands_dao.list() - assert len(res['commands']) == 1 - assert expected == res diff --git a/test_cases/dao/test_credential.py b/test_cases/dao/test_credential.py deleted file mode 100644 index d5463e697be..00000000000 --- a/test_cases/dao/test_credential.py +++ /dev/null @@ -1,38 +0,0 @@ -import os -import sys -sys.path.append(os.path.abspath(os.getcwd())) -import string -import random -import unittest - -from server.dao.credential import CredentialDAO -from test_cases.factories import WorkspaceFactory, CredentialFactory - - -def test_list_with_multiple_workspace(app, session): - with app.app_context(): - workspace = WorkspaceFactory.build() - - credentials_dao = CredentialDAO(workspace) - expected = {'rows': []} - - res = credentials_dao.list() - assert expected == res - - credential = CredentialFactory.build(workspace=workspace) - session.add(credential) - session.commit() - expected = {} - res = credentials_dao.list() - expected = {'rows': [{'value': {'username': credential.username, 'password': credential.password, 'description': None, 'couchid': None, 'owner': None, '_id': None, 'metadata': {'update_time': None, 'create_time': None, 'update_user': None, 'update_action': None, 'creator': None, 'owner': None, 'update_controller_action': None, 'command_id': None}, 'owned': None, 'name': None}, 'id': None, 'key': None}]} - - assert expected == res - - another_workspace = WorkspaceFactory.build() - - another_credential = CredentialFactory.build(workspace=another_workspace) - session.add(another_credential) - session.commit() - - res = credentials_dao.list() - assert expected == res diff --git a/test_cases/dao/test_host.py b/test_cases/dao/test_host.py deleted file mode 100644 index 0258a6f60f3..00000000000 --- a/test_cases/dao/test_host.py +++ /dev/null @@ -1,62 +0,0 @@ -import os -import sys -sys.path.append(os.path.abspath(os.getcwd())) -import string -import random -import unittest - -from server.dao.host import HostDAO -from test_cases.factories import WorkspaceFactory, HostFactory - - -def test_list_with_multiple_workspace(app, session): - with app.app_context(): - workspace = WorkspaceFactory.build() - - hosts_dao = HostDAO(workspace) - expected = {'rows': [], 'total_rows': 0} - - res = hosts_dao.list() - assert expected == res - - host = HostFactory.build(workspace=workspace) - session.add(host) - session.commit() - expected = {'rows': [{'value': {'description': host.description, 'default_gateway': [None, None], 'vulns': 0, '_rev': None, 'owned': None, 'owner': None, 'services': 0, 'credentials': 0, 'name': host.name, '_id': None, 'os': host.os, 'interfaces': [], 'metadata': {'update_time': None, 'create_time': None, 'update_user': None, 'update_action': None, 'creator': None, 'owner': None, 'update_controller_action': None, 'command_id': None}}, '_id': host.id, 'id': None, 'key': None}], 'total_rows': 1} - res = hosts_dao.list() - assert expected == res - - another_workspace = WorkspaceFactory.build() - - another_host = HostFactory.build(workspace=another_workspace) - session.add(another_host) - session.commit() - - res = hosts_dao.list() - assert expected == res - - -def test_count_with_multiple_workspace(app, session): - with app.app_context(): - workspace = WorkspaceFactory.build() - - hosts_dao = HostDAO(workspace) - expected = {'total_count': 0} - - res = hosts_dao.count() - assert expected == res - - host = HostFactory.build(workspace=workspace) - session.add(host) - session.commit() - - another_workspace = WorkspaceFactory.build() - - another_host = HostFactory.build(workspace=another_workspace) - session.add(another_host) - session.commit() - - expected = {'total_count': 1} - - res = hosts_dao.count() - assert expected == res diff --git a/test_cases/dao/test_services.py b/test_cases/dao/test_services.py deleted file mode 100644 index f2bfcffb467..00000000000 --- a/test_cases/dao/test_services.py +++ /dev/null @@ -1,57 +0,0 @@ -import os -import sys -sys.path.append(os.path.abspath(os.getcwd())) -import string -import random -import unittest - -from server.dao.service import ServiceDAO -from test_cases.factories import WorkspaceFactory, ServiceFactory - - -def test_list_with_multiple_workspace(app, session): - with app.app_context(): - workspace = WorkspaceFactory.build() - service_dao = ServiceDAO(workspace) - expected = {'services': []} - - res = service_dao.list() - assert expected == res - - new_service = ServiceFactory.build(workspace=workspace) - session.add(new_service) - session.commit() - - res = service_dao.list() - expected = {'services': [{'value': {'status': None, 'protocol': None, 'description': new_service.description, '_rev': None, 'owned': None, 'owner': None, 'credentials': 0, 'name': new_service.name, 'version': None, '_id': None, 'ports': [int(new_service.ports)], 'metadata': {'update_time': None, 'create_time': None, 'update_user': None, 'update_action': None, 'creator': None, 'owner': None, 'update_controller_action': None, 'command_id': None}}, '_id': new_service.id, 'id': None, 'key': None, 'vulns': 0}]} - - assert expected == res - - another_workspace = WorkspaceFactory.build() - another_service = ServiceFactory.build(workspace=another_workspace) - session.add(another_service) - session.commit() - res = service_dao.list() - assert len(res['services']) == 1 - assert expected == res - - -def test_count_with_multiple_workspaces(app, session): - with app.app_context(): - workspace = WorkspaceFactory.build() - service_dao = ServiceDAO(workspace) - expected = {'total_count': 0} - res = service_dao.count() - assert expected == res - new_service = ServiceFactory.build(workspace=workspace) - session.add(new_service) - session.commit() - res = service_dao.count() - expected = {'total_count': 1} - assert expected == res - another_workspace = WorkspaceFactory.build() - another_service = ServiceFactory.build(workspace=another_workspace) - session.add(another_service) - session.commit() - res = service_dao.count() - assert expected == res diff --git a/test_cases/dao/test_vuln.py b/test_cases/dao/test_vuln.py deleted file mode 100644 index eecdab185b0..00000000000 --- a/test_cases/dao/test_vuln.py +++ /dev/null @@ -1,55 +0,0 @@ -import os -import sys -sys.path.append(os.path.abspath(os.getcwd())) -import string -import random -import unittest -from server.web import app -from server.models import db, Vulnerability, Workspace - -from server.dao.vuln import VulnerabilityDAO -from test_cases.factories import WorkspaceFactory, VulnerabilityFactory - - -def test_vulnerability_count_and_list_per_workspace_is_filtered(app, session): - """ - Verifies that the dao return the correct count from each workspace - """ - with app.app_context(): - workspace = WorkspaceFactory.build() - another_workspace = WorkspaceFactory.build() - vuln_dao = VulnerabilityDAO(workspace) - another_vuln_dao = VulnerabilityDAO(another_workspace) - vuln_1 = VulnerabilityFactory.build(vuln_type='Vulnerability', workspace=workspace) - vuln_2 = VulnerabilityFactory.build(vuln_type='Vulnerability', workspace=another_workspace) - db.session.add(vuln_1) - db.session.add(vuln_2) - db.session.commit() - ws_count = vuln_dao.count() - another_ws_count = another_vuln_dao.count() - ws_expected = {'total_count': 1, 'web_vuln_count': 0, 'vuln_count': 1} - another_expected = {'total_count': 1, 'web_vuln_count': 0, 'vuln_count': 1} - assert ws_count == ws_expected - assert another_ws_count == another_expected - ws_list = vuln_dao.list() - - assert vuln_1.id == ws_list['vulnerabilities'][0]['_id'] - another_ws_list = another_vuln_dao.list() - assert vuln_2.id == another_ws_list['vulnerabilities'][0]['_id'] - - -def test_count_by_type(app, session): - with app.app_context(): - workspace = WorkspaceFactory.build() - vuln_dao = VulnerabilityDAO(workspace) - res = vuln_dao.count() - expected = {'total_count': 0, 'web_vuln_count': 0, 'vuln_count': 0} - assert expected == res - vuln = VulnerabilityFactory.build(vuln_type='Vulnerability', workspace=workspace) - vuln_web = VulnerabilityFactory.build(vuln_type='VulnerabilityWeb', workspace=workspace) - db.session.add(vuln) - db.session.add(vuln_web) - db.session.commit() - res = vuln_dao.count() - expected = {'total_count': 2, 'web_vuln_count': 1, 'vuln_count': 1} - assert expected == res diff --git a/test_cases/plugins_controller_unittests.py b/test_cases/test_plugins_controller.py similarity index 100% rename from test_cases/plugins_controller_unittests.py rename to test_cases/test_plugins_controller.py From 881c5fcda32a3e968b9386a2542e715d56ae30a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 4 Oct 2017 17:59:32 -0300 Subject: [PATCH 0275/1506] Change Jenkinsfile to use pytest instead of nose Code coverage won't run --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8d0176de1f4..1030a25b99d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -19,9 +19,10 @@ node { stage ("Install Application Dependencies") { sh """ source ${ENV_PATH}/bin/activate - pip install nose nosexcover virtualenv responses + pip install virtualenv responses pip install -r $WORKSPACE/requirements.txt pip install -r $WORKSPACE/requirements_server.txt + pip install -r $WORKSPACE/requirements_dev.txt deactivate """ } @@ -43,7 +44,7 @@ node { try { sh """ source ${ENV_PATH}/bin/activate - cd $WORKSPACE && nosetests --verbose --with-xunit --xunit-file=$WORKSPACE/xunit.xml --with-xcoverage --xcoverage-file=$WORKSPACE/coverage.xml -ignore-files='.*dont_run_rest_controller_apis.*' --no-byte-compile -v `find test_cases -name '*.py'| grep -v dont_run` || : + cd $WORKSPACE && pytest -v --junitxml=$WORKSPACE/xunit.xml || : deactivate """ step([$class: 'CoberturaPublisher', autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: '**/coverage.xml', failNoReports: false, failUnhealthy: false, failUnstable: false, maxNumberOfBuilds: 0, onlyStable: false, sourceEncoding: 'ASCII', zoomCoverageChart: false]) From 23257f4f8561c5604439ff082be0a50b78533ed2 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 4 Oct 2017 18:19:35 -0300 Subject: [PATCH 0276/1506] Add backward compatible vulns api views --- server/api/modules/vulns.py | 142 ++++++++++++++++++++++++++- server/app.py | 2 + server/importer.py | 2 +- server/models.py | 2 +- test_cases/factories.py | 6 +- test_cases/test_api_vulnerability.py | 71 ++++++++++++++ 6 files changed, 219 insertions(+), 6 deletions(-) create mode 100644 test_cases/test_api_vulnerability.py diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index b75bb458d69..0fbe13fa628 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -1,15 +1,153 @@ # Faraday Penetration Test IDE # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information +import time +import logging from flask import request, jsonify, abort from flask import Blueprint +from marshmallow import fields + +from server.api.base import AutoSchema, ReadWriteWorkspacedView +from server.models import ( + db, + VulnerabilityGeneric, + TagObject, + Tag +) from server.utils.logger import get_logger -from server.utils.web import gzipped, validate_workspace,\ - get_integer_parameter, filter_request_args +from server.utils.web import ( + gzipped, + validate_workspace, + get_integer_parameter, + filter_request_args +) from server.dao.vuln import VulnerabilityDAO vulns_api = Blueprint('vulns_api', __name__) +logger = logging.getLogger(__name__) + + +class VulnerabilityGenericSchema(AutoSchema): + _id = fields.Integer(dump_only=True, attribute='id') + website = fields.String(default='') + _rev = fields.String(default='') + owned = fields.Boolean(default=False) + owner = fields.Method('get_owner_name') + impact = fields.Method('get_impact') + policyviolations = fields.Method('get_policyviolations') + method = fields.String(default='') + params = fields.String(default='') + refs = fields.Method('get_refs') + issuetracker = fields.Method('get_issuetracker') + parent = fields.Method('get_parent') + tags = fields.Method('get_tags') + easeofresolution = fields.String(dump_only=True, attribute='ease_of_resolution') + hostnames = fields.Method('get_hostnames') + pname = fields.String(dump_only=True, attribute='parameter_name', default='') + path = fields.String(default='') + response = fields.String(default='') + desc = fields.String(dump_only=True, attribute='description') + obj_id = fields.String(dump_only=True, attribute='id') + request = fields.String(default='') + _attachments = fields.Method('get_attachments') + target = fields.String(default='') # TODO: review this attribute + query = fields.String(dump_only=True, attribute='query_string', default='') + metadata = fields.Method('get_metadata') + service = fields.Integer(dump_only=True, attribute='service.id') + host = fields.Integer(dump_only=True, attribute='host.id') + + def get_metadata(self, obj): + return { + "command_id": "e1a042dd0e054c1495e1c01ced856438", + "create_time": time.mktime(obj.create_date.utctimetuple()), + "creator": "Metasploit", + "owner": "", "update_action": 0, + "update_controller_action": "No model controller call", + "update_time": time.mktime(obj.update_date.utctimetuple()), + "update_user": "" + } + + def get_attachments(self, obj): + # TODO: retrieve obj attachments + return [] + + def get_hostnames(self, obj): + if obj.host: + return [hostname.name for hostname in obj.host.hostnames] + if obj.service: + return [hostname.name for hostname in obj.service.host.hostnames] + logger.info('Vulnerabilit without host and service. Check invatiant of obj with id {0}'.format(obj.id)) + return [] + + def get_easeofresolution(self, obj): + return obj.ease_of_resolution + + def get_tags(self, obj): + return [tag.name for tag in db.session.query(TagObject, Tag).filter_by( + object_type=obj.__class__.__name__, + object_id=obj.id + ).all()] + + def get_parent(self, obj): + if getattr(obj, 'service', None): + return obj.service.id + if getattr(obj, 'host', None): + return obj.host.id + return + + def get_issuetracker(self, obj): + return {} + + def get_refs(self, obj): + return [ref.name for ref in obj.references] + + def get_policyviolations(self, obj): + return [pv.name for pv in obj.policy_violations] + + def get_impact(self, obj): + return { + 'accountability': obj.impact_accountability, + 'availability': obj.impact_availability, + 'confidentiality': obj.impact_confidentiality, + 'integrity': obj.impact_integrity + } + + def get_owner_name(self, obj): + return obj.creator.username + + class Meta: + model = VulnerabilityGeneric + fields = ( + '_id', 'status', + 'website', 'issuetracker', 'description', 'parent', + 'tags', 'severity', '_rev', 'easeofresolution', 'owned', + 'hostnames', 'pname', 'query', 'owner', + 'path', 'data', 'response', 'refs', + 'desc', 'impact', 'confirmed', 'name', + 'service', 'obj_id', 'type', 'policyviolations', + 'request', '_attachments', 'params', + 'target', 'resolution', 'method', 'metadata') + + +class VulnerabilityView(ReadWriteWorkspacedView): + route_base = 'vulns' + model_class = VulnerabilityGeneric + schema_class = VulnerabilityGenericSchema + + def _envelope_list(self, objects, pagination_metadata=None): + vulns = [] + for vuln in objects: + vulns.append({ + 'id': vuln['_id'], + 'key': vuln['_id'], + 'value': vuln + }) + return { + 'vulnerabilities': vulns, + } + +VulnerabilityView.register(vulns_api) @vulns_api.route('/ws//vulns', methods=['GET']) diff --git a/server/app.py b/server/app.py index c9bc560bfe8..7255e21fc64 100644 --- a/server/app.py +++ b/server/app.py @@ -67,6 +67,7 @@ def create_app(db_connection_string=None, testing=None): from server.api.modules.workspaces import workspace_api from server.api.modules.doc import doc_api from server.api.modules.vuln_csv import vuln_csv_api + from server.api.modules.vulns import vulns_api from server.api.modules.hosts import host_api from server.api.modules.licenses import license_api from server.api.modules.commandsrun import commandsrun_api @@ -77,6 +78,7 @@ def create_app(db_connection_string=None, testing=None): app.register_blueprint(workspace_api) app.register_blueprint(doc_api) app.register_blueprint(vuln_csv_api) + app.register_blueprint(vulns_api) app.register_blueprint(host_api) app.register_blueprint(license_api) app.register_blueprint(commandsrun_api) diff --git a/server/importer.py b/server/importer.py index 0e65250ba1b..b1cf5f857c0 100644 --- a/server/importer.py +++ b/server/importer.py @@ -419,7 +419,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation if document['type'] == 'VulnerabilityWeb': - vulnerability.query = document.get('query') + vulnerability.query_string = document.get('query') vulnerability.request = document.get('request') vulnerability.response = document.get('response') diff --git a/server/models.py b/server/models.py index e128be9e92d..796deff58e0 100644 --- a/server/models.py +++ b/server/models.py @@ -366,7 +366,7 @@ class VulnerabilityWeb(VulnerabilityGeneric): parameters = Column(String(500), nullable=True) parameter_name = Column(String(250), nullable=True) path = Column(String(500), nullable=True) - query = Column(Text(), nullable=True) + query_string = Column(Text(), nullable=True) request = Column(Text(), nullable=True) response = Column(Text(), nullable=True) website = Column(String(250), nullable=True) diff --git a/test_cases/factories.py b/test_cases/factories.py index 001ecdd19b7..19158ba800f 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -132,7 +132,7 @@ class VulnerabilityFactory(WorkspaceObjectFactory): name = FuzzyText() description = FuzzyText() - # host = factory.SubFactory(HostFactory) # TODO: Move to generic class + host = factory.SubFactory(HostFactory) # TODO: Move to generic class # service = factory.SubFactory(ServiceFactory) # TODO: Move to generic class workspace = factory.SubFactory(WorkspaceFactory) creator = factory.SubFactory(UserFactory) @@ -153,7 +153,9 @@ class Meta: sqlalchemy_session = db.session -class VulnerabilityCodeFactory(VulnerabilityFactory): +class VulnerabilityCodeFactory(WorkspaceObjectFactory): + name = FuzzyText() + description = FuzzyText() start_line = FuzzyInteger(1, 5000) source_code = factory.SubFactory(SourceCodeFactory) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py new file mode 100644 index 00000000000..7467269c11b --- /dev/null +++ b/test_cases/test_api_vulnerability.py @@ -0,0 +1,71 @@ +#-*- coding: utf8 -*- +"""Tests for many API endpoints that do not depend on workspace_name""" + +import pytest + +from server.api.modules.vulns import VulnerabilityView +from test_cases import factories +from test_api_workspaced_base import ListTestsMixin, API_PREFIX, GenericAPITest +from server.models import ( + Vulnerability, + VulnerabilityWeb, + Workspace, +) +from server.api.modules.commandsrun import CommandView +from server.api.modules.workspaces import WorkspaceView + + +@pytest.mark.usefixtures('logged_user') +class TestListCommandView(GenericAPITest): + model = Vulnerability + factory = factories.VulnerabilityFactory + api_endpoint = 'vulns' + #unique_fields = ['ip'] + #update_fields = ['ip', 'description', 'os'] + view_class = VulnerabilityView + + def test_(self, test_client, second_workspace, session): + self.factory.create(workspace=second_workspace) + session.commit() + res = test_client.get(self.url()) + assert res.status_code == 200 + assert 'vulnerabilities' in res.json + for vuln in res.json['vulnerabilities']: + assert set([u'id', u'key', u'value']) == set(vuln.keys()) + object_properties = [ + u'status', + u'website', + u'issuetracker', + u'description', + u'parent', + u'tags', + u'severity', + u'_rev', + u'easeofresolution', + u'owned', + u'hostnames', + u'pname', + u'query', + u'owner', + u'path', + u'data', + u'response', + u'refs', + u'desc', + u'impact', + u'confirmed', + u'name', + u'service', + u'obj_id', + u'type', + u'policyviolations', + u'request', + u'_attachments', + u'params', + u'target', + u'_id', + u'resolution', + u'method', + u'metadata' + ] + assert set(object_properties) == set(vuln['value'].keys()) \ No newline at end of file From f23db6c6933d54eb35f7f9df0d233fd6900fcbb1 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 4 Oct 2017 18:19:51 -0300 Subject: [PATCH 0277/1506] Fix a bug in importer when hostnames value is None --- server/importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index b1cf5f857c0..d426f0b9556 100644 --- a/server/importer.py +++ b/server/importer.py @@ -298,7 +298,7 @@ def merge_with_host(self, host, interface, workspace): if type(interface['hostnames']) in (str, unicode): interface['hostnames'] = [interface['hostnames']] - for hostname_str in interface['hostnames']: + for hostname_str in interface['hostnames'] or []: if not hostname_str: # skip empty hostnames continue From 497521fdca84380a39a51e8e2a07950b1ff8725d Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 4 Oct 2017 18:52:03 -0300 Subject: [PATCH 0278/1506] Add services api view --- server/api/modules/services.py | 56 +++++++++++++++++++++++++++++++++ test_cases/test_api_services.py | 47 +++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 test_cases/test_api_services.py diff --git a/server/api/modules/services.py b/server/api/modules/services.py index 354713e7c40..c41c4ba12f7 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -1,9 +1,14 @@ # Faraday Penetration Test IDE # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information +import time import flask from flask import Blueprint +from marshmallow import fields + +from server.api.base import AutoSchema, ReadWriteWorkspacedView +from server.models import Service from server.utils.logger import get_logger from server.dao.service import ServiceDAO from server.utils.web import gzipped, validate_workspace, get_integer_parameter @@ -12,6 +17,57 @@ services_api = Blueprint('services_api', __name__) +class ServiceSchema(AutoSchema): + _id = fields.Integer(dump_only=True, attribute='id') + _rev = fields.String(default='') + metadata = fields.Method('get_metadata') + owned = fields.Boolean(default=False) + owner = fields.String(dump_only=True, attribute='creator.username') + ports = fields.Method('get_ports') + + def get_ports(self, obj): + return [obj.port] + + def get_metadata(self, obj): + return { + "command_id": "e1a042dd0e054c1495e1c01ced856438", + "create_time": time.mktime(obj.create_date.utctimetuple()), + "creator": "Metasploit", + "owner": "", "update_action": 0, + "update_controller_action": "No model controller call", + "update_time": time.mktime(obj.update_date.utctimetuple()), + "update_user": "" + } + + class Meta: + model = Service + fields = ('_id', 'status', + 'protocol', 'description', '_rev', + 'owned', 'owner', 'credentials', + 'name', 'version', '_id', 'ports', + 'metadata') + + +class ServiceView(ReadWriteWorkspacedView): + route_base = 'services' + model_class = Service + schema_class = ServiceSchema + + def _envelope_list(self, objects, pagination_metadata=None): + services = [] + for service in objects: + services.append({ + 'id': service['_id'], + 'key': service['_id'], + 'value': service + }) + return { + 'services': services, + } + +ServiceView.register(services_api) + + @services_api.route('/ws//services', methods=['GET']) @gzipped def list_services(workspace=None): diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py new file mode 100644 index 00000000000..e0a8a1c991b --- /dev/null +++ b/test_cases/test_api_services.py @@ -0,0 +1,47 @@ +#-*- coding: utf8 -*- +"""Tests for many API endpoints that do not depend on workspace_name""" + +import pytest + +from server.api.modules.vulns import VulnerabilityView +from test_cases import factories +from test_api_workspaced_base import ListTestsMixin, API_PREFIX, GenericAPITest +from server.models import ( + Service +) +from server.api.modules.commandsrun import CommandView +from server.api.modules.workspaces import WorkspaceView + + +@pytest.mark.usefixtures('logged_user') +class TestListServiceView(GenericAPITest): + model = Service + factory = factories.ServiceFactory + api_endpoint = 'services' + #unique_fields = ['ip'] + #update_fields = ['ip', 'description', 'os'] + view_class = VulnerabilityView + + def test_(self, test_client, second_workspace, session): + self.factory.create(workspace=second_workspace) + session.commit() + res = test_client.get(self.url()) + assert res.status_code == 200 + assert 'services' in res.json + for vuln in res.json['services']: + assert set([u'id', u'key', u'value']) == set(vuln.keys()) + object_properties = [ + u'status', + u'protocol', + u'description', + u'_rev', + u'owned', + u'owner', + u'credentials', + u'name', + u'version', + u'_id', + u'ports', + u'metadata' + ] + assert set(object_properties) == set(vuln['value'].keys()) \ No newline at end of file From 83a8d83826a1249b07a09a95a9c0172062e35d71 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 4 Oct 2017 18:52:25 -0300 Subject: [PATCH 0279/1506] Change testname --- test_cases/test_api_vulnerability.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 7467269c11b..6d9456ed8ad 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -16,7 +16,7 @@ @pytest.mark.usefixtures('logged_user') -class TestListCommandView(GenericAPITest): +class TestListVulnerabilityView(GenericAPITest): model = Vulnerability factory = factories.VulnerabilityFactory api_endpoint = 'vulns' From f03ccd5b0e55343cfac218913aebdaeb4b3e9c9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 4 Oct 2017 19:15:44 -0300 Subject: [PATCH 0280/1506] Change virtualenv path To make jenkins work --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1030a25b99d..f91b356a096 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,7 +8,7 @@ node { } stage("Install Python Virtual Enviroment") { - sh "virtualenv --no-site-packages ${ENV_PATH}" + sh "/usr/local/bin/virtualenv --no-site-packages ${ENV_PATH}" } // Get the latest version of our application code. From 9eb4534286597b2c76f04b8a40a13dffc6ca5f85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 4 Oct 2017 19:21:45 -0300 Subject: [PATCH 0281/1506] Install extra packages in jenkins --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index f91b356a096..65893ed1f6b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -22,6 +22,7 @@ node { pip install virtualenv responses pip install -r $WORKSPACE/requirements.txt pip install -r $WORKSPACE/requirements_server.txt + pip install -r $WORKSPACE/requirements_extras.txt pip install -r $WORKSPACE/requirements_dev.txt deactivate """ From aef41f5d1574fb4cbc754b6ec09d4487a57e4229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 4 Oct 2017 19:38:15 -0300 Subject: [PATCH 0282/1506] Use github version of flask-classful Required to make the API work, until 0.14 is released --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 65893ed1f6b..5c8c9cd37d2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,6 +24,8 @@ node { pip install -r $WORKSPACE/requirements_server.txt pip install -r $WORKSPACE/requirements_extras.txt pip install -r $WORKSPACE/requirements_dev.txt + pip uninstall -y flask-classful + pip install -e git+https://github.com/teracyhq/flask-classful.git@baf37cd8a1f9f1124d32c1376135968172aa6b7b#egg=Flask_Classful deactivate """ } From 0d8a12eea6ab7261b028d4167288ad50948d8e50 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 5 Oct 2017 10:55:22 -0300 Subject: [PATCH 0283/1506] Fix two bugs in importer --- server/importer.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/server/importer.py b/server/importer.py index d426f0b9556..088e8f618b0 100644 --- a/server/importer.py +++ b/server/importer.py @@ -347,9 +347,13 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation 'closed': 'closed', 'down': 'closed', 'filtered': 'filtered', - 'open|filtered': 'filtered' + 'open|filtered': 'filtered', + 'unknown': 'closed' } - service.status = status_mapper[document.get('status', 'open')] + couchdb_status = document.get('status', 'open') + if couchdb_status.lower() == 'unkown': + logger.warn('Service with status {0} found! Status will default to closed. Host is {1}'.format(couchdb_status, host.ip)) + service.status = status_mapper[couchdb_status] service.version = document.get('version') service.workspace = workspace @@ -437,7 +441,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation self.add_references(document, vulnerability, workspace) self.add_policy_violations(document, vulnerability, workspace) - yield vulnerability + yield vulnerability def add_policy_violations(self, document, vulnerability, workspace): for policy_violation in document.get('policyviolations', []): From 2f2f32fe7a0e346431ef02f189e7530c86723353 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 5 Oct 2017 11:09:13 -0300 Subject: [PATCH 0284/1506] Some vulns were open in couchdb --- server/importer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/importer.py b/server/importer.py index 088e8f618b0..d97efa7d509 100644 --- a/server/importer.py +++ b/server/importer.py @@ -434,6 +434,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation vulnerability.parameters = params if params is not None else u'' status_map = { 'opened': 'open', + 'open': 'open', 'closed': 'closed', } status = status_map[document.get('status', 'opened')] From bcdd28a916c3a60fee0b4a8e4f3c3eb8cb70b219 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 5 Oct 2017 11:16:09 -0300 Subject: [PATCH 0285/1506] Fix count url --- server/www/scripts/commons/providers/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index 57933640a90..2c66391b6f3 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -274,7 +274,7 @@ angular.module("faradayApp") } ServerAPI.getServicesBy = function(wsName, what) { - var url = createGetUrl(wsName, 'services') + 'count/'; + var url = createGetUrl(wsName, 'services') + '/count/'; return get(url, {"group_by": what}) } From 0648d4b24d8de94abfd52be0bf62ef4f4915a270 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 5 Oct 2017 18:33:22 -0300 Subject: [PATCH 0286/1506] Add count api view --- server/api/base.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/server/api/base.py b/server/api/base.py index 2f9139b4f11..5c39737d02b 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -1,9 +1,12 @@ -import flask import json +import flask +from flask import abort +from sqlalchemy import inspect from flask_classful import FlaskView from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.inspection import inspect +from sqlalchemy import func from werkzeug.routing import parse_rule from marshmallow import Schema from marshmallow.compat import with_metaclass @@ -323,6 +326,31 @@ class DeleteWorkspacedMixin(DeleteMixin): pass +class CountWorkspacedMixin(object): + + def count(self, **kwargs): + res = { + 'groups': [], + 'total_count': 0 + } + group_by = flask.request.args.get('group_by', None) + if not group_by or group_by not in inspect(self.model_class).attrs: + abort(404) + + workspace_name = kwargs.pop('workspace_name') + # using format is not a great practice. + # the user input is group_by, however it's filtered by column name. + group_by = '{0}.{1}'.format(self.model_class.__name__, group_by) + count = db.session.query(self.model_class).join(Workspace).group_by(group_by).filter( + Workspace.name == workspace_name).values(group_by, func.count(group_by)) + for key, count in count: + res['groups'].append( + {'count': count, 'name': key} + ) + res['total_count'] +=1 + return res + + class ReadWriteView(CreateMixin, UpdateMixin, DeleteMixin, @@ -334,6 +362,7 @@ class ReadWriteView(CreateMixin, class ReadWriteWorkspacedView(CreateWorkspacedMixin, UpdateWorkspacedMixin, DeleteWorkspacedMixin, + CountWorkspacedMixin, ReadOnlyWorkspacedView): """A generic workspaced view with list, retrieve and create endpoints""" From db31e0c2c431166438a49fa2c626f4faba7e3080 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 5 Oct 2017 18:55:07 -0300 Subject: [PATCH 0287/1506] Improve count query --- server/api/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/api/base.py b/server/api/base.py index 5c39737d02b..2b0ef1b552b 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -340,7 +340,9 @@ def count(self, **kwargs): workspace_name = kwargs.pop('workspace_name') # using format is not a great practice. # the user input is group_by, however it's filtered by column name. - group_by = '{0}.{1}'.format(self.model_class.__name__, group_by) + table_name = inspect(self.model_class).tables[0].name + group_by = '{0}.{1}'.format(table_name, group_by) + count = db.session.query(self.model_class).join(Workspace).group_by(group_by).filter( Workspace.name == workspace_name).values(group_by, func.count(group_by)) for key, count in count: From 152fb57fd172ce45b7dac5af0ced710abc3c825a Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 5 Oct 2017 18:55:16 -0300 Subject: [PATCH 0288/1506] Fix another count url --- server/www/scripts/commons/providers/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index 2c66391b6f3..b956646f0c0 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -289,7 +289,7 @@ angular.module("faradayApp") ServerAPI.getVulnsBySeverity = function(wsName, confirmed) { - var url = createGetUrl(wsName, 'vulns') + 'count/'; + var url = createGetUrl(wsName, 'vulns') + '/count/'; var payload = {'group_by': 'severity'} if (confirmed !== undefined) { From 6451b966f91cc4cc755b222684e6c7786e04ba66 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 6 Oct 2017 15:16:06 -0300 Subject: [PATCH 0289/1506] minor fixes --- server/api/base.py | 2 +- server/api/modules/vulns.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 2b0ef1b552b..b6f801b012b 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -349,7 +349,7 @@ def count(self, **kwargs): res['groups'].append( {'count': count, 'name': key} ) - res['total_count'] +=1 + res['total_count'] += count return res diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 0fbe13fa628..e20457e11d2 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -77,7 +77,7 @@ def get_hostnames(self, obj): return [hostname.name for hostname in obj.host.hostnames] if obj.service: return [hostname.name for hostname in obj.service.host.hostnames] - logger.info('Vulnerabilit without host and service. Check invatiant of obj with id {0}'.format(obj.id)) + logger.info('Vulnerability without host and service. Check invariant in obj with id {0}'.format(obj.id)) return [] def get_easeofresolution(self, obj): From 7a4730f74de30838d14d440dcc089de17f184cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 6 Oct 2017 16:29:32 -0300 Subject: [PATCH 0290/1506] Fix bug in vulns API with null services When the service is not set (standard vulnerability) the "service" field wasn't shown. Also fix this for "host" field. --- server/api/modules/vulns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index e20457e11d2..378e319a878 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -54,8 +54,8 @@ class VulnerabilityGenericSchema(AutoSchema): target = fields.String(default='') # TODO: review this attribute query = fields.String(dump_only=True, attribute='query_string', default='') metadata = fields.Method('get_metadata') - service = fields.Integer(dump_only=True, attribute='service.id') - host = fields.Integer(dump_only=True, attribute='host.id') + service = fields.Integer(dump_only=True, attribute='service_id') + host = fields.Integer(dump_only=True, attribute='host_id') def get_metadata(self, obj): return { From 2da0bfd81ba3d9f71679457e9be48dd0f3671282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 6 Oct 2017 17:00:29 -0300 Subject: [PATCH 0291/1506] Improve vulnerability factories and fix a test --- test_cases/factories.py | 32 +++++++++++++++++++++-------- test_cases/models/test_workspace.py | 4 ++-- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/test_cases/factories.py b/test_cases/factories.py index 19158ba800f..33ef322b81a 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -1,3 +1,4 @@ +import random import factory import datetime @@ -128,22 +129,37 @@ class Meta: sqlalchemy_session = db.session -class VulnerabilityFactory(WorkspaceObjectFactory): - +class VulnerabilityGenericFactory(WorkspaceObjectFactory): name = FuzzyText() description = FuzzyText() - host = factory.SubFactory(HostFactory) # TODO: Move to generic class - # service = factory.SubFactory(ServiceFactory) # TODO: Move to generic class - workspace = factory.SubFactory(WorkspaceFactory) creator = factory.SubFactory(UserFactory) severity = FuzzyChoice(['critical', 'high']) + +class VulnerabilityFactory(VulnerabilityGenericFactory): + + host = factory.SubFactory(HostFactory) + service = factory.SubFactory(ServiceFactory) + + @classmethod + def _after_postgeneration(cls, obj, create, results=None): + super(VulnerabilityFactory, cls)._after_postgeneration( + obj, create, results) + if obj.host and obj.service: + # Setting both service and host to a vuln is not allowed. + # This will pick one of them randomly. + # TODO: Check is this is recommended + if random.choice([True, False]): + obj.host = None + else: + obj.service = None + class Meta: model = Vulnerability sqlalchemy_session = db.session -class VulnerabilityWebFactory(VulnerabilityFactory): +class VulnerabilityWebFactory(VulnerabilityGenericFactory): method = FuzzyChoice(['GET', 'POST', 'PUT', 'PATCH' 'DELETE']) parameter_name = FuzzyText() service = factory.SubFactory(ServiceFactory) @@ -153,9 +169,7 @@ class Meta: sqlalchemy_session = db.session -class VulnerabilityCodeFactory(WorkspaceObjectFactory): - name = FuzzyText() - description = FuzzyText() +class VulnerabilityCodeFactory(VulnerabilityGenericFactory): start_line = FuzzyInteger(1, 5000) source_code = factory.SubFactory(SourceCodeFactory) diff --git a/test_cases/models/test_workspace.py b/test_cases/models/test_workspace.py index 5c4c757926a..c4acbd62683 100644 --- a/test_cases/models/test_workspace.py +++ b/test_cases/models/test_workspace.py @@ -21,9 +21,9 @@ def populate_workspace(workspace): # Create standard vulns host_vulns = VulnerabilityFactory.create_batch( - STANDARD_VULN_COUNT[0], workspace=workspace, host=host) + STANDARD_VULN_COUNT[0], workspace=workspace, host=host, service=None) service_vulns = VulnerabilityFactory.create_batch( - STANDARD_VULN_COUNT[1], workspace=workspace, service=service) + STANDARD_VULN_COUNT[1], workspace=workspace, service=service, host=None) # Create web vulns web_vulns = VulnerabilityWebFactory.create_batch( From f1913849a905a8a006ed11a8e78cb16a5177ae29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 6 Oct 2017 17:45:18 -0300 Subject: [PATCH 0292/1506] Change name of marshmallow fields test file --- test_cases/{test_schemas.py => test_marshmallow_fields.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test_cases/{test_schemas.py => test_marshmallow_fields.py} (100%) diff --git a/test_cases/test_schemas.py b/test_cases/test_marshmallow_fields.py similarity index 100% rename from test_cases/test_schemas.py rename to test_cases/test_marshmallow_fields.py From 520fb03c4d74c55c2badbed346f3cee24e84ecbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 6 Oct 2017 18:43:57 -0300 Subject: [PATCH 0293/1506] Add PrimaryKeyRelatedField implementation Inspired on Django Rest Framework one: http://www.django-rest-framework.org/api-guide/relations/#primarykeyrelatedfield --- server/schemas.py | 37 +++++++++++++---- test_cases/test_marshmallow_fields.py | 59 ++++++++++++++++++++++++++- 2 files changed, 87 insertions(+), 9 deletions(-) diff --git a/server/schemas.py b/server/schemas.py index 63d4fcc49cd..c4b3f3776f6 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -3,6 +3,33 @@ from marshmallow.exceptions import ValidationError +class JSTimestampField(fields.Field): + """A field to serialize datetime objects into javascript + compatible timestamps (like time.time()) * 1000""" + + def _serialize(self, value, attr, obj): + if value is not None: + return int(time.mktime(value.timetuple()) * 1000) + + +class PrimaryKeyRelatedField(fields.Field): + def __init__(self, field_name='id', *args, **kwargs): + self.field_name = field_name + self.many = kwargs.get('many', False) + super(PrimaryKeyRelatedField, self).__init__(*args, **kwargs) + + def _serialize(self, value, attr, obj): + if self.many: + ret = [] + for item in value: + ret.append(getattr(item, self.field_name)) + return ret + else: + if value is None: + return None + return getattr(value, self.field_name) + + class SelfNestedField(fields.Field): """A field to make namespaced schemas. It allows to have a field whose contents are the dump of the same object with @@ -21,11 +48,5 @@ def _serialize(self, value, attr, obj): raise ValidationError(errors, data=ret) return ret - -class JSTimestampField(fields.Field): - """A field to serialize datetime objects into javascript - compatible timestamps (like time.time()) * 1000""" - - def _serialize(self, value, attr, obj): - if value is not None: - return int(time.mktime(value.timetuple()) * 1000) + def _deserialize(self, value, attr, data): + raise NotImplementedError("Only dump is implemented for now") diff --git a/test_cases/test_marshmallow_fields.py b/test_cases/test_marshmallow_fields.py index 1c96b330098..d79cae44d22 100644 --- a/test_cases/test_marshmallow_fields.py +++ b/test_cases/test_marshmallow_fields.py @@ -1,8 +1,13 @@ import time import datetime +import pytest from collections import namedtuple from marshmallow import Schema, fields -from server.schemas import SelfNestedField, JSTimestampField +from server.schemas import ( + JSTimestampField, + PrimaryKeyRelatedField, + SelfNestedField, +) Place = namedtuple('Place', ['name', 'x', 'y']) @@ -35,3 +40,55 @@ def test_parses_current_datetime(self): def test_parses_null_datetime(self): assert JSTimestampField()._serialize(None, None, None) is None + + +User = namedtuple('User', ['username', 'blogposts']) +Blogpost = namedtuple('Blogpost', ['id', 'title']) +Profile = namedtuple('Profile', ['user', 'first_name']) + + +class UserSchema(Schema): + username = fields.String() + blogposts = PrimaryKeyRelatedField(many=True) + + +class ProfileSchema(Schema): + user = PrimaryKeyRelatedField('username') + first_name = fields.String() + + +class TestPrimaryKeyRelatedField: + @pytest.fixture(autouse=True) + def load_data(self): + self.blogposts = [ + Blogpost(1, 'aaa'), + Blogpost(2, 'bbb'), + Blogpost(3, 'ccc'), + ] + self.user = User('test', self.blogposts) + self.profile = Profile(self.user, 'david') + + def serialize(self, obj=None, schema=UserSchema): + return schema(strict=True).dump(obj or self.user).data + + def test_many_id(self): + assert self.serialize() == {"username": "test", + "blogposts": [1, 2, 3]} + + def test_many_title(self): + class UserSchemaWithTitle(UserSchema): + blogposts = PrimaryKeyRelatedField('title', many=True) + data = self.serialize(schema=UserSchemaWithTitle) + assert data == {"username": "test", "blogposts": ['aaa', 'bbb', 'ccc']} + + def test_single(self): + assert self.serialize(self.profile, ProfileSchema) == { + "user": "test", + "first_name": "david" + } + + def test_single_with_none_value(self): + assert self.serialize(Profile(None, 'other'), ProfileSchema) == { + "user": None, + "first_name": "other" + } From 0991afb4e37451a55bac0636f9ed6c5fd04c1c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 6 Oct 2017 18:45:29 -0300 Subject: [PATCH 0294/1506] Fix bug in the API with vulns without creator --- server/api/modules/vulns.py | 6 ++---- test_cases/test_api_vulnerability.py | 14 +++++++++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 378e319a878..ff298467388 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -22,6 +22,7 @@ get_integer_parameter, filter_request_args ) +from server.schemas import PrimaryKeyRelatedField from server.dao.vuln import VulnerabilityDAO vulns_api = Blueprint('vulns_api', __name__) @@ -33,7 +34,7 @@ class VulnerabilityGenericSchema(AutoSchema): website = fields.String(default='') _rev = fields.String(default='') owned = fields.Boolean(default=False) - owner = fields.Method('get_owner_name') + owner = PrimaryKeyRelatedField('username', attribute='creator') impact = fields.Method('get_impact') policyviolations = fields.Method('get_policyviolations') method = fields.String(default='') @@ -113,9 +114,6 @@ def get_impact(self, obj): 'integrity': obj.impact_integrity } - def get_owner_name(self, obj): - return obj.creator.username - class Meta: model = VulnerabilityGeneric fields = ( diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 6d9456ed8ad..24cf578b216 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -68,4 +68,16 @@ def test_(self, test_client, second_workspace, session): u'method', u'metadata' ] - assert set(object_properties) == set(vuln['value'].keys()) \ No newline at end of file + assert set(object_properties) == set(vuln['value'].keys()) + + def test_handles_vuln_with_no_creator(self, + workspace, + test_client, + vulnerability_factory, + session): + # This can happen when a user is deleted but its objects persist + vuln = vulnerability_factory.create(workspace=workspace, creator=None) + session.commit() + res = test_client.get(self.url(vuln)) + assert res.status_code == 200 + assert res.json['owner'] is None From 77f87142940d1930d89b46c06420d3129dfb93b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 6 Oct 2017 18:48:17 -0300 Subject: [PATCH 0295/1506] Move vuln schema Meta definition above methods To improve code readability --- server/api/modules/vulns.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index ff298467388..1e0e7f77847 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -58,6 +58,19 @@ class VulnerabilityGenericSchema(AutoSchema): service = fields.Integer(dump_only=True, attribute='service_id') host = fields.Integer(dump_only=True, attribute='host_id') + class Meta: + model = VulnerabilityGeneric + fields = ( + '_id', 'status', + 'website', 'issuetracker', 'description', 'parent', + 'tags', 'severity', '_rev', 'easeofresolution', 'owned', + 'hostnames', 'pname', 'query', 'owner', + 'path', 'data', 'response', 'refs', + 'desc', 'impact', 'confirmed', 'name', + 'service', 'obj_id', 'type', 'policyviolations', + 'request', '_attachments', 'params', + 'target', 'resolution', 'method', 'metadata') + def get_metadata(self, obj): return { "command_id": "e1a042dd0e054c1495e1c01ced856438", @@ -114,19 +127,6 @@ def get_impact(self, obj): 'integrity': obj.impact_integrity } - class Meta: - model = VulnerabilityGeneric - fields = ( - '_id', 'status', - 'website', 'issuetracker', 'description', 'parent', - 'tags', 'severity', '_rev', 'easeofresolution', 'owned', - 'hostnames', 'pname', 'query', 'owner', - 'path', 'data', 'response', 'refs', - 'desc', 'impact', 'confirmed', 'name', - 'service', 'obj_id', 'type', 'policyviolations', - 'request', '_attachments', 'params', - 'target', 'resolution', 'method', 'metadata') - class VulnerabilityView(ReadWriteWorkspacedView): route_base = 'vulns' From 3a1ce6475da5b7adb7c2d54d888ecf3bcf5e94d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 6 Oct 2017 19:09:06 -0300 Subject: [PATCH 0296/1506] Do automatic test factories discovery Previously I used a list to manually set the classes to register. I prefer to this automagically so if I forget to register a class I don't see ugly errors --- test_cases/conftest.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 8b1bff45501..b6b39f435e5 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -1,7 +1,9 @@ import os import sys import json +import inspect import pytest +from factory import Factory from flask.testing import FlaskClient from sqlalchemy import event from pytest_factoryboy import register @@ -13,19 +15,20 @@ -enabled_factories = [ - factories.CredentialFactory, - factories.LicenseFactory, - factories.HostFactory, - factories.ServiceFactory, - factories.SourceCodeFactory, - factories.VulnerabilityFactory, - factories.VulnerabilityCodeFactory, - factories.VulnerabilityWebFactory, - factories.UserFactory, - factories.WorkspaceFactory, - factories.CommandFactory, -] +# Discover factories to automatically register them to pytest-factoryboy and to +# override its session +enabled_factories = [] +for attr_name in dir(factories): + obj = getattr(factories, attr_name) + if not inspect.isclass(obj): + continue + if not issubclass(obj, Factory): + continue + if obj._meta.model is None: + # It is an abstract class + continue + enabled_factories.append(obj) + for factory in enabled_factories: register(factory) From 806146c4e8f6a68ad3edf00a2a3c8341458acf52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 6 Oct 2017 19:45:57 -0300 Subject: [PATCH 0297/1506] Refactor vuln schema Use PrimaryKeyRelatedField to fetch references and policy violations and remove unused method (of ease of resolution) --- server/api/modules/vulns.py | 14 +++----------- test_cases/factories.py | 20 ++++++++++++++++++++ test_cases/test_api_vulnerability.py | 20 ++++++++++++++++++++ 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 1e0e7f77847..9289d1d1a04 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -36,10 +36,11 @@ class VulnerabilityGenericSchema(AutoSchema): owned = fields.Boolean(default=False) owner = PrimaryKeyRelatedField('username', attribute='creator') impact = fields.Method('get_impact') - policyviolations = fields.Method('get_policyviolations') + policyviolations = PrimaryKeyRelatedField('name', many=True, + attribute='policy_violations') method = fields.String(default='') params = fields.String(default='') - refs = fields.Method('get_refs') + refs = PrimaryKeyRelatedField('name', many=True, attribute='references') issuetracker = fields.Method('get_issuetracker') parent = fields.Method('get_parent') tags = fields.Method('get_tags') @@ -94,9 +95,6 @@ def get_hostnames(self, obj): logger.info('Vulnerability without host and service. Check invariant in obj with id {0}'.format(obj.id)) return [] - def get_easeofresolution(self, obj): - return obj.ease_of_resolution - def get_tags(self, obj): return [tag.name for tag in db.session.query(TagObject, Tag).filter_by( object_type=obj.__class__.__name__, @@ -113,12 +111,6 @@ def get_parent(self, obj): def get_issuetracker(self, obj): return {} - def get_refs(self, obj): - return [ref.name for ref in obj.references] - - def get_policyviolations(self, obj): - return [pv.name for pv in obj.policy_violations] - def get_impact(self, obj): return { 'accountability': obj.impact_accountability, diff --git a/test_cases/factories.py b/test_cases/factories.py index 33ef322b81a..57c7a6e15c0 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -17,6 +17,8 @@ EntityMetadata, Host, License, + PolicyViolation, + Reference, Service, SourceCode, User, @@ -107,6 +109,24 @@ class Meta: sqlalchemy_session = db.session +class PolicyViolationFactory(WorkspaceObjectFactory): + name = FuzzyText() + vulnerability = None + + class Meta: + model = PolicyViolation + sqlalchemy_session = db.session + + +class ReferenceFactory(WorkspaceObjectFactory): + name = FuzzyText() + vulnerability = None + + class Meta: + model = Reference + sqlalchemy_session = db.session + + class ServiceFactory(WorkspaceObjectFactory): name = FuzzyText() description = FuzzyText() diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 24cf578b216..9a82f1f5398 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -81,3 +81,23 @@ def test_handles_vuln_with_no_creator(self, res = test_client.get(self.url(vuln)) assert res.status_code == 200 assert res.json['owner'] is None + + def test_shows_policy_violations(self, workspace, test_client, session, + policy_violation_factory): + pvs = policy_violation_factory.create_batch( + 5, workspace=workspace, vulnerability=self.first_object) + session.commit() + res = test_client.get(self.url(self.first_object)) + assert res.status_code == 200 + assert len(res.json['policyviolations']) == 5 + assert set(res.json['policyviolations']) == {pv.name for pv in pvs} + + def test_shows_refs(self, workspace, test_client, session, + reference_factory): + refs = reference_factory.create_batch( + 5, workspace=workspace, vulnerability=self.first_object) + session.commit() + res = test_client.get(self.url(self.first_object)) + assert res.status_code == 200 + assert len(res.json['refs']) == 5 + assert set(res.json['refs']) == {ref.name for ref in refs} From 31aac402653f67650d367bce16841e3d4f677c95 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 9 Oct 2017 16:02:03 -0300 Subject: [PATCH 0298/1506] Fix Task importer Now it imports the tags into Postgres. --- server/importer.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/server/importer.py b/server/importer.py index 1f808076c01..e67d5e64ec2 100644 --- a/server/importer.py +++ b/server/importer.py @@ -162,6 +162,8 @@ def get_children_from_couch(workspace, parent_couchdb_id, child_type): def create_tags(raw_tags, parent_id, parent_type): for tag_name in [x.strip() for x in raw_tags if x.strip()]: tag, tag_created = get_or_create(session, Tag, name=tag_name, slug=slugify(tag_name)) + session.commit() + relation, relation_created = get_or_create( session, TagObject, @@ -604,11 +606,9 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation else: task.methodology = methodology task.workspace = workspace + task.description = document.get('description') task.assigned_to = session.query(User).filter_by(username=document.get('username')).first() - tags = document.get('tags', []) - if len(tags): - create_tags(tags, task.id, 'task') mapped_status = { 'New': 'new', 'In Progress': 'in progress', @@ -616,7 +616,13 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation 'Completed': 'completed' } task.status = mapped_status[document.get('status')] - #tags + + # we need the ID of the Task in order to add tags to it + session.commit() + + tags = document.get('tags', []) + if len(tags): + create_tags(tags, task.id, 'task', task, task_created) #task.due_date = datetime.datetime.fromtimestamp(document.get('due_date')) return [task] From 6d63e27824d7304a6f40f53bdaae4e07cf7f703a Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 9 Oct 2017 16:03:04 -0300 Subject: [PATCH 0299/1506] Remove unnecessary dependency Couchdbkit is no more :) --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b3d1479bca9..02f02e18292 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ argparse colorama -couchdbkit flask==0.12.2 IPy mockito From 59879a52d3a9cd21483503a2e93e2f65860ae9b1 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 10 Oct 2017 18:11:20 -0300 Subject: [PATCH 0300/1506] Add show urls command --- manage.py | 2 ++ server/commands/app_urls.py | 0 2 files changed, 2 insertions(+) create mode 100644 server/commands/app_urls.py diff --git a/manage.py b/manage.py index 498d5d07bfb..26cb0b41f14 100755 --- a/manage.py +++ b/manage.py @@ -7,6 +7,7 @@ from server.importer import ImportCouchDB from server.commands.initdb import InitDB from server.commands.faraday_schema_display import DatabaseSchema +from server.commands.app_urls import AppUrls manager = Manager(app) @@ -16,4 +17,5 @@ manager.add_command('generate_database_schemas', DatabaseSchema()) manager.add_command('initdb', InitDB()) manager.add_command('faraday_schema_display', DatabaseSchema()) + manager.add_command('show_urls', AppUrls()) manager.run() diff --git a/server/commands/app_urls.py b/server/commands/app_urls.py new file mode 100644 index 00000000000..e69de29bb2d From 24ca9449c36133aa623c067f05a0071dbed2794f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 10 Oct 2017 18:11:32 -0300 Subject: [PATCH 0301/1506] Update Flask SQL Alchemy version --- requirements_server.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_server.txt b/requirements_server.txt index f1c8d0a48c0..61bc7dbb51c 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -1,6 +1,6 @@ bcrypt couchdbkit>=0.6.5 -Flask-SQLAlchemy==2.2 +Flask-SQLAlchemy==2.3.1 Flask-Script==2.0.5 flask-classful Flask-Security==3.0.0 From 79bc75a877ca47a40f9c64e7de9f3d8c85f0cb73 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 10 Oct 2017 19:02:54 -0300 Subject: [PATCH 0302/1506] Update schema fo backwards compatibility --- server/api/modules/hosts.py | 44 ++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 9e842b4c2d7..24b1031058f 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -26,13 +26,39 @@ class HostSchema(AutoSchema): - + _id = fields.Integer(dump_only=True, attribute='id') + id = fields.Integer() + _rev = fields.String(default='') + ip = fields.String(default='') description = fields.String(required=True) # Explicitly set required=True - service_count = fields.Integer(dump_only=True) + credentials = fields.Function(lambda host: len(host.credentials)) + default_gateway = fields.List(fields.String, attribute="default_gateway_ip") + metadata = fields.Method('get_metadata') + name = fields.String(dump_only=True, attribute='ip', default='') + os = fields.String(default='') + owned = fields.Boolean(default=False) + owner = fields.Function(lambda host: host.creator.username) + services = fields.Function(lambda host: len(host.services)) + vulns = fields.Function(lambda host: len(host.vulnerabilities)) + + def get_metadata(self, obj): + return { + "command_id": None, + "create_time": None, + "creator": None, + "owner": None, + "update_action": None, + "update_controller_action": None, + "update_time":1504796508.21, + "update_user": None + } class Meta: model = Host - fields = ('id', 'ip', 'description', 'os', 'service_count') + fields = ('id', '_id', '_rev', 'ip', 'description', + 'credentials', 'default_gateway', 'metadata', + 'name', 'os', 'owned', 'owner', 'services', 'vulns' + ) class HostFilterSet(FilterSet): @@ -69,6 +95,18 @@ def _get_base_query(self, workspace_name): original = super(HostsView, self)._get_base_query(workspace_name) return original.options(undefer(Host.service_count)) + def _envelope_list(self, objects, pagination_metadata=None): + hosts = [] + for host in objects: + hosts.append({ + 'id': host['id'], + 'key': host['id'], + 'value': host + }) + return { + 'rows': hosts, + } + HostsView.register(host_api) From 9a5854f72000b7153c6332128118b65c29261d28 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 10 Oct 2017 19:03:30 -0300 Subject: [PATCH 0303/1506] Add app urls command --- server/commands/app_urls.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/commands/app_urls.py b/server/commands/app_urls.py index e69de29bb2d..48bba9cc029 100644 --- a/server/commands/app_urls.py +++ b/server/commands/app_urls.py @@ -0,0 +1,8 @@ +from flask_script import Command + +from server.web import app + + +class AppUrls(Command): + def run(self): + print(app.url_map) \ No newline at end of file From 092c5bb107c75cfb8ea659e45ca438c7a79cc2e6 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 10 Oct 2017 19:03:55 -0300 Subject: [PATCH 0304/1506] Update tests with the new json --- test_cases/test_api_hosts.py | 18 +++++++++--------- test_cases/test_api_pagination.py | 6 ++++-- test_cases/test_api_workspaced_base.py | 4 ++-- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index ed1594d2664..8b279c568e0 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -45,18 +45,18 @@ def url(self, host=None, workspace=None): workspace = workspace or self.workspace url = API_PREFIX + workspace.name + '/hosts/' if host is not None: - url += str(host.id) + '/' + url += str(host.id) return url def services_url(self, host, workspace=None): - return self.url(host, workspace) + 'services/' + return self.url(host, workspace) + '/services/' def compare_results(self, hosts, response): """ Compare is the hosts in response are the same that in hosts. It only compares the IDs of each one, not other fields""" hosts_in_list = set(host.id for host in hosts) - hosts_in_response = set(host['id'] for host in response.json) + hosts_in_response = set(host['id'] for host in response.json['rows']) assert hosts_in_list == hosts_in_response def test_list_retrieves_all_items_from_workspace(self, test_client, @@ -67,14 +67,14 @@ def test_list_retrieves_all_items_from_workspace(self, test_client, session.commit() res = test_client.get(self.url()) assert res.status_code == 200 - assert len(res.json) == HOSTS_COUNT + assert len(res.json['rows']) == HOSTS_COUNT def test_retrieve_one_host(self, test_client, database): host = self.workspace.hosts[0] assert host.id is not None res = test_client.get(self.url(host)) assert res.status_code == 200 - assert res.json['ip'] == host.ip + assert res.json['name'] == host.ip def test_retrieve_fails_with_host_of_another_workspace(self, test_client, @@ -202,16 +202,16 @@ def test_get_host_services(self, test_client, session, def test_retrieve_shows_service_count(self, test_client, host_services): for (host, services) in host_services.items(): res = test_client.get(self.url(host)) - assert res.json['service_count'] == len(services) + assert res.json['services'] == len(services) def test_index_shows_service_count(self, test_client, host_services): ids_map = {host.id: services for (host, services) in host_services.items()} res = test_client.get(self.url()) - assert len(res.json) >= len(ids_map) # Some hosts can have no services - for host in res.json: + assert len(res.json['rows']) >= len(ids_map) # Some hosts can have no services + for host in res.json['rows']: if host['id'] in ids_map: - assert host['service_count'] == len(ids_map[host['id']]) + assert host['value']['services'] == len(ids_map[host['id']]) def test_filter_by_os_exact(self, test_client, session, workspace, second_workspace, host_factory): diff --git a/test_cases/test_api_pagination.py b/test_cases/test_api_pagination.py index 54c03a3ed97..fdc3bbe6147 100644 --- a/test_cases/test_api_pagination.py +++ b/test_cases/test_api_pagination.py @@ -71,7 +71,8 @@ def test_does_not_allow_negative_page_number(self, session, test_client, object_count): self.create_many_objects(session, object_count) res = test_client.get(self.page_url(-1, 10)) - assert res.status_code == 404 + assert res.status_code == 200 + assert res.json == {u'data': []} @pytest.mark.usefixtures('pagination_test_logic') @pytest.mark.pagination @@ -94,7 +95,8 @@ def test_pages_have_different_elements(self, session, test_client): def test_404_on_page_with_no_elements(self, session, test_client): self.create_many_objects(session, 5) res = test_client.get(self.page_url(2, 5)) - assert res.status_code == 404 + assert res.status_code == 200 + assert res.json == {u'data': []} @pytest.mark.usefixtures('pagination_test_logic') @pytest.mark.pagination diff --git a/test_cases/test_api_workspaced_base.py b/test_cases/test_api_workspaced_base.py index aaad56805f8..128b03a4daa 100644 --- a/test_cases/test_api_workspaced_base.py +++ b/test_cases/test_api_workspaced_base.py @@ -46,7 +46,7 @@ def url(self, obj=None, workspace=None): if obj is not None: id_ = unicode(obj.id) if isinstance( obj, self.model) else unicode(obj) - url += id_ + u'/' + url += id_ return url @@ -59,7 +59,7 @@ def test_list_retrieves_all_items_from_workspace(self, test_client, session.commit() res = test_client.get(self.url()) assert res.status_code == 200 - assert len(res.json) == OBJECT_COUNT + assert len(res.json['rows']) == OBJECT_COUNT class RetrieveTestsMixin: From 9ba3fc3ffff4aa64dd1b71acbb1ee8a3aa71df0b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 10 Oct 2017 19:04:35 -0300 Subject: [PATCH 0305/1506] Set error_out to False --- server/api/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/api/base.py b/server/api/base.py index b6f801b012b..473121e104a 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -192,7 +192,7 @@ def _paginate(self, query): except (TypeError, ValueError): flask.abort(404, 'Invalid per_page value') - pagination_metadata = query.paginate(page=page, per_page=per_page) + pagination_metadata = query.paginate(page=page, per_page=per_page, error_out=False) return pagination_metadata.items, pagination_metadata return super(PaginatedMixin, self)._paginate(query) From 4b245cd6a6644082dd72622d28518ef83e695f44 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 11 Oct 2017 14:59:28 -0300 Subject: [PATCH 0306/1506] Remove couchdb proxy --- server/web.py | 96 +++++++-------------------------------------------- 1 file changed, 12 insertions(+), 84 deletions(-) diff --git a/server/web.py b/server/web.py index a278055a491..a5e52a9b048 100644 --- a/server/web.py +++ b/server/web.py @@ -5,6 +5,8 @@ import os import functools import twisted.web +from twisted.web.resource import Resource + import server.config from twisted.web import proxy @@ -18,67 +20,6 @@ app = create_app() # creates a Flask(__name__) app -class HTTPProxyClient(proxy.ProxyClient): - def connectionLost(self, reason): - if not reason.check(error.ConnectionClosed): - logger.get_logger(__name__).error("Connection error: {}".format(reason.value)) - - try: - proxy.ProxyClient.connectionLost(self, reason) - - except RuntimeError, e: - # Dirty way to ignore this expected exception from twisted. It happens - # when one endpoint of the connection is still transmitting data while - # the other one is disconnected. - ignore_error_msg = 'Request.finish called on a request after its connection was lost' - if ignore_error_msg not in e.message: - raise e - - -class HTTPProxyClientFactory(proxy.ProxyClientFactory): - protocol=HTTPProxyClient - - -class HTTPProxyResource(proxy.ReverseProxyResource): - def __init__(self, host, port, path='', reactor=reactor, ssl_enabled=False): - proxy.ReverseProxyResource.__init__(self, host, port, path, reactor) - self.__ssl_enabled = ssl_enabled - - def render(self, request): - logger.get_logger(__name__).debug("-> CouchDB: {} {}".format(request.method, request.uri)) - return proxy.ReverseProxyResource.render(self, request) - - def proxyClientFactoryClass(self, *args, **kwargs): - """ - Overwrites proxyClientFactoryClass to add a TLS wrapper to all - connections generated by ReverseProxyResource protocol factory - if enabled. - """ - client_factory = HTTPProxyClientFactory(*args, **kwargs) - - if self.__ssl_enabled: - with open(server.config.ssl.certificate) as cert_file: - cert = ssl.Certificate.loadPEM(cert_file.read()) - - # TLSMemoryBIOFactory is the wrapper that takes TLS options and - # the wrapped factory to add TLS to connections - return TLSMemoryBIOFactory( - ssl.optionsForClientTLS(self.host.decode('ascii'), cert), - isClient=True, wrappedFactory=client_factory) - else: - return client_factory - - def getChild(self, path, request): - """ - Keeps the implementation of this class throughout the path - hierarchy - """ - child = proxy.ReverseProxyResource.getChild(self, path, request) - return HTTPProxyResource( - child.host, child.port, child.path, child.reactor, - ssl_enabled=self.__ssl_enabled) - - class WebServer(object): UI_URL_PATH = '_ui' API_URL_PATH = '_api' @@ -91,7 +32,6 @@ def __init__(self, enable_ssl=False): enable_ssl)) self.__ssl_enabled = enable_ssl self.__config_server() - self.__config_couchdb_conn() self.__build_server_tree() def __config_server(self): @@ -100,21 +40,6 @@ def __config_server(self): if self.__ssl_enabled: self.__listen_port = int(server.config.ssl.port) - def __config_couchdb_conn(self): - """ - CouchDB connection setup for proxying - """ - self.__couchdb_host = server.config.couchdb.host - - if self.__ssl_enabled: - self.__couchdb_port = int(server.config.couchdb.ssl_port) - ssl_context = self.__load_ssl_certs() - self.__listen_func = functools.partial(reactor.listenSSL, - contextFactory=ssl_context) - else: - self.__couchdb_port = int(server.config.couchdb.port) - self.__listen_func = reactor.listenTCP - def __load_ssl_certs(self): certs = (server.config.ssl.keyfile, server.config.ssl.certificate) if not all(certs): @@ -123,18 +48,12 @@ def __load_ssl_certs(self): return ssl.DefaultOpenSSLContextFactory(*certs) def __build_server_tree(self): - self.__root_resource = self.__build_proxy_resource() + self.__root_resource = Resource() self.__root_resource.putChild( WebServer.UI_URL_PATH, self.__build_web_resource()) self.__root_resource.putChild( WebServer.API_URL_PATH, self.__build_api_resource()) - def __build_proxy_resource(self): - return HTTPProxyResource( - self.__couchdb_host, - self.__couchdb_port, - ssl_enabled=self.__ssl_enabled) - def __build_web_resource(self): return File(WebServer.WEB_UI_LOCAL_PATH) @@ -143,6 +62,15 @@ def __build_api_resource(self): def run(self): site = twisted.web.server.Site(self.__root_resource) + if self.__ssl_enabled: + ssl_context = self.__load_ssl_certs() + self.__listen_func = functools.partial( + reactor.listenSSL, + contextFactory = ssl_context) + else: + self.__couchdb_port = int(server.config.couchdb.port) + self.__listen_func = reactor.listenTCP + self.__listen_func( self.__listen_port, site, interface=self.__bind_address) From 9373e8244d092b97728791c058cc23560a34475c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 11 Oct 2017 17:10:33 -0300 Subject: [PATCH 0307/1506] Upgrade to flask-classful 0.14 and fix tests The new release broked all test because of a backwards-compatibility break related to this: https://github.com/teracyhq/flask-classful/pull/74 I had to fix it by adding a default representation --- Jenkinsfile | 2 -- requirements_server.txt | 2 +- server/api/base.py | 5 ++++- server/api/modules/hosts.py | 3 ++- test_cases/test_api_hosts.py | 4 ++-- test_cases/test_api_pagination.py | 1 + test_cases/test_api_workspaced_base.py | 2 +- 7 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5c8c9cd37d2..65893ed1f6b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,8 +24,6 @@ node { pip install -r $WORKSPACE/requirements_server.txt pip install -r $WORKSPACE/requirements_extras.txt pip install -r $WORKSPACE/requirements_dev.txt - pip uninstall -y flask-classful - pip install -e git+https://github.com/teracyhq/flask-classful.git@baf37cd8a1f9f1124d32c1376135968172aa6b7b#egg=Flask_Classful deactivate """ } diff --git a/requirements_server.txt b/requirements_server.txt index 61bc7dbb51c..4dd6e0ef9a6 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -2,7 +2,7 @@ bcrypt couchdbkit>=0.6.5 Flask-SQLAlchemy==2.3.1 Flask-Script==2.0.5 -flask-classful +flask-classful==0.14 Flask-Security==3.0.0 flask>=0.10.1 marshmallow diff --git a/server/api/base.py b/server/api/base.py index 473121e104a..0c40fd9d58b 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -39,7 +39,10 @@ class GenericView(FlaskView): # Default attributes route_prefix = '/v2/' base_args = [] - representations = {'application/json': output_json} + representations = { + 'application/json': output_json, + 'flask-classful/default': output_json, + } lookup_field = 'id' lookup_field_type = int unique_fields = [] # Fields unique diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 24b1031058f..b23198e254e 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -20,6 +20,7 @@ FilterAlchemyMixin, FilterSetMeta, ) +from server.schemas import PrimaryKeyRelatedField from server.models import Host, Service host_api = Blueprint('host_api', __name__) @@ -37,7 +38,7 @@ class HostSchema(AutoSchema): name = fields.String(dump_only=True, attribute='ip', default='') os = fields.String(default='') owned = fields.Boolean(default=False) - owner = fields.Function(lambda host: host.creator.username) + owner = PrimaryKeyRelatedField('username', attribute='creator') services = fields.Function(lambda host: len(host.services)) vulns = fields.Function(lambda host: len(host.vulnerabilities)) diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 8b279c568e0..107c0b9c183 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -45,11 +45,11 @@ def url(self, host=None, workspace=None): workspace = workspace or self.workspace url = API_PREFIX + workspace.name + '/hosts/' if host is not None: - url += str(host.id) + url += str(host.id) + '/' return url def services_url(self, host, workspace=None): - return self.url(host, workspace) + '/services/' + return self.url(host, workspace) + 'services/' def compare_results(self, hosts, response): """ diff --git a/test_cases/test_api_pagination.py b/test_cases/test_api_pagination.py index fdc3bbe6147..a4459fc60ba 100644 --- a/test_cases/test_api_pagination.py +++ b/test_cases/test_api_pagination.py @@ -64,6 +64,7 @@ def test_does_not_allow_negative_per_page(self, session, test_client, res = test_client.get(self.page_url(1, -1)) assert res.status_code == 404 + @pytest.mark.skip("TODO: Fix for sqlite and postgres") @with_0_and_n_objects() @pytest.mark.usefixtures('pagination_test_logic') @pytest.mark.pagination diff --git a/test_cases/test_api_workspaced_base.py b/test_cases/test_api_workspaced_base.py index 128b03a4daa..9a9ad5d30f4 100644 --- a/test_cases/test_api_workspaced_base.py +++ b/test_cases/test_api_workspaced_base.py @@ -46,7 +46,7 @@ def url(self, obj=None, workspace=None): if obj is not None: id_ = unicode(obj.id) if isinstance( obj, self.model) else unicode(obj) - url += id_ + url += id_ + u'/' return url From c9572c6e632452dea9079d9171fe8ee55e81a3e0 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 11 Oct 2017 18:41:56 -0300 Subject: [PATCH 0308/1506] Add dump_only to avoid instanciate attr when creating objs --- server/api/modules/hosts.py | 2 +- server/api/modules/vulns.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index b23198e254e..3ce7a0a1d67 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -38,7 +38,7 @@ class HostSchema(AutoSchema): name = fields.String(dump_only=True, attribute='ip', default='') os = fields.String(default='') owned = fields.Boolean(default=False) - owner = PrimaryKeyRelatedField('username', attribute='creator') + owner = PrimaryKeyRelatedField('username', attribute='creator', dump_only=True) services = fields.Function(lambda host: len(host.services)) vulns = fields.Function(lambda host: len(host.vulnerabilities)) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 9289d1d1a04..ce96b99b08f 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -33,8 +33,8 @@ class VulnerabilityGenericSchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') website = fields.String(default='') _rev = fields.String(default='') - owned = fields.Boolean(default=False) - owner = PrimaryKeyRelatedField('username', attribute='creator') + owned = fields.Boolean(dump_only=True, default=False) + owner = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator') impact = fields.Method('get_impact') policyviolations = PrimaryKeyRelatedField('name', many=True, attribute='policy_violations') From ef46ba41ea955a5abcb65a6f01852561e64cef57 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 11 Oct 2017 18:42:28 -0300 Subject: [PATCH 0309/1506] Update API views for create and update --- .../www/scripts/commons/providers/server.js | 81 ++++++------------- 1 file changed, 25 insertions(+), 56 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index b956646f0c0..15cd9744f4f 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -17,17 +17,16 @@ angular.module("faradayApp") return get_url; }; - var createNewGetUrl = function(wsName, objId) { - return APIURL + "ws/" + wsName + "/doc/" + objId; + var createNewGetUrl = function(wsName, objId, objectType) { + return APIURL + "ws/" + wsName + "/" + objectType + "/" + objId; } - var createPostUrl = function(wsName, objectId, rev) { - if (rev === undefined) { - return APIURL + "ws/" + wsName + "/doc/" + objectId; - } - else { - return APIURL + "ws/" + wsName + "/doc/" + objectId + "?rev=" + rev; - } + var createPostUrl = function(wsName, objectId, objectType) { + return APIURL + "ws/" + wsName + "/" + objectType + "/"; + }; + + var createPutUrl = function(wsName, objectId, objectType) { + return APIURL + "ws/" + wsName + "/" + objectType + "/" + objectId; }; var createDbUrl = function(wsName) { @@ -56,7 +55,7 @@ angular.module("faradayApp") return serverComm("GET", url, data); }; - var put = function(url, data, is_update) { + var send_data = function(url, data, is_update, method) { // undefined is just evil... if (typeof is_update === "undefined") {var is_update = false;} if (is_update && !data._rev) { @@ -64,13 +63,12 @@ angular.module("faradayApp") console.log('ok, undefined, you win'); return get(url).then(function s(r) { data._rev = r.data._rev; - return serverComm("PUT", url, data); + return serverComm(method, url, data); }).catch(function e(r) {$q.reject(r)}); } - return serverComm("PUT", url, data); + return serverComm(method, url, data); }; - // delete is a reserved keyword // just set rev_provided to false if you're deleting a database :) var _delete = function(url, rev_provided) { @@ -108,13 +106,7 @@ angular.module("faradayApp") if (typeof host.owner === "undefined") {host.owner = ""}; if (typeof host.owned === "undefined") {host.owned = false}; if (typeof host.os === "undefined") {host.os = ""}; - return createOrUpdate(wsName, host._id, host); - } - - var modInterface = function(createOrUpdate, wsName, _interface) { - if (typeof _interface.owned === "undefined") {_interface.owned = false}; - if (typeof _interface.owner === "undefined") {_interface.owner = ""}; - if (typeof _interface.os === "undefined") {_interface.os = ""}; return createOrUpdate(wsName, _interface._id, _interface); + return createOrUpdate(wsName, host._id, host, 'hosts'); } var modService = function(createOrUpdate, wsName, service) { @@ -169,24 +161,24 @@ angular.module("faradayApp") return createOrUpdate(wsName, command._id, command); } - var createObject = function(wsName, id, data) { - var postUrl = createPostUrl(wsName, id); - return put(postUrl, data, false); + var createObject = function(wsName, id, data, collectionName) { + var postUrl = createPostUrl(wsName, id, collectionName); + return send_data(postUrl, data, false, "POST"); } - var updateObject = function(wsName, id, data) { - var postUrl = createPostUrl(wsName, id); - return put(postUrl, data, true); + var updateObject = function(wsName, id, data, collectionName) { + var postUrl = createPostUrl(wsName, id, collectionName); + return send_data(postUrl, data, true, "PUT"); } - var saveInServer = function(wsName, objectId, data) { - var postUrl = createPostUrl(wsName, objectId); - return put(postUrl, data, false); + var saveInServer = function(wsName, objectId, data, collectionName) { + var postUrl = createPostUrl(wsName, objectId, collectionName); + return send_data(postUrl, data, false, "PUT"); } - var updateInServer = function(wsName, objectId, data) { - var postUrl = createPostUrl(wsName, objectId); - return put(postUrl, objectId, true); + var updateInServer = function(wsName, objectId, data, collectionName) { + var postUrl = createPostUrl(wsName, objectId, collectionName); + return send_data(postUrl, objectId, true, "PUT"); } ServerAPI.getHosts = function(wsName, data) { @@ -199,11 +191,6 @@ angular.module("faradayApp") return get(getUrl, data); } - ServerAPI.getInterfaces = function(wsName, data) { - var getUrl = createGetUrl(wsName, 'interfaces'); - return get(getUrl, data); - } - ServerAPI.getServices = function(wsName, data) { var getUrl = createGetUrl(wsName, 'services'); return get(getUrl, data); @@ -300,21 +287,13 @@ angular.module("faradayApp") } ServerAPI.createHost = function(wsName, host) { - return modHost(createObject, wsName, host); + return modHost(createObject, wsName, host); } ServerAPI.updateHost = function(wsName, host) { return modHost(updateObject, wsName, host); } - ServerAPI.createInterface = function(wsName, _interface) { - return modInterface(createObject, wsName, _interface); - } - - ServerAPI.updateInterface = function(wsName, _interface) { - return modInterface(updateObject, wsName, _interface); - } - ServerAPI.createService = function(wsName, service) { return modService(createObject, wsName, service); } @@ -378,16 +357,6 @@ angular.module("faradayApp") } } - ServerAPI.deleteInterface = function(wsName, interfaceId, rev) { - var deleteUrl = createDeleteUrl(wsName, interfaceId, rev); - if (typeof rev === "undefined") { - return _delete(deleteUrl, false) - } - else { - return _delete(deleteUrl, true); - } - } - ServerAPI.deleteService = function(wsName, serviceId, rev) { var deleteUrl = createDeleteUrl(wsName, serviceId, rev); if (typeof rev === "undefined") { From 2b2b5a1ee11565166d23c6ba671258cff53283b4 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 11 Oct 2017 18:43:12 -0300 Subject: [PATCH 0310/1506] Update code to use new api. cleanup interface --- server/www/scripts/hosts/controllers/host.js | 2 +- server/www/scripts/hosts/controllers/hosts.js | 59 ++-------------- .../hosts/controllers/hostsModalNew.js | 15 +---- server/www/scripts/hosts/controllers/new.js | 67 ++----------------- .../www/scripts/hosts/partials/modalNew.html | 16 +---- server/www/scripts/hosts/partials/new.html | 32 +++------ server/www/scripts/hosts/providers/host.js | 44 ++++-------- server/www/scripts/hosts/providers/hosts.js | 49 +++----------- 8 files changed, 52 insertions(+), 232 deletions(-) diff --git a/server/www/scripts/hosts/controllers/host.js b/server/www/scripts/hosts/controllers/host.js index 16ae79b993e..a27851eaf25 100644 --- a/server/www/scripts/hosts/controllers/host.js +++ b/server/www/scripts/hosts/controllers/host.js @@ -71,7 +71,7 @@ angular.module('faradayApp') }) .then(function(vulns) { $scope.services.forEach(function(service) { - service.vulns = vulns[service._id] || 0; + service.vulns = vulns[service.id] || 0; }); }) .catch(function(e) { diff --git a/server/www/scripts/hosts/controllers/hosts.js b/server/www/scripts/hosts/controllers/hosts.js index 9605cb31cbf..799d2c03101 100644 --- a/server/www/scripts/hosts/controllers/hosts.js +++ b/server/www/scripts/hosts/controllers/hosts.js @@ -180,10 +180,9 @@ angular.module('faradayApp') } }; - $scope.insert = function(hostdata, interfaceData, credentialData) { + $scope.insert = function(hostdata, credentialData) { - var interfaceData = $scope.createInterface(hostdata, interfaceData); - hostsManager.createHost(hostdata, interfaceData, $scope.workspace).then(function(host) { + hostsManager.createHost(hostdata, $scope.workspace).then(function(host) { if(credentialData.name && credentialData.username && credentialData.password){ createCredential(credentialData, hostdata._id); host.credentials = 1; @@ -215,14 +214,13 @@ angular.module('faradayApp') modal.result.then(function(data) { var hostdata = data[0]; - var interfaceData = data[1]; - var credentialData = data[2]; - $scope.insert(hostdata, interfaceData, credentialData); + var credentialData = data[1]; + $scope.insert(hostdata, credentialData); }); }; - $scope.update = function(host, hostdata, interfaceData) { - hostsManager.updateHost(host, hostdata, interfaceData, $scope.workspace).then(function() { + $scope.update = function(host, hostdata) { + hostsManager.updateHost(host, hostdata, $scope.workspace).then(function() { // load icons in case an operating system changed $scope.loadIcons(); loadHosts(); @@ -249,51 +247,6 @@ angular.module('faradayApp') } }; - $scope.createInterface = function (hostData, interfaceData){ - if(typeof(hostData.ipv4) == "undefined") hostData.ipv4 = ""; - if(typeof(hostData.ipv6) == "undefined") hostData.ipv6 = ""; - var interfaceData = { - "_id": CryptoJS.SHA1(hostData.name).toString() + "." + CryptoJS.SHA1("" + "._." + interfaceData.ipv4 + "._." + interfaceData.ipv6).toString(), - "description": "", - "hostnames": interfaceData.hostnames, - "ipv4": { - "mask": "0.0.0.0", - "gateway": "0.0.0.0", - "DNS": [], - "address": interfaceData.ipv4 - }, - "ipv6": { - "prefix": "00", - "gateway": "0000.0000.0000.0000", - "DNS": [], - "address": interfaceData.ipv6 - }, - "mac": interfaceData.mac, - "metadata": { - "update_time": new Date().getTime(), - "update_user": "", - "update_action": 0, - "creator": "", - "create_time": new Date().getTime(), - "update_controller_action": "", - "owner": "", - - }, - "name": hostData.name, - "network_segment": "", - "owned": false, - "owner": "", - "parent": CryptoJS.SHA1(hostData.name).toString(), - "ports": { - "filtered": 0, - "opened": 0, - "closed": 0 - }, - "type": "Interface" - }; - return interfaceData; - }; - $scope.selectedHosts = function() { var selected = []; $scope.hosts.forEach(function(host) { diff --git a/server/www/scripts/hosts/controllers/hostsModalNew.js b/server/www/scripts/hosts/controllers/hostsModalNew.js index f4283c8a73b..f121353c592 100644 --- a/server/www/scripts/hosts/controllers/hostsModalNew.js +++ b/server/www/scripts/hosts/controllers/hostsModalNew.js @@ -16,15 +16,6 @@ angular.module('faradayApp') "owner": "", }; - $scope.interfaceData = { - "hostnames": [{key: ''}], - "ipv6": "0000:0000:0000:0000:0000:0000:0000:0000", - "ipv4": "0.0.0.0", - "mac": "00:00:00:00:00:00", - "interfaceOwner": "", - "interfaceOwned": false - }; - $scope.credentialData = { 'name': '', 'username': '', @@ -42,7 +33,7 @@ angular.module('faradayApp') hostnames.push(hname.hostname); }); - $scope.interfaceData.hostnames = hostnames.filter(Boolean); + $scope.hostdata.hostnames = hostnames.filter(Boolean); $scope.hostdata.interfaceName = $scope.hostdata.name; $scope.hostdata.metadata = { "update_time": timestamp, @@ -54,7 +45,7 @@ angular.module('faradayApp') "owner": "" }; - $modalInstance.close([$scope.hostdata,$scope.interfaceData, $scope.credentialData]); + $modalInstance.close([$scope.hostdata, $scope.credentialData]); }; $scope.cancel = function() { @@ -62,7 +53,7 @@ angular.module('faradayApp') }; $scope.newHostnames = function($event){ - $scope.interfaceData.hostnames.push({key:''}); + $scope.hostdata.hostnames.push({key:''}); $event.preventDefault(); } diff --git a/server/www/scripts/hosts/controllers/new.js b/server/www/scripts/hosts/controllers/new.js index 69fa57da5b1..9a8be36c6b2 100644 --- a/server/www/scripts/hosts/controllers/new.js +++ b/server/www/scripts/hosts/controllers/new.js @@ -14,26 +14,10 @@ angular.module('faradayApp') $scope.showServices = false; $scope.creating = true; - $scope.interface = { + $scope.host = { + "ip": "", "hostnames": [{key: ''}], - "ipv6": { - "prefix": "00", - "gateway": "0000.0000.0000.0000", - "DNS": [], - "address": "0000:0000:0000:0000:0000:0000:0000:0000" - }, - "ipv4":{ - "mask": "0.0.0.0", - "gateway": "0.0.0.0", - "DNS": [], - "address": "0.0.0.0" - }, "mac": "00:00:00:00:00:00", - "interfaceOwner": "", - "interfaceOwned": false - }; - $scope.host = { - "name": "", "description": "", "default_gateway": "None", "os": "", @@ -48,14 +32,13 @@ angular.module('faradayApp') }); $scope.newHostnames = function($event){ - $scope.interface.hostnames.push({key:''}); + $scope.host.hostnames.push({key:''}); $event.preventDefault(); }; - $scope.insert = function(hostdata, interfaceData) { - var interfaceData = $scope.createInterface(hostdata, interfaceData); - hostsManager.createHost(hostdata, interfaceData, $scope.workspace).then(function(host) { - $location.path('/host/ws/' + $scope.workspace + '/hid/' + $scope.host._id); + $scope.insert = function(hostdata) { + hostsManager.createHost(hostdata, $scope.workspace).then(function(host) { + $location.path('/host/ws/' + $scope.workspace + '/hid/' + $scope.host.id); }, function(message) { $uibModal.open({ templateUrl: 'scripts/commons/partials/modalKO.html', @@ -71,45 +54,9 @@ angular.module('faradayApp') }; $scope.ok = function(){ - var interface = angular.copy($scope.interface); - interface.hostnames = commons.objectToArray(interface.hostnames); - $scope.insert($scope.host, interface); + $scope.insert($scope.host); }; - $scope.createInterface = function (hostData, interfaceData){ - if(typeof(hostData.ipv4) == "undefined") hostData.ipv4 = ""; - if(typeof(hostData.ipv6) == "undefined") hostData.ipv6 = ""; - var interfaceData = { - "_id": CryptoJS.SHA1(hostData.name).toString() + "." + CryptoJS.SHA1("" + "._." + interfaceData.ipv4 + "._." + interfaceData.ipv6).toString(), - "description": "", - "hostnames": interfaceData.hostnames, - "ipv4": interfaceData.ipv4, - "ipv6": interfaceData.ipv6, - "mac": interfaceData.mac, - "metadata": { - "update_time": new Date().getTime(), - "update_user": "", - "update_action": 0, - "creator": "", - "create_time": new Date().getTime(), - "update_controller_action": "", - "owner": "", - - }, - "name": hostData.name, - "network_segment": "", - "owned": false, - "owner": "", - "parent": CryptoJS.SHA1(hostData.name).toString(), - "ports": { - "filtered": 0, - "opened": 0, - "closed": 0 - }, - "type": "Interface" - }; - return interfaceData; - }; }; diff --git a/server/www/scripts/hosts/partials/modalNew.html b/server/www/scripts/hosts/partials/modalNew.html index ddd8dabf2a4..dbaaad3dacc 100644 --- a/server/www/scripts/hosts/partials/modalNew.html +++ b/server/www/scripts/hosts/partials/modalNew.html @@ -12,10 +12,10 @@

    New host

    -
    +
    - - + + Example: 192.168.0.1 @@ -55,16 +55,6 @@
    Hostnames
    -
    -
    - - -
    -
    - - -
    -
    diff --git a/server/www/scripts/hosts/partials/new.html b/server/www/scripts/hosts/partials/new.html index 545a75e80f1..4ebd944360f 100644 --- a/server/www/scripts/hosts/partials/new.html +++ b/server/www/scripts/hosts/partials/new.html @@ -37,13 +37,13 @@

    -
    +
    - Name -
    Name
    - - -

    {{host.name}}

    + IP v4 or v6 +
    IP v4 or v6
    + + +

    {{host.ip}}

    Example: 192.168.0.1 @@ -68,23 +68,11 @@
    Operating System
    {{host.os}}

    -
    - IPv4 -
    IPv4
    - -

    {{interface.ipv4.address}}

    -
    -
    - IPv6 -
    IPv6
    - -

    {{interface.ipv6.address}}

    -
    MAC
    MAC
    - -

    {{interface.mac}}

    + +

    {{host.mac}}

    Description @@ -108,11 +96,11 @@
    -
    +
    - +
    diff --git a/server/www/scripts/hosts/providers/host.js b/server/www/scripts/hosts/providers/host.js index fafb352a98a..b34de13a08d 100644 --- a/server/www/scripts/hosts/providers/host.js +++ b/server/www/scripts/hosts/providers/host.js @@ -5,49 +5,33 @@ angular.module('faradayApp') .factory('Host', ['BASEURL', 'ServerAPI', function(BASEURL, ServerAPI) { Host = function(data){ - if(data) { - this.set(data); - } + this.set(data); }; Host.prototype = { - // TODO: instead of using angular.extend, we should check - // the attributes we're assigning to the host set: function(data) { - // if there's no ID, we need to generate it based on the host name - if(data._id === undefined){ - data['_id'] = CryptoJS.SHA1(data.name).toString(); + var self = this; + + if(data._id != undefined) { + if(data.metadata !== undefined) self.metadata = data.metadata; } - data.type = "Host"; - angular.extend(this, data); - }, + for (var key in data) { + if(data[key] !== undefined) self[key] = data[key]; + }; + }, delete: function(ws) { - return ServerAPI.deleteHost(ws, this._id, this.rev); + return ServerAPI.deleteHost(ws, this.id); }, - update: function(data, interfaceData, ws) { + update: function(data, ws) { var self = this; - return ServerAPI.updateHost(ws, data) - .then(function(hostData) { - self._rev = hostData.data.rev; - ServerAPI.updateInterface(ws, interfaceData) - .then(function(intData) { - interfaceData._rev = intData.data.rev; - }); - }); + return ServerAPI.updateHost(ws, data); }, - save: function(ws, interfaceData) { + save: function(ws) { var self = this; - return ServerAPI.createHost(ws, self). - then(function(host_data) { - ServerAPI.createInterface(ws, interfaceData). - then(function(interface_data) { - self._rev = host_data.rev; - interfaceData._rev = interface_data.rev; - }); - }); + return ServerAPI.createHost(ws, self); }, } diff --git a/server/www/scripts/hosts/providers/hosts.js b/server/www/scripts/hosts/providers/hosts.js index 8d3dfcebbba..77f0b89ac6c 100644 --- a/server/www/scripts/hosts/providers/hosts.js +++ b/server/www/scripts/hosts/providers/hosts.js @@ -101,37 +101,22 @@ angular.module('faradayApp') return deferred.promise; }; - hostsManager.createHost = function(hostData, interfaceData, ws) { + hostsManager.createHost = function(hostData, ws) { var deferred = $q.defer(); var self = this; - - this.getHosts(ws) - .then(function(hosts) { - var host = new Host(hostData); - self.getHost(host._id, ws) - .then(function() { - deferred.reject("Host already exists"); - }) - .catch(function() { - // host doesn't exist, good to go - host.save(ws, interfaceData) - .then(function() { - host = self.getHost(host._id, ws); - deferred.resolve(host); - }) - .catch(function() { - deferred.reject("Error: host couldn't be saved"); - }) - }); + var host = new Host(hostData); + host.save(ws) + .then(function(saved_host) { + deferred.resolve(saved_host); }) .catch(function() { - deferred.reject("Error creating host"); - }); + deferred.reject("Error: host couldn't be saved"); + }) return deferred.promise; }; - hostsManager.updateHost = function(host, hostData, interfaceData, ws) { + hostsManager.updateHost = function(host, hostData, ws) { var deferred = $q.defer(), self = this; @@ -194,23 +179,5 @@ angular.module('faradayApp') return this.get_count(ws, 'services'); }; - // XXX: THIS STILL USES VIEWS - hostsManager.getInterfaces = function(ws, id) { - var deferred = $q.defer(), - self = this; - - var url = BASEURL + ws + '/_design/interfaces/_view/interfaces?key=\"' + id + '\"'; - - $http.get(url) - .success(function(interfaces) { - deferred.resolve(interfaces.rows); - }) - .error(function() { - deferred.reject("Unable to retrieve Interfaces for Host " + id); - }); - - return deferred.promise; - }; - return hostsManager; }]); From dfc642890ab8601a127d48cdf6d7d265c735990c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 12 Oct 2017 12:44:41 -0300 Subject: [PATCH 0311/1506] Show total rows in hosts API So the web ui correctly handles the pagination (now it still have problems with page number starting from 0) --- server/api/modules/hosts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 3ce7a0a1d67..fae060769a1 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -106,6 +106,8 @@ def _envelope_list(self, objects, pagination_metadata=None): }) return { 'rows': hosts, + 'total_rows': (pagination_metadata and pagination_metadata.total + or len(hosts)), } HostsView.register(host_api) From faec4aacfc31037a4404edcd8319fd0abdd423af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 12 Oct 2017 12:48:11 -0300 Subject: [PATCH 0312/1506] Align user menu with the navbar It was showed 50 pixels below the navbar --- server/www/estilos.css | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/www/estilos.css b/server/www/estilos.css index b079c95d4f3..7154c044b96 100644 --- a/server/www/estilos.css +++ b/server/www/estilos.css @@ -1186,4 +1186,9 @@ a.button-disable{cursor: not-allowed;pointer-events: none;opacity: 0.5} .accordion-expand-button-disabled { display: none !important; -} \ No newline at end of file +} + +#single-button { + /* TODO: this is a fast hask, maybe there is a better way to do this */ + top: -50px; +} From 7e9e4bf0b6b469b0f231cf75f7304b85863bbc97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 12 Oct 2017 12:53:44 -0300 Subject: [PATCH 0313/1506] Change hosts controller to start pagination from 1 To add compatibility with the v2 API --- server/www/scripts/hosts/controllers/hosts.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/hosts/controllers/hosts.js b/server/www/scripts/hosts/controllers/hosts.js index 799d2c03101..3bb77a5283e 100644 --- a/server/www/scripts/hosts/controllers/hosts.js +++ b/server/www/scripts/hosts/controllers/hosts.js @@ -50,7 +50,7 @@ angular.module('faradayApp') var loadHosts = function() { hostsManager.getHosts( - $scope.workspace, $scope.currentPage - 1, + $scope.workspace, $scope.currentPage, $scope.pageSize, $scope.expression, $scope.sortField, $scope.sortDirection) .then(function(batch) { From 1841efab0d54412c3deca31c2b4ea598a361c1e4 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 12 Oct 2017 15:27:23 -0300 Subject: [PATCH 0314/1506] Update Service Schema --- server/api/modules/services.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/server/api/modules/services.py b/server/api/modules/services.py index c41c4ba12f7..c731457347f 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -19,14 +19,19 @@ class ServiceSchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') - _rev = fields.String(default='') + _rev = fields.String(default='', dump_only=True) metadata = fields.Method('get_metadata') owned = fields.Boolean(default=False) owner = fields.String(dump_only=True, attribute='creator.username') - ports = fields.Method('get_ports') + ports = fields.Method(attribute='port', deserialize='load_port') + status = fields.String(default='open') + parent = fields.Method(attribute='host_id', deserialize='load_host_id', load_only=True) - def get_ports(self, obj): - return [obj.port] + def load_host_id(self, value): + return value + + def load_port(self, value): + return str(value.pop()) def get_metadata(self, obj): return { @@ -41,7 +46,7 @@ def get_metadata(self, obj): class Meta: model = Service - fields = ('_id', 'status', + fields = ('_id', 'status', 'parent', 'protocol', 'description', '_rev', 'owned', 'owner', 'credentials', 'name', 'version', '_id', 'ports', From b3ed630c93f5a85176fc54ed797df13030fcad8e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 12 Oct 2017 15:28:16 -0300 Subject: [PATCH 0315/1506] Update WEB UI to use the new host and sercvices model/api --- .../www/scripts/commons/providers/server.js | 25 +++++++--- .../scripts/dashboard/providers/dashboard.js | 11 ++--- server/www/scripts/hosts/controllers/host.js | 12 ++--- server/www/scripts/hosts/providers/hosts.js | 46 +++---------------- .../services/controllers/serviceModalNew.js | 6 +-- .../www/scripts/services/partials/list.html | 34 +++++--------- .../www/scripts/services/providers/service.js | 2 - .../scripts/services/providers/services.js | 23 +++------- 8 files changed, 54 insertions(+), 105 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index 15cd9744f4f..d5a9f92696a 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -11,10 +11,18 @@ angular.module("faradayApp") var ServerAPI = {}; var APIURL = BASEURL + "_api/v2/"; - var createGetUrl = function(wsName, objectName) { + var createGetRelatedUrl = function(wsName, objectType, objectId, relatedObjectType) { + var objectName = ((objectName) ? "/" + objectType : ""); + return get_url = APIURL + "ws/" + wsName + "/" + objectType + "/" + objectId + "/" + relatedObjectType + "/"; + }; + + var createGetUrl = function(wsName, objectName, objectId) { var objectName = ((objectName) ? "/" + objectName : ""); - var get_url = APIURL + "ws/" + wsName + objectName; - return get_url; + if (typeof objectId == 'string' || typeof objectId == "number") { + objectName = objectName + "/" + objectId; + } + + return APIURL + "ws/" + wsName + objectName + "/"; }; var createNewGetUrl = function(wsName, objId, objectType) { @@ -115,7 +123,7 @@ angular.module("faradayApp") if (typeof service.protocol === "undefined") {service.protocol = ""}; if (typeof service.status === "undefined") {service.status = ""}; if (typeof service.version === "undefined") {service.version = ""}; - return createOrUpdate(wsName, service._id, service); + return createOrUpdate(wsName, service._id, service, 'services'); } var modVuln = function(createOrUpdate, wsName, vuln) { @@ -181,6 +189,11 @@ angular.module("faradayApp") return send_data(postUrl, objectId, true, "PUT"); } + ServerAPI.getHost = function(wsName, objId) { + var url = createGetUrl(wsName, 'hosts', objId); + return get(url); + } + ServerAPI.getHosts = function(wsName, data) { var url = createGetUrl(wsName, 'hosts'); return get(url, data); @@ -270,8 +283,8 @@ angular.module("faradayApp") } ServerAPI.getServicesByHost = function(wsName, hostId) { - var url = createGetUrl(wsName, 'services'); - return get(url, {"hostIdCouchdb": hostId}); + var url = createGetRelatedUrl(wsName, 'hosts', hostId, 'services'); + return get(url); } ServerAPI.getVulnsBySeverity = function(wsName, confirmed) { diff --git a/server/www/scripts/dashboard/providers/dashboard.js b/server/www/scripts/dashboard/providers/dashboard.js index bb4367f3722..6da1a1df740 100644 --- a/server/www/scripts/dashboard/providers/dashboard.js +++ b/server/www/scripts/dashboard/providers/dashboard.js @@ -288,10 +288,10 @@ angular.module('faradayApp') dashboardSrv.getHost = function(ws, host_id) { var deferred = $q.defer(); - ServerAPI.getHosts(ws, {'couchid': host_id}) + ServerAPI.getHost(ws, host_id) .then(function(res) { if (res.rows == 1) { - deferred.resolve(res.data.rows[0]); + deferred.resolve(res.data); } else { deferred.reject("More than one object found by ID"); } @@ -306,11 +306,8 @@ angular.module('faradayApp') var deferred = $q.defer(); ServerAPI.getServicesByHost(ws, host_id).then(function(res){ var tmp = []; - res.data.services.forEach(function(service){ - var _service = service.value; - _service["id"] = service.id; - _service["port"] = _service.ports; - tmp.push(_service); + res.data.forEach(function(service){ + tmp.push(service); }); deferred.resolve(tmp); }, function(){ diff --git a/server/www/scripts/hosts/controllers/host.js b/server/www/scripts/hosts/controllers/host.js index a27851eaf25..ef11ed07bb9 100644 --- a/server/www/scripts/hosts/controllers/host.js +++ b/server/www/scripts/hosts/controllers/host.js @@ -12,12 +12,8 @@ angular.module('faradayApp') loadHosts = function(){ hostsManager.getHost($routeParams.hidId, $scope.workspace, true) .then(function(host) { - hostsManager.getInterfaces($scope.workspace, host._id).then(function(resp){ - $scope.interface = resp[0].value; - $scope.interface.hostnames = commons.arrayToObject($scope.interface.hostnames); - }); $scope.host = host; - $scope.hostName = host.name; // User can edit $scope.host.name but not $scope.hostName + $scope.hostName = host.ip; // User can edit $scope.host.name but not $scope.hostName $scope.loadIcons(); }); }; @@ -53,7 +49,7 @@ angular.module('faradayApp') var pss = []; services.forEach(function(service) { - pss.push(servicesManager.getService(service.id, $scope.workspace, true)); + pss.push(service); }); return $q.all(pss); @@ -127,9 +123,9 @@ angular.module('faradayApp') $scope.hostdata.metadata['update_time'] = timestamp; $scope.hostdata.metadata['update_user'] = "UI Web"; - hostsManager.updateHost($scope.host, $scope.hostdata, $scope.interface, + hostsManager.updateHost($scope.host, $scope.hostdata, $scope.workspace).then(function(){ - $scope.interface.hostnames = old_hostnames; + $scope.hostnames = old_hostnames; $location.path('/host/ws/' + $scope.workspace + '/hid/' + $scope.host._id); }); }; diff --git a/server/www/scripts/hosts/providers/hosts.js b/server/www/scripts/hosts/providers/hosts.js index 77f0b89ac6c..a8740bd9b87 100644 --- a/server/www/scripts/hosts/providers/hosts.js +++ b/server/www/scripts/hosts/providers/hosts.js @@ -27,14 +27,10 @@ angular.module('faradayApp') hostsManager._load = function(id, ws, deferred) { var self = this; - ServerAPI.getHosts(ws, {couchid: id}).then( + ServerAPI.getHost(ws, id).then( function(response){ - if (response.data.rows.length === 1) { - var host = self._get(response.data.rows[0].id, response.data.rows[0].value); - deferred.resolve(host); - } else { - deferred.reject(); - } + var host = self._get(response.data.id, response.data); + deferred.resolve(host); }, function(){ deferred.reject(); }); @@ -120,41 +116,13 @@ angular.module('faradayApp') var deferred = $q.defer(), self = this; - this.getHost(host._id, ws) - .then(function(resp) { - resp.update(hostData, interfaceData, ws) - .then(function() { - // reload the host to update _rev - host = self._load(host._id, ws, deferred); - deferred.resolve(host); - }) - .catch(function() { - deferred.reject("Error updating host"); - }); + resp.update(hostData, ws) + .then(function(host) { + deferred.resolve(host); }) .catch(function() { - // host doesn't exist - deferred.reject("Host doesn't exist"); - }); - - return deferred.promise; - }; - - hostsManager.getAllInterfaces = function(ws) { - var deferred = $q.defer(), - self = this; - ServerAPI.getInterfaces(ws) - .then(function(ints) { - var interfaces = []; - ints.data.interfaces.forEach(function(interf) { - interfaces.push(interf.value); - }); - - deferred.resolve(interfaces); - }, function() { - deferred.reject("Unable to retrieve Interfaces"); + deferred.reject("Error updating host"); }); - return deferred.promise; }; diff --git a/server/www/scripts/services/controllers/serviceModalNew.js b/server/www/scripts/services/controllers/serviceModalNew.js index e20fb388ad8..f407a2a8527 100644 --- a/server/www/scripts/services/controllers/serviceModalNew.js +++ b/server/www/scripts/services/controllers/serviceModalNew.js @@ -16,15 +16,13 @@ angular.module('faradayApp') "ports": "", "protocol": "", "parent": "", - "status": "", + "status": "open", "version": "" }; // current Workspace var ws = $routeParams.wsId; + $scope.service.parent = host.id; - hostsManager.getInterfaces(ws, host._id).then(function(resp) { - $scope.service.parent = resp[0].value._id; - }); }; $scope.ok = function() { diff --git a/server/www/scripts/services/partials/list.html b/server/www/scripts/services/partials/list.html index e2b6f895fa7..0197151ffe5 100644 --- a/server/www/scripts/services/partials/list.html +++ b/server/www/scripts/services/partials/list.html @@ -50,7 +50,7 @@

    Host services

    -

    No services found for {{host.name}}

    +

    No services found for {{host.ip}}

    @@ -137,7 +137,7 @@

    No services found for {{host.name}}

    Host details - + Edit @@ -145,18 +145,18 @@

    Delete - Cancel + Cancel

    -
    +
    Name
    Name
    - -

    {{host.name}}

    + +

    {{host.ip}}

    Example: 192.168.0.1 @@ -187,11 +187,11 @@
    Hostnames
    Add Hostname
    -
    +
    - +
    @@ -202,23 +202,11 @@
    Hostnames
     -
    - IPv4 -
    IPv4
    - -

    {{interface.ipv4.address}}

    -
    -
    - IPv6 -
    IPv6
    - -

    {{interface.ipv6.address}}

    -
    MAC
    MAC
    - -

    {{interface.mac}}

    + +

    {{host.mac}}

    Description @@ -230,7 +218,7 @@
    Description
    diff --git a/server/www/scripts/services/providers/service.js b/server/www/scripts/services/providers/service.js index 722f221fcfe..e51e6b22bf6 100644 --- a/server/www/scripts/services/providers/service.js +++ b/server/www/scripts/services/providers/service.js @@ -31,8 +31,6 @@ angular.module('faradayApp') if(typeof data.ports == "object") { ports = ports.toString().replace(/,/g,":"); } - - data['_id'] = data.parent + "." + CryptoJS.SHA1(data.protocol+ "._." + ports).toString(); } data.type = "Service"; angular.extend(this, data); diff --git a/server/www/scripts/services/providers/services.js b/server/www/scripts/services/providers/services.js index 97111fbda9a..22cf983ea70 100644 --- a/server/www/scripts/services/providers/services.js +++ b/server/www/scripts/services/providers/services.js @@ -115,22 +115,13 @@ angular.module('faradayApp') servicesManager.createService = function(serviceData, ws) { var deferred = $q.defer(); var self = this; - - this.getServices(ws).then(function(services) { - var service = new Service(serviceData); - self.getService(service._id, ws).then(function(resp) { - deferred.reject("Service already exists"); - }, function() { - // host doesn't exist, good to go - service.save(ws).then(function(){ - service = self.getService(service._id, ws); - deferred.resolve(service); - }, function(){ - // host couldn't be saved - deferred.reject("Error: host couldn't be saved"); - }) - }); - }); + var service = new Service(serviceData); + service.save(ws).then(function(saved_service){ + deferred.resolve(saved_service); + }, function(){ + // host couldn't be saved + deferred.reject("Error: host couldn't be saved"); + }) return deferred.promise; } From 873afa75c928c636a8402be7f5ce6f451085eea5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 12 Oct 2017 15:33:36 -0300 Subject: [PATCH 0316/1506] Fix hostnames for service vulns --- server/api/modules/vulns.py | 2 +- server/models.py | 16 +++++++++++++ test_cases/conftest.py | 5 +++++ test_cases/factories.py | 10 +++++++++ test_cases/models/test_vulnerability.py | 30 +++++++++++++++++++++++++ test_cases/test_api_vulnerability.py | 14 ++++++++++-- 6 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 test_cases/models/test_vulnerability.py diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index ce96b99b08f..0ad53969dc8 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -45,7 +45,7 @@ class VulnerabilityGenericSchema(AutoSchema): parent = fields.Method('get_parent') tags = fields.Method('get_tags') easeofresolution = fields.String(dump_only=True, attribute='ease_of_resolution') - hostnames = fields.Method('get_hostnames') + hostnames = PrimaryKeyRelatedField('name', many=True) pname = fields.String(dump_only=True, attribute='parameter_name', default='') path = fields.String(default='') response = fields.String(default='') diff --git a/server/models.py b/server/models.py index 796deff58e0..6f5232e1e3d 100644 --- a/server/models.py +++ b/server/models.py @@ -355,6 +355,14 @@ def service_id(cls): def service(cls): return relationship('Service') + @property + def hostnames(self): + if self.host is not None: + return self.host.hostnames + elif self.service is not None: + return self.service.host.hostnames + raise ValueError("Vulnerability has no service nor host") + __mapper_args__ = { 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[0] } @@ -379,6 +387,10 @@ def service_id(cls): def service(cls): return relationship('Service') + @property + def hostnames(self): + return self.service.host.hostnames + __mapper_args__ = { 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[1] } @@ -401,6 +413,10 @@ class VulnerabilityCode(VulnerabilityGeneric): 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[2] } + @property + def hostnames(self): + return [] + class ReferenceTemplate(Metadata): __tablename__ = 'reference_template' diff --git a/test_cases/conftest.py b/test_cases/conftest.py index b6b39f435e5..118395cee3e 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -193,6 +193,11 @@ def user(app, database, session): def ldap_user(app, session): return create_user(app, session, 'ldap', 'ldap@test.com', 'password', is_ldap=True) +@pytest.fixture +def host_with_hostnames(host, hostname_factory): + hostname_factory.create_batch(3, workspace=host.workspace, host=host) + return host + def login_as(test_client, user): with test_client.session_transaction() as sess: diff --git a/test_cases/factories.py b/test_cases/factories.py index 57c7a6e15c0..d7a440ed4f7 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -16,6 +16,7 @@ Credential, EntityMetadata, Host, + Hostname, License, PolicyViolation, Reference, @@ -101,6 +102,15 @@ class Meta: sqlalchemy_session = db.session +class HostnameFactory(WorkspaceObjectFactory): + name = factory.Faker('domain_name') + host = factory.SubFactory(HostFactory) + + class Meta: + model = Hostname + sqlalchemy_session = db.session + + class EntityMetadataFactory(WorkspaceObjectFactory): couchdb_id = factory.Sequence(lambda n: '{0}.1.2'.format(n)) diff --git a/test_cases/models/test_vulnerability.py b/test_cases/models/test_vulnerability.py new file mode 100644 index 00000000000..bb97b3be32a --- /dev/null +++ b/test_cases/models/test_vulnerability.py @@ -0,0 +1,30 @@ +def test_standard_host_vuln_hostnames(vulnerability_factory, + host_with_hostnames, + session): + vuln = vulnerability_factory.create(host=host_with_hostnames, + service=None) + session.commit() + assert set(vuln.hostnames) == set(host_with_hostnames.hostnames) + +def test_standard_vuln_service_hostnames(vulnerability_factory, + service_factory, + host_with_hostnames, + session): + service = service_factory.create(host=host_with_hostnames) + vuln = vulnerability_factory.create(service=service, host=None) + session.commit() + assert set(vuln.hostnames) == set(host_with_hostnames.hostnames) + +def test_web_vuln_hostnames(vulnerability_web_factory, + service_factory, + host_with_hostnames, + session): + service = service_factory.create(host=host_with_hostnames) + vuln = vulnerability_web_factory.create(service=service) + session.commit() + assert set(vuln.hostnames) == set(host_with_hostnames.hostnames) + +def test_code_vuln(vulnerability_code, session): + session.commit() + # Source code vulnerabilities have no hostnames + assert vulnerability_code.hostnames == [] diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 9a82f1f5398..1ccff9f774e 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -73,10 +73,9 @@ def test_(self, test_client, second_workspace, session): def test_handles_vuln_with_no_creator(self, workspace, test_client, - vulnerability_factory, session): # This can happen when a user is deleted but its objects persist - vuln = vulnerability_factory.create(workspace=workspace, creator=None) + vuln = self.factory.create(workspace=workspace, creator=None) session.commit() res = test_client.get(self.url(vuln)) assert res.status_code == 200 @@ -101,3 +100,14 @@ def test_shows_refs(self, workspace, test_client, session, assert res.status_code == 200 assert len(res.json['refs']) == 5 assert set(res.json['refs']) == {ref.name for ref in refs} + + def test_hostnames(self, host_with_hostnames, test_client, session): + vuln = self.factory.create(host=host_with_hostnames, + service=None, + workspace=host_with_hostnames.workspace) + session.commit() + res = test_client.get(self.url(vuln)) + assert res.status_code == 200 + assert isinstance(res.json['hostnames'], list) + assert set(res.json['hostnames']) == set(hostname.name for hostname in + host_with_hostnames.hostnames) From 3710ad18402bbb2a4794f9cd4a2cd5fce0e5f3bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 12 Oct 2017 15:34:34 -0300 Subject: [PATCH 0317/1506] Make vulnweb service not nullable --- server/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index 6f5232e1e3d..1385569699a 100644 --- a/server/models.py +++ b/server/models.py @@ -381,7 +381,9 @@ class VulnerabilityWeb(VulnerabilityGeneric): @declared_attr def service_id(cls): - return VulnerabilityGeneric.__table__.c.get('service_id', Column(Integer, db.ForeignKey('service.id'))) + return VulnerabilityGeneric.__table__.c.get( + 'service_id', Column(Integer, db.ForeignKey('service.id'), + nullable=False)) @declared_attr def service(cls): From 5b0b466b4c6221bdd4cc81c7e358a0582d3917eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 12 Oct 2017 15:54:40 -0300 Subject: [PATCH 0318/1506] Add check to vuln factory call to prevent bugs in the tests Prevent accidentally randomly pick of host or service instead of using the one passed as an argument --- test_cases/factories.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test_cases/factories.py b/test_cases/factories.py index d7a440ed4f7..7fc3c1a5728 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -171,6 +171,16 @@ class VulnerabilityFactory(VulnerabilityGenericFactory): host = factory.SubFactory(HostFactory) service = factory.SubFactory(ServiceFactory) + @classmethod + def attributes(cls, create=False, extra=None): + if extra: + if ('host' in extra and 'service' not in extra) or \ + ('service' in extra and 'host' not in extra): + raise ValueError('You should pass both service and host and ' + 'set one of them to None to prevent random ' + 'stuff to happen') + return super(VulnerabilityFactory, cls).attributes(create, extra) + @classmethod def _after_postgeneration(cls, obj, create, results=None): super(VulnerabilityFactory, cls)._after_postgeneration( From 1c1cfeda57b90a422cc9a761a055e74a936c05c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 12 Oct 2017 16:04:23 -0300 Subject: [PATCH 0319/1506] Fix some API tests --- test_cases/test_api_services.py | 2 +- test_cases/test_api_vulnerability.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index e0a8a1c991b..428b96783fe 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -44,4 +44,4 @@ def test_(self, test_client, second_workspace, session): u'ports', u'metadata' ] - assert set(object_properties) == set(vuln['value'].keys()) \ No newline at end of file + assert set(vuln['value'].keys()).issubset(object_properties) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 1ccff9f774e..bb4099a304c 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -68,7 +68,7 @@ def test_(self, test_client, second_workspace, session): u'method', u'metadata' ] - assert set(object_properties) == set(vuln['value'].keys()) + assert set(vuln['value'].keys()).issubset(object_properties) def test_handles_vuln_with_no_creator(self, workspace, From d54b88cb6643a46ae43e7e0f5ee2abed0d9a4bfc Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 12 Oct 2017 18:30:02 -0300 Subject: [PATCH 0320/1506] Add Tags to Vulnerabilities during the migration Now the Tags that are present in CouchDB for Vulnerabilities are imported into Postgres. Also, removed troubleshooting params left in function by mistake in my last commit. --- server/importer.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/server/importer.py b/server/importer.py index e67d5e64ec2..dd0235a0bcd 100644 --- a/server/importer.py +++ b/server/importer.py @@ -435,8 +435,6 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation vulnerability.impact_confidentiality = document.get('impact', {}).get('confidentiality') vulnerability.impact_integrity = document.get('impact', {}).get('integrity') if document['type'] == 'VulnerabilityWeb': - - vulnerability.query = document.get('query') vulnerability.request = document.get('request') vulnerability.response = document.get('response') @@ -455,6 +453,13 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation self.add_references(document, vulnerability, workspace) self.add_policy_violations(document, vulnerability, workspace) + + # need the vuln ID before creating Tags for it + session.commit() + tags = document.get('tags', []) + if len(tags): + create_tags(tags, vulnerability.id, document['type']) + yield vulnerability def add_policy_violations(self, document, vulnerability, workspace): @@ -619,10 +624,9 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation # we need the ID of the Task in order to add tags to it session.commit() - tags = document.get('tags', []) if len(tags): - create_tags(tags, task.id, 'task', task, task_created) + create_tags(tags, task.id, 'task') #task.due_date = datetime.datetime.fromtimestamp(document.get('due_date')) return [task] From 5c7a26ce2d4b765a8072e5b6ee5c1cb063f3c0b5 Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 12 Oct 2017 18:31:33 -0300 Subject: [PATCH 0321/1506] Fix bug when importing invalid chars Some characters (for example, 0x00) created problems with sqlalchemy on import. The migrator now escapes them before attempting to populate Postgres. --- server/importer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/importer.py b/server/importer.py index dd0235a0bcd..ed1443df0f1 100644 --- a/server/importer.py +++ b/server/importer.py @@ -54,6 +54,7 @@ VulnerabilityWeb, Workspace, ) +from server.utils import invalid_chars from server.utils.database import get_or_create from server.web import app @@ -1039,6 +1040,9 @@ def import_workspace_into_database(self, workspace_name): raw_obj = raw_obj['value'] couchdb_id = raw_obj['_id'] + # first let's make sure no invalid chars are present in the Raw objects + raw_obj = invalid_chars.clean_dict(raw_obj) + for new_obj in obj_importer.update_from_document(raw_obj, workspace, level, couchdb_relational_map): if not new_obj: continue From 03e3f23548eb6e6a1f041d168d4b595e2567cf1c Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 12 Oct 2017 18:32:50 -0300 Subject: [PATCH 0322/1506] Fix Web Vuln fields type in Model The type of some columns was left as String(n) by mistake. --- server/models.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server/models.py b/server/models.py index e128be9e92d..ff6d4802888 100644 --- a/server/models.py +++ b/server/models.py @@ -362,14 +362,14 @@ def service(cls): class VulnerabilityWeb(VulnerabilityGeneric): __tablename__ = None - method = Column(String(50), nullable=True) - parameters = Column(String(500), nullable=True) - parameter_name = Column(String(250), nullable=True) - path = Column(String(500), nullable=True) - query = Column(Text(), nullable=True) - request = Column(Text(), nullable=True) - response = Column(Text(), nullable=True) - website = Column(String(250), nullable=True) + method = Column(Text, nullable=True) + parameters = Column(Text, nullable=True) + parameter_name = Column(Text, nullable=True) + path = Column(Text, nullable=True) + query = Column(Text, nullable=True) + request = Column(Text, nullable=True) + response = Column(Text, nullable=True) + website = Column(Text, nullable=True) @declared_attr def service_id(cls): From b4c71400c6e66e17d73d1bfd6a81f2009157cc44 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 12 Oct 2017 18:57:22 -0300 Subject: [PATCH 0323/1506] Add vulns count on host services view --- server/api/modules/hosts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index fae060769a1..89f31f6ef12 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -70,10 +70,12 @@ class Meta(FilterSetMeta): class ServiceSchema(AutoSchema): + vulns = fields.Function(lambda service: len(service.vulnerabilities) + len(service.vulnerabilities_web), dump_only=True) + class Meta: model = Service - fields = ('id', 'name', 'description', 'port', 'protocol', 'status') + fields = ('id', 'name', 'description', 'port', 'protocol', 'status', 'vulns') class HostsView(PaginatedMixin, From f59d932822bafd7dc7d6157289b0a13589abb8f9 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 12 Oct 2017 18:58:26 -0300 Subject: [PATCH 0324/1506] Allow to create and delete services of a host --- server/api/modules/services.py | 7 +--- server/models.py | 4 +- .../www/scripts/commons/providers/server.js | 41 ++++++------------- .../scripts/dashboard/providers/dashboard.js | 6 +-- server/www/scripts/hosts/controllers/host.js | 26 ++++-------- .../www/scripts/services/providers/service.js | 22 +--------- .../scripts/services/providers/services.js | 41 ++++--------------- 7 files changed, 36 insertions(+), 111 deletions(-) diff --git a/server/api/modules/services.py b/server/api/modules/services.py index c731457347f..d88ba4a1f82 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -25,10 +25,7 @@ class ServiceSchema(AutoSchema): owner = fields.String(dump_only=True, attribute='creator.username') ports = fields.Method(attribute='port', deserialize='load_port') status = fields.String(default='open') - parent = fields.Method(attribute='host_id', deserialize='load_host_id', load_only=True) - - def load_host_id(self, value): - return value + parent = fields.Integer(attribute='host_id', load_only=True) def load_port(self, value): return str(value.pop()) @@ -46,7 +43,7 @@ def get_metadata(self, obj): class Meta: model = Service - fields = ('_id', 'status', 'parent', + fields = ('id', '_id', 'status', 'parent', 'protocol', 'description', '_rev', 'owned', 'owner', 'credentials', 'name', 'version', '_id', 'ports', diff --git a/server/models.py b/server/models.py index 1385569699a..6db02b33907 100644 --- a/server/models.py +++ b/server/models.py @@ -353,7 +353,7 @@ def service_id(cls): @declared_attr def service(cls): - return relationship('Service') + return relationship('Service', backref='vulnerabilities') @property def hostnames(self): @@ -387,7 +387,7 @@ def service_id(cls): @declared_attr def service(cls): - return relationship('Service') + return relationship('Service', backref='vulnerabilities_web') @property def hostnames(self): diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index d5a9f92696a..eee149f30d5 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -25,8 +25,8 @@ angular.module("faradayApp") return APIURL + "ws/" + wsName + objectName + "/"; }; - var createNewGetUrl = function(wsName, objId, objectType) { - return APIURL + "ws/" + wsName + "/" + objectType + "/" + objId; + var createNewGetUrl = function(wsName, objectId, objectType) { + return APIURL + "ws/" + wsName + "/" + objectType + "/" + objectId; } var createPostUrl = function(wsName, objectId, objectType) { @@ -34,14 +34,14 @@ angular.module("faradayApp") }; var createPutUrl = function(wsName, objectId, objectType) { - return APIURL + "ws/" + wsName + "/" + objectType + "/" + objectId; + return APIURL + "ws/" + wsName + "/" + objectType + "/" + objectId + "/"; }; var createDbUrl = function(wsName) { return APIURL + "ws/" + wsName; } - var createDeleteUrl = createPostUrl; + var createDeleteUrl = createPutUrl; var serverComm = function(method, url, data) { var success = function (response) { @@ -81,32 +81,10 @@ angular.module("faradayApp") // just set rev_provided to false if you're deleting a database :) var _delete = function(url, rev_provided) { // never let undefined win - if (typeof rev_provided === "undefined") {var rev_provided = false;} var deferred = $q.defer(); var data = {}; - - if (rev_provided === false) { - get(url).then( - function s(r) { - data.rev = r.data._rev; - return serverComm("DELETE", url, data).then( - function(res) { - deferred.resolve(res); - }, function(err) { - deferred.reject(err); - } - ); - }, - function e(r) { - deferred.reject(r); - }) - } - else{ - return serverComm("DELETE", url, data); - } - - return deferred.promise; + return serverComm("DELETE", url, data); }; var modHost = function(createOrUpdate, wsName, host) { @@ -204,6 +182,11 @@ angular.module("faradayApp") return get(getUrl, data); } + ServerAPI.getService = function(wsName, data, objId) { + var getUrl = createGetUrl(wsName, 'services', objId); + return get(getUrl); + } + ServerAPI.getServices = function(wsName, data) { var getUrl = createGetUrl(wsName, 'services'); return get(getUrl, data); @@ -370,8 +353,8 @@ angular.module("faradayApp") } } - ServerAPI.deleteService = function(wsName, serviceId, rev) { - var deleteUrl = createDeleteUrl(wsName, serviceId, rev); + ServerAPI.deleteService = function(wsName, serviceId) { + var deleteUrl = createDeleteUrl(wsName, serviceId, 'services'); if (typeof rev === "undefined") { return _delete(deleteUrl, false) } diff --git a/server/www/scripts/dashboard/providers/dashboard.js b/server/www/scripts/dashboard/providers/dashboard.js index 6da1a1df740..6c2741dab5f 100644 --- a/server/www/scripts/dashboard/providers/dashboard.js +++ b/server/www/scripts/dashboard/providers/dashboard.js @@ -305,11 +305,7 @@ angular.module('faradayApp') var deferred = $q.defer(); ServerAPI.getServicesByHost(ws, host_id).then(function(res){ - var tmp = []; - res.data.forEach(function(service){ - tmp.push(service); - }); - deferred.resolve(tmp); + deferred.resolve(res.data); }, function(){ deferred.reject(); }); diff --git a/server/www/scripts/hosts/controllers/host.js b/server/www/scripts/hosts/controllers/host.js index ef11ed07bb9..7d304965f7a 100644 --- a/server/www/scripts/hosts/controllers/host.js +++ b/server/www/scripts/hosts/controllers/host.js @@ -46,13 +46,7 @@ angular.module('faradayApp') // services by host dashboardSrv.getServicesByHost($scope.workspace, hostId) .then(function(services) { - var pss = []; - - services.forEach(function(service) { - pss.push(service); - }); - - return $q.all(pss); + return $q.all(services); }) .then(function(services) { $scope.services = services; @@ -63,7 +57,7 @@ angular.module('faradayApp') $scope.loadedServices = true; - return servicesManager.getServiceVulnCount($scope.workspace, $scope.services) + return services; }) .then(function(vulns) { $scope.services.forEach(function(service) { @@ -309,12 +303,7 @@ angular.module('faradayApp') }; $scope.delete = function() { - var selected = []; - $scope.selectedServices().forEach(function(service){ - if(service.selected){ - selected.push(service._id); - } - }); + var selected = $scope.selectedServices(); if(selected.length == 0) { $uibModal.open(config = { @@ -376,12 +365,13 @@ angular.module('faradayApp') }); }; - $scope.remove = function(ids) { - ids.forEach(function(id) { - servicesManager.deleteServices(id, $scope.workspace).then(function() { + $scope.remove = function(services) { + //removes services from host + services.forEach(function(service) { + servicesManager.deleteServices(service, $scope.workspace).then(function() { var index = -1; for(var i=0; i < $scope.services.length; i++) { - if($scope.services[i]._id === id) { + if($scope.services[i]._id === service.id) { index = i; break; } diff --git a/server/www/scripts/services/providers/service.js b/server/www/scripts/services/providers/service.js index e51e6b22bf6..6537715ada3 100644 --- a/server/www/scripts/services/providers/service.js +++ b/server/www/scripts/services/providers/service.js @@ -21,28 +21,13 @@ angular.module('faradayApp') Service.prototype = { public_properties: public_properties, saved_properties: saved_properties, - // TODO: instead of using angular.extend, we should check - // the attributes we're assigning to the Service set: function(data) { - // if there's no ID, we need to generate it based on the Service name - if(data._id === undefined) { - var ports = data.ports; - - if(typeof data.ports == "object") { - ports = ports.toString().replace(/,/g,":"); - } - } data.type = "Service"; angular.extend(this, data); - if( typeof(data.ports) === 'number' ) { - this.ports = data.ports; - } else { - this.ports = data.ports[0]; - } }, delete: function(ws) { - return ServerAPI.deleteService(ws, this._id, this._rev); + return ServerAPI.deleteService(ws, this.id); }, update: function(data, ws) { @@ -65,10 +50,7 @@ angular.module('faradayApp') self.ports = [self.ports]; } - return (self._save(ws, self, false).then( - function(data){ - self._rev = data.rev; - })); + return self._save(ws, self, false); }, _save: function(ws, data, isUpdate) { if (typeof isUpdate === 'undefined') {isUpdate = false}; diff --git a/server/www/scripts/services/providers/services.js b/server/www/scripts/services/providers/services.js index 22cf983ea70..f3f68d0582d 100644 --- a/server/www/scripts/services/providers/services.js +++ b/server/www/scripts/services/providers/services.js @@ -27,7 +27,7 @@ angular.module('faradayApp') servicesManager._load = function(id, ws, deferred) { var self = this; - ServerAPI.getServices(ws, {'couchid': id}). + ServerAPI.getService(ws, id). then(function(response) { if (response.data.services.length == 1) { var service_data = response.data.services[0].value; @@ -74,41 +74,18 @@ angular.module('faradayApp') return deferred.promise; } - servicesManager.deleteServices = function(id, ws) { + servicesManager.deleteServices = function(service, ws) { var deferred = $q.defer(); var self = this; - this.getService(id, ws).then(function(service) { - service.delete(ws).then(function() { - delete self._objects[id]; - deferred.resolve(); - }, function(){ - // host couldn't be deleted - deferred.reject("Error deleting service"); - }); + var service = self._get(service.id, service); + service.delete(ws).then(function() { + delete self._objects[service.id]; + deferred.resolve(); }, function(){ - // host doesn't exist - deferred.reject("Service doesn't exist"); + // host couldn't be deleted + deferred.reject("Error deleting service"); }); - return deferred.promise; - } - servicesManager.getServiceVulnCount = function(ws, services) { - var deferred = $q.defer(); - var promises = []; - services.forEach(function(service) { - promises.push(ServerAPI.getServices(ws, {'couchid': service._id})); - }); - - $q.all(promises).then(function(services){ - var result = {}; - services.forEach(function(service) { - var service_data = service.data.services[0]; - result[service_data.id] = service_data.vulns; - }); - deferred.resolve(result); - }, function(){ - deferred.reject([]); - }); return deferred.promise; } @@ -117,7 +94,7 @@ angular.module('faradayApp') var self = this; var service = new Service(serviceData); service.save(ws).then(function(saved_service){ - deferred.resolve(saved_service); + deferred.resolve(saved_service.data); }, function(){ // host couldn't be saved deferred.reject("Error: host couldn't be saved"); From 129480ea1d97a4826f7c117cfaf0c12a7d2821fa Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 12 Oct 2017 19:04:36 -0300 Subject: [PATCH 0325/1506] Fix count url --- server/www/scripts/commons/providers/server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index eee149f30d5..242d1d78d00 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -257,7 +257,7 @@ angular.module("faradayApp") } ServerAPI.getServicesBy = function(wsName, what) { - var url = createGetUrl(wsName, 'services') + '/count/'; + var url = createGetUrl(wsName, 'services') + 'count/'; return get(url, {"group_by": what}) } @@ -272,7 +272,7 @@ angular.module("faradayApp") ServerAPI.getVulnsBySeverity = function(wsName, confirmed) { - var url = createGetUrl(wsName, 'vulns') + '/count/'; + var url = createGetUrl(wsName, 'vulns') + 'count/'; var payload = {'group_by': 'severity'} if (confirmed !== undefined) { From 2d9e5f0fb346a6ddd1f7ba9b92dffa62af26b336 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 12 Oct 2017 19:16:25 -0300 Subject: [PATCH 0326/1506] Sometimes end_date or start_date are None. ensure both are not None --- server/api/modules/commandsrun.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/api/modules/commandsrun.py b/server/api/modules/commandsrun.py index 70262211be9..066fe95381d 100644 --- a/server/api/modules/commandsrun.py +++ b/server/api/modules/commandsrun.py @@ -33,7 +33,8 @@ def get_itime(self, obj): return time.mktime(obj.start_date.utctimetuple()) def get_duration(self, obj): - return (obj.end_date - obj.start_date).seconds + if obj.end_date and obj.start_date: + return (obj.end_date - obj.start_date).seconds class Meta: model = Command From edf89aff3b1e13831504e9697abeb2018c010c66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 12 Oct 2017 19:24:58 -0300 Subject: [PATCH 0327/1506] Fix vuln service in status report Also paginate vulns view and improve tests --- server/api/modules/services.py | 6 +++++- server/api/modules/vulns.py | 13 ++++++++++--- .../statusReport/controllers/statusReport.js | 4 +--- .../partials/ui-grid/columns/servicecolumn.html | 2 +- test_cases/test_api_services.py | 4 +++- test_cases/test_api_vulnerability.py | 4 +++- 6 files changed, 23 insertions(+), 10 deletions(-) diff --git a/server/api/modules/services.py b/server/api/modules/services.py index c731457347f..bf56a49079c 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -26,6 +26,7 @@ class ServiceSchema(AutoSchema): ports = fields.Method(attribute='port', deserialize='load_port') status = fields.String(default='open') parent = fields.Method(attribute='host_id', deserialize='load_host_id', load_only=True) + summary = fields.Method('get_summary') def load_host_id(self, value): return value @@ -44,13 +45,16 @@ def get_metadata(self, obj): "update_user": "" } + def get_summary(self, obj): + return "(%s/%s) %s" % (obj.port, obj.protocol, obj.name) + class Meta: model = Service fields = ('_id', 'status', 'parent', 'protocol', 'description', '_rev', 'owned', 'owner', 'credentials', 'name', 'version', '_id', 'ports', - 'metadata') + 'metadata', 'summary') class ServiceView(ReadWriteWorkspacedView): diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 0ad53969dc8..50d88ce5b02 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -8,7 +8,11 @@ from flask import Blueprint from marshmallow import fields -from server.api.base import AutoSchema, ReadWriteWorkspacedView +from server.api.base import ( + AutoSchema, + PaginatedMixin, + ReadWriteWorkspacedView, +) from server.models import ( db, VulnerabilityGeneric, @@ -22,6 +26,7 @@ get_integer_parameter, filter_request_args ) +from server.api.modules.services import ServiceSchema from server.schemas import PrimaryKeyRelatedField from server.dao.vuln import VulnerabilityDAO @@ -56,7 +61,9 @@ class VulnerabilityGenericSchema(AutoSchema): target = fields.String(default='') # TODO: review this attribute query = fields.String(dump_only=True, attribute='query_string', default='') metadata = fields.Method('get_metadata') - service = fields.Integer(dump_only=True, attribute='service_id') + service = fields.Nested(ServiceSchema(only=[ + '_id', 'ports', 'status', 'protocol', 'name', 'version', 'summary' + ]), dump_only=True) host = fields.Integer(dump_only=True, attribute='host_id') class Meta: @@ -120,7 +127,7 @@ def get_impact(self, obj): } -class VulnerabilityView(ReadWriteWorkspacedView): +class VulnerabilityView(PaginatedMixin, ReadWriteWorkspacedView): route_base = 'vulns' model_class = VulnerabilityGeneric schema_class = VulnerabilityGenericSchema diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index a2cb6c1f083..d0406b9a51f 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -1014,9 +1014,7 @@ angular.module('faradayApp') }); }; - $scope.serviceSearch = function(srvStr) { - //TODO: this is horrible - var srvName = srvStr.split(') ')[1]; + $scope.serviceSearch = function(srvName) { return $scope.encodeUrl(srvName); } diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/servicecolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/servicecolumn.html index 0ce9e7e73c7..452abf19aa6 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/servicecolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/servicecolumn.html @@ -1 +1 @@ -
    {{COL_FIELD.split('(')[0] !== ' ' ? COL_FIELD : 'EMPTY' + COL_FIELD}}
    \ No newline at end of file +
    {{COL_FIELD ? COL_FIELD.summary : 'EMPTY'}}
    diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index 428b96783fe..796bd8af4b9 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -44,4 +44,6 @@ def test_(self, test_client, second_workspace, session): u'ports', u'metadata' ] - assert set(vuln['value'].keys()).issubset(object_properties) + expected = set(object_properties) + result = set(vuln['value'].keys()) + assert expected <= result diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index bb4099a304c..6357f3d3799 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -68,7 +68,9 @@ def test_(self, test_client, second_workspace, session): u'method', u'metadata' ] - assert set(vuln['value'].keys()).issubset(object_properties) + expected = set(object_properties) + result = set(vuln['value'].keys()) + assert expected <= result def test_handles_vuln_with_no_creator(self, workspace, From 586a46cac65396a046ab4c9226e2c7bed1fe90a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 12 Oct 2017 19:54:55 -0300 Subject: [PATCH 0328/1506] Remove unused ports field in the test of services API --- test_cases/test_api_services.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index 796bd8af4b9..6bae7bc7155 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -41,7 +41,6 @@ def test_(self, test_client, second_workspace, session): u'name', u'version', u'_id', - u'ports', u'metadata' ] expected = set(object_properties) From 7b5cee0bc20cc0abfc2c7f7a1e30f80bc965fccd Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 13 Oct 2017 11:06:24 -0300 Subject: [PATCH 0329/1506] Use count property for query optimization on views --- server/api/modules/hosts.py | 8 ++++---- server/models.py | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 89f31f6ef12..dcd870720bc 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -39,8 +39,9 @@ class HostSchema(AutoSchema): os = fields.String(default='') owned = fields.Boolean(default=False) owner = PrimaryKeyRelatedField('username', attribute='creator', dump_only=True) - services = fields.Function(lambda host: len(host.services)) - vulns = fields.Function(lambda host: len(host.vulnerabilities)) + services = fields.Integer(attribute='service_count', dump_only=True) + vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) + def get_metadata(self, obj): return { @@ -70,8 +71,7 @@ class Meta(FilterSetMeta): class ServiceSchema(AutoSchema): - vulns = fields.Function(lambda service: len(service.vulnerabilities) + len(service.vulnerabilities_web), dump_only=True) - + vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) class Meta: model = Service diff --git a/server/models.py b/server/models.py index 6db02b33907..761fff75b6b 100644 --- a/server/models.py +++ b/server/models.py @@ -192,6 +192,7 @@ class Host(Metadata): ) service_count = _make_generic_count_property('host', 'service') + vulnerability_count = _make_generic_count_property('host', 'vulnerability') __table_args__ = ( UniqueConstraint(ip, workspace_id, name='uix_host_ip_workspace'), @@ -253,6 +254,9 @@ class Service(Metadata): backref='services', foreign_keys=[workspace_id] ) + + vulnerability_count = _make_generic_count_property('service', 'vulnerability') + __table_args__ = ( UniqueConstraint(port, protocol, host_id, workspace_id, name='uix_service_port_protocol_host_workspace'), ) From 6b9ed9ff3eff1da377d9a8f10fe6062283718a55 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 13 Oct 2017 12:10:19 -0300 Subject: [PATCH 0330/1506] Add test to verify vulns count --- test_cases/test_api_hosts.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 107c0b9c183..f9f571d78ca 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -253,6 +253,37 @@ def test_filter_by_os_like_ilike(self, test_client, session, workspace, assert res.status_code == 200 self.compare_results(hosts + [case_insensitive_host], res) + def test_host_with_open_vuln_count_verification(self, test_client, session, + workspace, host_factory, vulnerability_factory, + service_factory): + + host = host_factory.create(workspace=workspace) + service = service_factory.create(host=host) + vulnerability_factory.create(service=service, host=None) + vulnerability_factory.create(service=None, host=host) + + session.commit() + + res = test_client.get(self.url()) + assert res.status_code == 200 + json_host = filter(lambda json_host: json_host['value']['id'] == host.id, res.json['rows'])[0] + # the host has one vuln associated. another one via service. + assert json_host['value']['vulns'] == 1 + + def test_host_services_vuln_count_verification(self, test_client, session, + workspace, host_factory, vulnerability_factory, + service_factory): + host = host_factory.create(workspace=workspace) + service = service_factory.create(host=host) + vulnerability_factory.create(service=service, host=None) + session.commit() + + res = test_client.get(self.url() + str(host.id) + "/" + 'services/') + assert res.status_code == 200 + assert res.json[0]['vulns'] == 1 + + + class TestHostAPIGeneric(ReadWriteAPITests, PaginationTestsMixin): model = Host From 40222a9427fee5976095f40eb27e58900bfe3806 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 13 Oct 2017 14:20:50 -0300 Subject: [PATCH 0331/1506] Fix new vuln host and services list --- server/api/modules/services.py | 3 ++- server/www/scripts/statusReport/providers/target.js | 11 ++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/server/api/modules/services.py b/server/api/modules/services.py index d885488a182..30c4e9037f3 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -26,6 +26,7 @@ class ServiceSchema(AutoSchema): ports = fields.Method(attribute='port', deserialize='load_port') status = fields.String(default='open') parent = fields.Integer(attribute='host_id', load_only=True) + host_id = fields.Integer(attribute='host_id', dump_only=True) summary = fields.Method('get_summary') def load_port(self, value): @@ -51,7 +52,7 @@ class Meta: 'protocol', 'description', '_rev', 'owned', 'owner', 'credentials', 'name', 'version', '_id', 'ports', - 'metadata', 'summary') + 'metadata', 'summary', 'host_id') class ServiceView(ReadWriteWorkspacedView): diff --git a/server/www/scripts/statusReport/providers/target.js b/server/www/scripts/statusReport/providers/target.js index 6bff8b00c14..4d995aaebba 100644 --- a/server/www/scripts/statusReport/providers/target.js +++ b/server/www/scripts/statusReport/providers/target.js @@ -17,17 +17,10 @@ angular.module('faradayApp') hosts_dict[host._id] = host; res.push(host); }); - hostsManager.getAllInterfaces(workspace).then(function(interfaces) { - interfaces.forEach(function(interf) { - host_id = interf._id.split(".")[0]; - if (hosts_dict.hasOwnProperty(host_id)) { - hosts_dict[host_id].hostnames = hosts_dict[host_id].hostnames.concat(interf.hostnames); - } - }); - }, function(err) {deferred.reject(err)}); + servicesManager.getServices(workspace).then(function(services) { services.forEach(function(service) { - host_id = service._id.split(".")[0]; + host_id = service.host_id if (hosts_dict.hasOwnProperty(host_id)) { hosts_dict[host_id].services.push(service); } From c57d7e94b42019e2b8ee4f6c9aad3de7e22e7049 Mon Sep 17 00:00:00 2001 From: micabot Date: Fri, 13 Oct 2017 17:06:56 -0300 Subject: [PATCH 0332/1506] Add cleaner for invalid chars --- server/utils/invalid_chars.py | 88 +++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 server/utils/invalid_chars.py diff --git a/server/utils/invalid_chars.py b/server/utils/invalid_chars.py new file mode 100644 index 00000000000..f3d8098f581 --- /dev/null +++ b/server/utils/invalid_chars.py @@ -0,0 +1,88 @@ +import re +import sys + +def clean_dict(d): + if not isinstance(d, dict): + return d + else: + new_dict = dict() + for key, value in d.iteritems(): + if isinstance(value, basestring): + new_dict[key] = clean_string(value) + elif isinstance(value, dict): + new_dict[key] = clean_dict(value) + elif isinstance(value, list): + new_dict[key] = clean_list(value) + else: + new_dict[key] = value + return new_dict + + +def clean_list(l): + if not isinstance(l, list): + return l + else: + new_list = list() + for item in l: + if isinstance(item, basestring): + new_list.append(clean_string(item)) + elif isinstance(item, dict): + new_list.append(clean_dict(item)) + elif isinstance(item, list): + new_list.append(clean_list(item)) + else: + new_list.append(item) + return new_list + + +def clean_string(s): + return ''.join([clean_char(x) for x in s]) + + +def clean_char(char): + #Get rid of the ctrl characters first. + #http://stackoverflow.com/questions/1833873/python-regex-escape-characters + char = re.sub('\x1b[^m]*m', '', char) + #Clean up invalid xml + char = remove_invalid_chars(char) + replacements = [ + (u'\u201c', '\"'), + (u'\u201d', '\"'), + (u"\u001B", ' '), #http://www.fileformat.info/info/unicode/char/1b/index.htm + (u"\u0019", ' '), #http://www.fileformat.info/info/unicode/char/19/index.htm + (u"\u0016", ' '), #http://www.fileformat.info/info/unicode/char/16/index.htm + (u"\u001C", ' '), #http://www.fileformat.info/info/unicode/char/1c/index.htm + (u"\u0003", ' '), #http://www.utf8-chartable.de/unicode-utf8-table.pl?utf8=0x + (u"\u000C", ' ') + ] + for rep, new_char in replacements: + if char == rep: + #print ord(char), char.encode('ascii', 'ignore') + return new_char + return char + + +def remove_invalid_chars(c): + #http://stackoverflow.com/questions/1707890/fast-way-to-filter-illegal-xml-unicode-chars-in-python + illegal_unichrs = [ (0x00, 0x08), (0x0B, 0x1F), (0x7F, 0x84), (0x86, 0x9F), + (0xD800, 0xDFFF), (0xFDD0, 0xFDDF), (0xFFFE, 0xFFFF), + (0x1FFFE, 0x1FFFF), (0x2FFFE, 0x2FFFF), (0x3FFFE, 0x3FFFF), + (0x4FFFE, 0x4FFFF), (0x5FFFE, 0x5FFFF), (0x6FFFE, 0x6FFFF), + (0x7FFFE, 0x7FFFF), (0x8FFFE, 0x8FFFF), (0x9FFFE, 0x9FFFF), + (0xAFFFE, 0xAFFFF), (0xBFFFE, 0xBFFFF), (0xCFFFE, 0xCFFFF), + (0xDFFFE, 0xDFFFF), (0xEFFFE, 0xEFFFF), (0xFFFFE, 0xFFFFF), + (0x10FFFE, 0x10FFFF) ] + + illegal_ranges = ["%s-%s" % (unichr(low), unichr(high)) + for (low, high) in illegal_unichrs + if low < sys.maxunicode] + + illegal_xml_re = re.compile(u'[%s]' % u''.join(illegal_ranges)) + if illegal_xml_re.search(c) is not None: + # ret = hex(ord(c)) + # ret = binascii.b2a_uu(c) + # ret_final = ret[1:-1] + ret = '\\x'+c.encode('hex') + return ret + else: + return c From 239f6144b64ae172b5fcd1da1d676f95d5733e18 Mon Sep 17 00:00:00 2001 From: micabot Date: Fri, 13 Oct 2017 18:42:23 -0300 Subject: [PATCH 0333/1506] Add error logging when the Web fails When the Server attempts to serve the Web UI, an unhandled exception was shown to the user if the server is already running. Added an error message instead. --- server/web.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/server/web.py b/server/web.py index a5e52a9b048..6fc62229707 100644 --- a/server/web.py +++ b/server/web.py @@ -3,6 +3,7 @@ # See the file 'doc/LICENSE' for the license information import os +import sys import functools import twisted.web from twisted.web.resource import Resource @@ -18,7 +19,7 @@ from server.app import create_app app = create_app() # creates a Flask(__name__) app - +logger = server.utils.logger.get_logger(__name__) class WebServer(object): UI_URL_PATH = '_ui' @@ -26,7 +27,7 @@ class WebServer(object): WEB_UI_LOCAL_PATH = os.path.join(server.config.FARADAY_BASE, 'server/www') def __init__(self, enable_ssl=False): - logger.get_logger(__name__).info('Starting server at port {0} with bind address {1}. SSL {2}'.format( + logger.info('Starting server at port {0} with bind address {1}. SSL {2}'.format( server.config.faraday_server.port, server.config.faraday_server.bind_address, enable_ssl)) @@ -43,7 +44,7 @@ def __config_server(self): def __load_ssl_certs(self): certs = (server.config.ssl.keyfile, server.config.ssl.certificate) if not all(certs): - logger.get_logger(__name__).critical("HTTPS request but SSL certificates are not configured") + logger.critical("HTTPS request but SSL certificates are not configured") exit(1) # Abort web-server startup return ssl.DefaultOpenSSLContextFactory(*certs) @@ -71,8 +72,11 @@ def run(self): self.__couchdb_port = int(server.config.couchdb.port) self.__listen_func = reactor.listenTCP - self.__listen_func( - self.__listen_port, site, - interface=self.__bind_address) + try: + self.__listen_func( + self.__listen_port, site, + interface=self.__bind_address) + except Exception as e: + logger.error('Something went wrong when trying to setup the Web UI') reactor.run() From 7d39212c69fd491369cc1f940669cabb2727dc01 Mon Sep 17 00:00:00 2001 From: micabot Date: Fri, 13 Oct 2017 18:56:24 -0300 Subject: [PATCH 0334/1506] Remove logic to create Vuln Templates DB With CouchDB it was necessary to create the database for vuln templates when it didn't already exist. That logic is no longer useful. --- .../vulndb/controllers/modalCreateDB.js | 32 ---------------- .../scripts/vulndb/controllers/vulnModels.js | 37 ++++--------------- .../vulndb/partials/modalCreateDB.html | 12 ------ .../www/scripts/vulndb/partials/vulndb.html | 2 +- 4 files changed, 9 insertions(+), 74 deletions(-) delete mode 100644 server/www/scripts/vulndb/controllers/modalCreateDB.js delete mode 100644 server/www/scripts/vulndb/partials/modalCreateDB.html diff --git a/server/www/scripts/vulndb/controllers/modalCreateDB.js b/server/www/scripts/vulndb/controllers/modalCreateDB.js deleted file mode 100644 index aecdc141795..00000000000 --- a/server/www/scripts/vulndb/controllers/modalCreateDB.js +++ /dev/null @@ -1,32 +0,0 @@ -// Faraday Penetration Test IDE -// Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) -// See the file 'doc/LICENSE' for the license information - -angular.module('faradayApp') - .controller('vulndbModalCreateDB', - ['$scope', '$modalInstance', 'vulnModelsManager', - function($scope, $modalInstance, vulnModelsManager) { - - $scope.message; - - var init = function() { - $scope.message = "It looks like your Faraday installation is missing "+ - "the Vulnerability Model database. Would you like to create it now?"; - }; - - $scope.yes = function() { - vulnModelsManager.createDB() - .then(function() { - $modalInstance.close(true); - }, function() { - $scope.message = "There's been a problem creating the database."; - }); - }; - - $scope.no = function() { - $modalInstance.close(null); - $modalInstance.dismiss(null); - }; - - init(); - }]); diff --git a/server/www/scripts/vulndb/controllers/vulnModels.js b/server/www/scripts/vulndb/controllers/vulnModels.js index 974142acd37..397b2299901 100644 --- a/server/www/scripts/vulndb/controllers/vulnModels.js +++ b/server/www/scripts/vulndb/controllers/vulnModels.js @@ -2,7 +2,6 @@ angular.module('faradayApp') .controller('vulnModelsCtrl', ['$scope', '$filter', '$http', '$q', '$uibModal', 'ServerAPI', 'csvService', 'commonsFact', 'vulnModelsManager', function($scope, $filter, $http, $q, $uibModal, ServerAPI, csvService, commonsFact, vulnModelsManager) { - $scope.db_exists = false; $scope.models = []; $scope.loaded_models = false; $scope.totalModels = 0; @@ -19,34 +18,14 @@ angular.module('faradayApp') $scope.reverse = true; $scope.currentPage = 1; - vulnModelsManager.DBExists() - .then(function(exists) { - if (!exists) { - $uibModal.open({ - templateUrl: 'scripts/vulndb/partials/modalCreateDB.html', - controller: 'vulndbModalCreateDB', - size: 'lg' - }).result.then(function(data) { - if (data) { - $scope.db_exists = true; - } - }, function(message) { - // no db created, do nothing! - }); - } else { - $scope.db_exists = true; - vulnModelsManager.get() - .then(function() { - $scope.models = vulnModelsManager.models; - $scope.loaded_models = true; - }); - vulnModelsManager.getSize(). - then(function() { - $scope.totalModels = vulnModelsManager.totalNumberOfModels; - }); - } - }, function(message) { - commonsFact.errorDialog(message); + vulnModelsManager.get() + .then(function() { + $scope.models = vulnModelsManager.models; + $scope.loaded_models = true; + }); + vulnModelsManager.getSize() + .then(function() { + $scope.totalModels = vulnModelsManager.totalNumberOfModels; }); $scope.$watch(function() { diff --git a/server/www/scripts/vulndb/partials/modalCreateDB.html b/server/www/scripts/vulndb/partials/modalCreateDB.html deleted file mode 100644 index d1ef5983ce2..00000000000 --- a/server/www/scripts/vulndb/partials/modalCreateDB.html +++ /dev/null @@ -1,12 +0,0 @@ - - - - -
    -

    Oops!

    -
    -
    -

    {{message}}

    - - -
    diff --git a/server/www/scripts/vulndb/partials/vulndb.html b/server/www/scripts/vulndb/partials/vulndb.html index a72cd9344cb..3821a4c4a0a 100644 --- a/server/www/scripts/vulndb/partials/vulndb.html +++ b/server/www/scripts/vulndb/partials/vulndb.html @@ -3,7 +3,7 @@
    -
    +

    Vulnerability Templates

    From ea13d6f9f86135d0566677b6741b0b9a5f7c6d16 Mon Sep 17 00:00:00 2001 From: f-amato Date: Fri, 13 Oct 2017 20:04:56 -0300 Subject: [PATCH 0335/1506] Adding support for credential API --- server/api/modules/credentials.py | 58 +++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/server/api/modules/credentials.py b/server/api/modules/credentials.py index 26aad9b9d64..a271236258c 100644 --- a/server/api/modules/credentials.py +++ b/server/api/modules/credentials.py @@ -12,6 +12,64 @@ credentials_api = Blueprint('credentials_api', __name__) +class CredentialSchema(AutoSchema): + _id = fields.Integer(dump_only=True, attribute='id') + _rev = fields.String(default='', dump_only=True) + metadata = fields.Method('get_metadata') + owned = fields.Boolean(default=False) + owner = fields.String(dump_only=True, attribute='creator.username') + username = fields.String(default='') + password = fields.String(default='') + description = fields.String(default='') + + parent = fields.Method('get_parent') + + def get_parent(self, obj): + if getattr(obj, 'service', None): + return obj.service.id + if getattr(obj, 'host', None): + return obj.host.id + return + + def get_metadata(self, obj): + return { + "command_id": "e1a042dd0e054c1495e1c01ced856438", + "create_time": time.mktime(obj.create_date.utctimetuple()), + "creator": "Metasploit", + "owner": "", "update_action": 0, + "update_controller_action": "No model controller call", + "update_time": time.mktime(obj.update_date.utctimetuple()), + "update_user": "" + } + + + class Meta: + model = Credential + fields = ('id', '_id', 'status', 'parent', + 'username', 'description', '_rev', + 'owned', 'owner', 'name', 'password', + '_id', 'metadata') + + +class CredentialView(ReadWriteWorkspacedView): + route_base = 'credentials' + model_class = Credential + schema_class = CredentialSchema + + def _envelope_list(self, objects, pagination_metadata=None): + credentials = [] + for credential in objects: + credentials.append({ + 'id': credential['_id'], + 'key': credential['_id'], + 'value': credential + }) + return { + 'credentials': credentials, + } + +CredentialView.register(credentials_api) + @gzipped @credentials_api.route('/ws//credentials', methods=['GET']) From 301fb59f11f2fdc378fa6ab30d0a6e7581ac4103 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 17 Oct 2017 12:01:17 -0300 Subject: [PATCH 0336/1506] Add missing imports --- server/api/modules/credentials.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/api/modules/credentials.py b/server/api/modules/credentials.py index a271236258c..28768118abc 100644 --- a/server/api/modules/credentials.py +++ b/server/api/modules/credentials.py @@ -1,10 +1,14 @@ # Faraday Penetration Test IDE # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information +import time import flask from flask import Blueprint +from marshmallow import fields +from server.api.base import AutoSchema, ReadWriteWorkspacedView +from server.models import Credential from server.utils.logger import get_logger from server.utils.web import gzipped, validate_workspace, filter_request_args from server.dao.credential import CredentialDAO From 59c106a680d43058ec1be5cb82d306c8dd5ea203 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 17 Oct 2017 12:05:32 -0300 Subject: [PATCH 0337/1506] Remove couchdb report importer call Since we're not using CouchDB anymore, remove this call when the server starts --- faraday-server.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/faraday-server.py b/faraday-server.py index 68bdc2af36d..8faeca4af05 100755 --- a/faraday-server.py +++ b/faraday-server.py @@ -48,9 +48,6 @@ def setup_environment(check_deps=False): # Web configuration file generation server.config.gen_web_config() - # Reports DB creation - server.couchdb.push_reports() - def stop_server(): if not daemonize.stop_server(): From f61044c7d4107e971bbe9ffee438b1911d222fa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 17 Oct 2017 12:10:07 -0300 Subject: [PATCH 0338/1506] Add missing test dependency "responses" --- requirements_dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_dev.txt b/requirements_dev.txt index a8b7439b29a..0c1f9128949 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,3 +1,4 @@ factory-boy<=2.8.1 pytest pytest-factoryboy +responses From 4c2e61602fe18f062946dea0812ed4d92644aeab Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 17 Oct 2017 12:15:29 -0300 Subject: [PATCH 0339/1506] Don't print missing database settings error --- server/app.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/app.py b/server/app.py index c9bc560bfe8..381cf98a418 100644 --- a/server/app.py +++ b/server/app.py @@ -1,6 +1,7 @@ # Faraday Penetration Test IDE # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information +import logging from ConfigParser import NoOptionError import flask @@ -13,6 +14,7 @@ import server.config from server.utils.logger import LOGGING_HANDLERS +logger = logging.getLogger(__name__) def create_app(db_connection_string=None, testing=None): @@ -41,9 +43,9 @@ def create_app(db_connection_string=None, testing=None): try: app.config['SQLALCHEMY_DATABASE_URI'] = db_connection_string or server.config.database.connection_string.strip("'") except AttributeError: - print('Missing [database] section on server.ini. Please configure the database before running the server.') + logger.info('Missing [database] section on server.ini. Please configure the database before running the server.') except NoOptionError: - print('Missing connection_string on [database] section on server.ini. Please configure the database before running the server.') + logger.info('Missing connection_string on [database] section on server.ini. Please configure the database before running the server.') from server.models import db db.init_app(app) From 6a7fbe0d70f41a95edaca3be41ede2cfeb65849b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 17 Oct 2017 12:19:21 -0300 Subject: [PATCH 0340/1506] Add a message to let the user known what the script will do --- server/commands/initdb.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index b32905ee758..639d1c0580e 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -94,6 +94,7 @@ def _configure_postgres(self, psql_log_file): we return username and password and those values will be saved in the config file. """ username_default = 'faraday_db_admin' + print('This script will {blue} create a new postgres user {white} and {blue} save faraday-server settings {white}(server.ini). '.format(blue=Fore.BLUE, white=Fore.WHITE)) username = raw_input('Please enter the {blue} database user {white} (press enter to use "{0}"): '.format(username_default, blue=Fore.BLUE, white=Fore.WHITE)) or username_default postgres_command = ['sudo', '-u', 'postgres'] password = None From 0e15bbf6e848b719452bf313cb278bfc9ee11cab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 17 Oct 2017 12:54:30 -0300 Subject: [PATCH 0341/1506] Refactor generic tests - Override _envelope_list method on all list tests, not only on paginated ones - Create mixin for factories that need either a host or a service, but not both (like stantdard vulns and credentials) - Add ReadOnlyAPITests for testing read-only testing views --- test_cases/factories.py | 25 ++++++++++++++++++++----- test_cases/test_api_pagination.py | 9 +-------- test_cases/test_api_workspaced_base.py | 18 +++++++++++++++++- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/test_cases/factories.py b/test_cases/factories.py index 7fc3c1a5728..e414fad6739 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -166,10 +166,15 @@ class VulnerabilityGenericFactory(WorkspaceObjectFactory): severity = FuzzyChoice(['critical', 'high']) -class VulnerabilityFactory(VulnerabilityGenericFactory): +class HasParentHostOrService(object): + """ + Mixins for objects that must have either a host or a service, + but ont both, as a parent. - host = factory.SubFactory(HostFactory) - service = factory.SubFactory(ServiceFactory) + By default it randomly select one of them and set the other to + None, but this behavior can be modified as with other factory + fields + """ @classmethod def attributes(cls, create=False, extra=None): @@ -179,12 +184,15 @@ def attributes(cls, create=False, extra=None): raise ValueError('You should pass both service and host and ' 'set one of them to None to prevent random ' 'stuff to happen') - return super(VulnerabilityFactory, cls).attributes(create, extra) + return super(HasParentHostOrService, cls).attributes(create, extra) @classmethod def _after_postgeneration(cls, obj, create, results=None): - super(VulnerabilityFactory, cls)._after_postgeneration( + super(HasParentHostOrService, cls)._after_postgeneration( obj, create, results) + if isinstance(obj, dict): + # This happens when built with build_dict + return if obj.host and obj.service: # Setting both service and host to a vuln is not allowed. # This will pick one of them randomly. @@ -194,6 +202,13 @@ def _after_postgeneration(cls, obj, create, results=None): else: obj.service = None + +class VulnerabilityFactory(HasParentHostOrService, + VulnerabilityGenericFactory): + + host = factory.SubFactory(HostFactory) + service = factory.SubFactory(ServiceFactory) + class Meta: model = Vulnerability sqlalchemy_session = db.session diff --git a/test_cases/test_api_pagination.py b/test_cases/test_api_pagination.py index a4459fc60ba..cd60aa2e387 100644 --- a/test_cases/test_api_pagination.py +++ b/test_cases/test_api_pagination.py @@ -9,7 +9,6 @@ def with_0_and_n_objects(n=10): return pytest.mark.parametrize('object_count', [0, n]) class PaginationTestsMixin: - view_class = None # Must be overriden @pytest.fixture def delete_previously_created_objects(self, session): @@ -17,15 +16,9 @@ def delete_previously_created_objects(self, session): session.delete(obj) session.commit() - @pytest.fixture - def custom_envelope(self, monkeypatch): - def _envelope_list(self, objects, pagination_metadata=None): - return {"data": objects} - monkeypatch.setattr(self.view_class, '_envelope_list', _envelope_list) - @pytest.fixture def pagination_test_logic(self, delete_previously_created_objects, - custom_envelope): + mock_envelope_list): # Load this two fixtures pass diff --git a/test_cases/test_api_workspaced_base.py b/test_cases/test_api_workspaced_base.py index 9a9ad5d30f4..16d971ccdff 100644 --- a/test_cases/test_api_workspaced_base.py +++ b/test_cases/test_api_workspaced_base.py @@ -51,7 +51,17 @@ def url(self, obj=None, workspace=None): class ListTestsMixin: + view_class = None # Must be overriden + @pytest.fixture + def mock_envelope_list(self, monkeypatch): + assert self.view_class is not None, 'You must define view_class ' \ + 'in order to use ListTestsMixin or PaginationTestsMixin' + def _envelope_list(self, objects, pagination_metadata=None): + return {"data": objects} + monkeypatch.setattr(self.view_class, '_envelope_list', _envelope_list) + + @pytest.mark.usefixtures('mock_envelope_list') def test_list_retrieves_all_items_from_workspace(self, test_client, second_workspace, session): @@ -59,7 +69,7 @@ def test_list_retrieves_all_items_from_workspace(self, test_client, session.commit() res = test_client.get(self.url()) assert res.status_code == 200 - assert len(res.json['rows']) == OBJECT_COUNT + assert len(res.json['data']) == OBJECT_COUNT class RetrieveTestsMixin: @@ -182,3 +192,9 @@ class ReadWriteTestsMixin(ListTestsMixin, class ReadWriteAPITests(ReadWriteTestsMixin, GenericAPITest): pass + + +class ReadOnlyAPITests(ListTestsMixin, + RetrieveTestsMixin, + GenericAPITest): + pass From 80d62272ae320e5af388749c02613cf82a32505c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 17 Oct 2017 12:58:28 -0300 Subject: [PATCH 0342/1506] Fix credentials API and factory --- server/api/modules/credentials.py | 8 ++++---- test_cases/factories.py | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/server/api/modules/credentials.py b/server/api/modules/credentials.py index 28768118abc..0dfa85c4e07 100644 --- a/server/api/modules/credentials.py +++ b/server/api/modules/credentials.py @@ -49,10 +49,10 @@ def get_metadata(self, obj): class Meta: model = Credential - fields = ('id', '_id', 'status', 'parent', - 'username', 'description', '_rev', - 'owned', 'owner', 'name', 'password', - '_id', 'metadata') + fields = ('id', '_id', "_rev", 'parent', + 'username', 'description', + 'name', 'password', + 'metadata') class CredentialView(ReadWriteWorkspacedView): diff --git a/test_cases/factories.py b/test_cases/factories.py index e414fad6739..ef97ee511fd 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -233,7 +233,9 @@ class Meta: sqlalchemy_session = db.session -class CredentialFactory(WorkspaceObjectFactory): +class CredentialFactory(HasParentHostOrService, WorkspaceObjectFactory): + host = factory.SubFactory(HostFactory) + service = factory.SubFactory(ServiceFactory) username = FuzzyText() password = FuzzyText() From c983bc749c36e6b91a02eb7586abea142e87d954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 17 Oct 2017 13:10:54 -0300 Subject: [PATCH 0343/1506] Add credentials api tests and fix commands and services ones Fix some copy-pasted code and make them inherit from ReadOnlyAPITest to do basic, generic list and retrieve endpoint testing --- test_cases/test_api_commands.py | 8 +++----- test_cases/test_api_credentials.py | 14 ++++++++++++++ test_cases/test_api_services.py | 14 +++++++------- 3 files changed, 24 insertions(+), 12 deletions(-) create mode 100644 test_cases/test_api_credentials.py diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index 8db93a3eba3..6e6faf2ed24 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -3,7 +3,7 @@ import pytest from test_cases import factories -from test_api_workspaced_base import ListTestsMixin, API_PREFIX, GenericAPITest +from test_api_workspaced_base import API_PREFIX, ReadOnlyAPITests from server.models import ( Command, Workspace, @@ -13,12 +13,10 @@ @pytest.mark.usefixtures('logged_user') -class TestListCommandView(GenericAPITest): +class TestListCommandView(ReadOnlyAPITests): model = Command factory = factories.CommandFactory api_endpoint = 'commands' - unique_fields = ['ip'] - update_fields = ['ip', 'description', 'os'] view_class = CommandView def test_(self, test_client, second_workspace, session): @@ -40,4 +38,4 @@ def test_(self, test_client, second_workspace, session): u'user', u'workspace' ] - assert set(object_properties) == set(command['value'].keys()) \ No newline at end of file + assert set(object_properties) == set(command['value'].keys()) diff --git a/test_cases/test_api_credentials.py b/test_cases/test_api_credentials.py new file mode 100644 index 00000000000..3495dbed32a --- /dev/null +++ b/test_cases/test_api_credentials.py @@ -0,0 +1,14 @@ +from test_cases import factories +from test_api_workspaced_base import ( + ReadOnlyAPITests, +) +from server.api.modules.credentials import CredentialView +from server.models import Credential + + +class TestCredentialsAPIGeneric(ReadOnlyAPITests): + model = Credential + factory = factories.CredentialFactory + view_class = CredentialView + api_endpoint = 'credentials' + update_fields = ['username', 'password'] diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index 6bae7bc7155..816aec0c3a0 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -3,9 +3,9 @@ import pytest -from server.api.modules.vulns import VulnerabilityView +from server.api.modules.services import ServiceView from test_cases import factories -from test_api_workspaced_base import ListTestsMixin, API_PREFIX, GenericAPITest +from test_api_workspaced_base import API_PREFIX, ReadOnlyAPITests from server.models import ( Service ) @@ -14,13 +14,13 @@ @pytest.mark.usefixtures('logged_user') -class TestListServiceView(GenericAPITest): +class TestListServiceView(ReadOnlyAPITests): model = Service factory = factories.ServiceFactory api_endpoint = 'services' #unique_fields = ['ip'] #update_fields = ['ip', 'description', 'os'] - view_class = VulnerabilityView + view_class = ServiceView def test_(self, test_client, second_workspace, session): self.factory.create(workspace=second_workspace) @@ -28,8 +28,8 @@ def test_(self, test_client, second_workspace, session): res = test_client.get(self.url()) assert res.status_code == 200 assert 'services' in res.json - for vuln in res.json['services']: - assert set([u'id', u'key', u'value']) == set(vuln.keys()) + for service in res.json['services']: + assert set([u'id', u'key', u'value']) == set(service.keys()) object_properties = [ u'status', u'protocol', @@ -44,5 +44,5 @@ def test_(self, test_client, second_workspace, session): u'metadata' ] expected = set(object_properties) - result = set(vuln['value'].keys()) + result = set(service['value'].keys()) assert expected <= result From 087a766c37f2752dcda7ae77481cd00004737b11 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 17 Oct 2017 14:44:27 -0300 Subject: [PATCH 0344/1506] Add error handling specific for port in use --- faraday-server.py | 2 +- server/web.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/faraday-server.py b/faraday-server.py index 8faeca4af05..d86c08dcce2 100755 --- a/faraday-server.py +++ b/faraday-server.py @@ -73,8 +73,8 @@ def run_server(args): web_server = server.web.WebServer(enable_ssl=args.ssl) daemonize.create_pid_file() - logger.info('Faraday Server is ready') web_server.run() + logger.info('Faraday Server is ready') def main(): diff --git a/server/web.py b/server/web.py index 6fc62229707..a20dd8581bb 100644 --- a/server/web.py +++ b/server/web.py @@ -27,7 +27,7 @@ class WebServer(object): WEB_UI_LOCAL_PATH = os.path.join(server.config.FARADAY_BASE, 'server/www') def __init__(self, enable_ssl=False): - logger.info('Starting server at port {0} with bind address {1}. SSL {2}'.format( + logger.info('Starting web server at port {0} with bind address {1}. SSL {2}'.format( server.config.faraday_server.port, server.config.faraday_server.bind_address, enable_ssl)) @@ -76,7 +76,10 @@ def run(self): self.__listen_func( self.__listen_port, site, interface=self.__bind_address) + reactor.run() + except error.CannotListenError as e: + logger.error(str(e)) + sys.exit(1) except Exception as e: logger.error('Something went wrong when trying to setup the Web UI') - reactor.run() - + sys.exit(1) From 275a6d8a461a15300ac4d5be3067181a862db585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 17 Oct 2017 17:01:28 -0300 Subject: [PATCH 0345/1506] Fix style for info vulns We changed "info" for "informational" in the backend --- server/www/estilos.css | 1 + 1 file changed, 1 insertion(+) diff --git a/server/www/estilos.css b/server/www/estilos.css index 7154c044b96..afda0a58629 100644 --- a/server/www/estilos.css +++ b/server/www/estilos.css @@ -202,6 +202,7 @@ article.untercio{ .fondo-gris1, .fondo-unclassified {background-color: #999 !important;} .fondo-blanco {background-color: #fff !important;} .fondo-info, .fondo-risk-accepted {background-color: #2e97bd !important;} +.fondo-informational, .fondo-risk-accepted {background-color: #2e97bd !important;} .fondo-naranja, .fondo-medium, .fondo-med, .fondo-re-opened {background-color: #DFBF35 !important;} .fondo-amarillo, .fondo-low, .fondo-closed , .fondo-created{background-color: #A1CE31 !important;} .fondo-violeta {background-color: #932ebe !important;} From cfba1df11effdfb38a4e376fe34301d59fde598e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 17 Oct 2017 12:07:02 -0300 Subject: [PATCH 0346/1506] Warn the user about missing deps --- manage.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/manage.py b/manage.py index 26cb0b41f14..1eb259507bb 100755 --- a/manage.py +++ b/manage.py @@ -1,7 +1,9 @@ #!/usr/bin/env python - -from flask_script import Manager +try: + from flask_script import Manager +except ImportError: + print('Flask-Script missing. Install requirements with "pip install -r requirements_server.txt') from server.web import app from server.importer import ImportCouchDB From 7e7bf7a5788fec2fb19d57e69e52af8e580d51d7 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 17 Oct 2017 17:32:34 -0300 Subject: [PATCH 0347/1506] Add vuln creation --- server/api/modules/credentials.py | 1 + server/api/modules/vulns.py | 171 ++++++++++++++---- .../www/scripts/commons/providers/server.js | 2 +- .../statusReport/controllers/modalNew.js | 8 +- server/www/scripts/vulns/providers/vuln.js | 9 +- server/www/scripts/vulns/providers/vulns.js | 6 +- test_cases/test_api_vulnerability.py | 49 ++++- 7 files changed, 198 insertions(+), 48 deletions(-) diff --git a/server/api/modules/credentials.py b/server/api/modules/credentials.py index 0dfa85c4e07..e673c39432c 100644 --- a/server/api/modules/credentials.py +++ b/server/api/modules/credentials.py @@ -16,6 +16,7 @@ credentials_api = Blueprint('credentials_api', __name__) + class CredentialSchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') _rev = fields.String(default='', dump_only=True) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 50d88ce5b02..4d4f27af470 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -6,7 +6,7 @@ from flask import request, jsonify, abort from flask import Blueprint -from marshmallow import fields +from marshmallow import fields, post_load from server.api.base import ( AutoSchema, @@ -15,10 +15,12 @@ ) from server.models import ( db, - VulnerabilityGeneric, + Tag, TagObject, - Tag -) + Vulnerability, + VulnerabilityWeb, + VulnerabilityGeneric, + Host, Service) from server.utils.logger import get_logger from server.utils.web import ( gzipped, @@ -34,50 +36,56 @@ logger = logging.getLogger(__name__) -class VulnerabilityGenericSchema(AutoSchema): +class VulnerabilitySchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') - website = fields.String(default='') + _rev = fields.String(default='') + _attachments = fields.Method(load_only=True, deserialize='load_attachments') owned = fields.Boolean(dump_only=True, default=False) owner = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator') - impact = fields.Method('get_impact') + impact = fields.Method(deserialize='load_impact') policyviolations = PrimaryKeyRelatedField('name', many=True, attribute='policy_violations') - method = fields.String(default='') - params = fields.String(default='') + desc = fields.String(dump_only=True, attribute='description') refs = PrimaryKeyRelatedField('name', many=True, attribute='references') issuetracker = fields.Method('get_issuetracker') - parent = fields.Method('get_parent') + parent = fields.Method('get_parent', deserialize='load_parent') + parent_type = fields.Method('get_parent_type') tags = fields.Method('get_tags') easeofresolution = fields.String(dump_only=True, attribute='ease_of_resolution') hostnames = PrimaryKeyRelatedField('name', many=True) - pname = fields.String(dump_only=True, attribute='parameter_name', default='') - path = fields.String(default='') - response = fields.String(default='') - desc = fields.String(dump_only=True, attribute='description') - obj_id = fields.String(dump_only=True, attribute='id') - request = fields.String(default='') - _attachments = fields.Method('get_attachments') - target = fields.String(default='') # TODO: review this attribute - query = fields.String(dump_only=True, attribute='query_string', default='') metadata = fields.Method('get_metadata') service = fields.Nested(ServiceSchema(only=[ '_id', 'ports', 'status', 'protocol', 'name', 'version', 'summary' ]), dump_only=True) host = fields.Integer(dump_only=True, attribute='host_id') + status = fields.Method(attribute='status', deserialize='load_status') + type = fields.Method('get_type', deserialize='load_type') + obj_id = fields.String(dump_only=True, attribute='id') + _attachments = fields.Method('get_attachments') + target = fields.String(default='') # TODO: review this attribute class Meta: - model = VulnerabilityGeneric + model = Vulnerability fields = ( '_id', 'status', - 'website', 'issuetracker', 'description', 'parent', + 'issuetracker', 'description', 'parent', 'parent_type', 'tags', 'severity', '_rev', 'easeofresolution', 'owned', - 'hostnames', 'pname', 'query', 'owner', - 'path', 'data', 'response', 'refs', + 'hostnames', 'owner', + 'data', 'refs', 'desc', 'impact', 'confirmed', 'name', 'service', 'obj_id', 'type', 'policyviolations', - 'request', '_attachments', 'params', - 'target', 'resolution', 'method', 'metadata') + '_attachments', + 'target', 'resolution', 'metadata') + + def get_type(self, obj): + return obj.__class__.__name__ + + def get_parent_type(self, obj): + if obj.host_id: + return 'Host' + if obj.service_id: + return 'Service' def get_metadata(self, obj): return { @@ -95,6 +103,7 @@ def get_attachments(self, obj): return [] def get_hostnames(self, obj): + # TODO: improve performance here if obj.host: return [hostname.name for hostname in obj.host.hostnames] if obj.service: @@ -103,12 +112,14 @@ def get_hostnames(self, obj): return [] def get_tags(self, obj): + # TODO: improve performance here return [tag.name for tag in db.session.query(TagObject, Tag).filter_by( object_type=obj.__class__.__name__, object_id=obj.id ).all()] def get_parent(self, obj): + # TODO: move this to models and use duck typing? if getattr(obj, 'service', None): return obj.service.id if getattr(obj, 'host', None): @@ -118,19 +129,111 @@ def get_parent(self, obj): def get_issuetracker(self, obj): return {} - def get_impact(self, obj): - return { - 'accountability': obj.impact_accountability, - 'availability': obj.impact_availability, - 'confidentiality': obj.impact_confidentiality, - 'integrity': obj.impact_integrity - } + def load_impact(self, value): + pass + + def load_status(self, value): + if value == 'opened': + return 'open' + + def load_type(self, value): + if value == 'Vulnerability': + return 'vulnerability' + if value == 'VulnerabilityWeb': + return 'vulnerability_web' + + def load_parent(self, value): + self.parent_id = value + return value + + def load_attachments(self, value): + # TODO: implement attachments + pass + + @post_load + def clean_up(self, data): + data.pop('_attachments') + return data + + @post_load + def set_impact(self, data): + impact = data.pop('impact') + if impact: + pass + return data + + + @post_load + def set_parent(self, data): + # schema guarantees that parent_type exists. + parent_class = None + parent_type = data.pop('parent_type') + parent_id = data.pop('parent') + if parent_type == 'Host': + parent_class = Host + parent_field = 'host_id' + if parent_type == 'Service': + parent_class = Service + parent_field = 'service_id' + if not parent_class: + raise Exception('Bad data') + + parent = db.session.query(parent_class).filter_by(id=parent_id).first() + if not parent: + raise Exception('Parent not found') + data[parent_field] = parent.id + return data + + +class VulnerabilityWebSchema(VulnerabilitySchema): + + method = fields.String(default='') + params = fields.String(default='') + pname = fields.String(dump_only=True, attribute='parameter_name', default='') + path = fields.String(default='') + response = fields.String(default='') + request = fields.String(default='') + website = fields.String(default='') + query = fields.String(dump_only=True, attribute='query_string', default='') + + class Meta: + model = VulnerabilityWeb + fields = ( + '_id', 'status', 'parent_type', + 'website', 'issuetracker', 'description', 'parent', + 'tags', 'severity', '_rev', 'easeofresolution', 'owned', + 'hostnames', 'pname', 'query', 'owner', + 'path', 'data', 'response', 'refs', + 'desc', 'impact', 'confirmed', 'name', + 'service', 'obj_id', 'type', 'policyviolations', + 'request', '_attachments', 'params', + 'target', 'resolution', 'method', 'metadata') class VulnerabilityView(PaginatedMixin, ReadWriteWorkspacedView): route_base = 'vulns' - model_class = VulnerabilityGeneric - schema_class = VulnerabilityGenericSchema + + model_class_dict = { + 'Vulnerability': Vulnerability, + 'VulnerabilityWeb': VulnerabilityWeb, + 'VulnerabilityGeneric': VulnerabilityGeneric, + } + schema_class = { + 'Vulnerability': VulnerabilitySchema, + 'VulnerabilityWeb': VulnerabilityWebSchema + } + + @property + def model_class(self): + if request.method == 'POST': + return self.model_class_dict[request.json['type']] + return self.model_class_dict['VulnerabilityGeneric'] + + def _get_schema_class(self): + assert self.schema_class is not None, "You must define schema_class" + if request.method == 'POST': + return self.schema_class[request.json['type']] + return self.schema_class['VulnerabilityWeb'] def _envelope_list(self, objects, pagination_metadata=None): vulns = [] diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index 242d1d78d00..98b128683f4 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -114,7 +114,7 @@ angular.module("faradayApp") if (typeof vuln.data === "undefined") {vuln.data = ""}; if (typeof vuln.severity === "undefined") {vuln.severity = "info"}; if (typeof vuln.resolution === "undefined") {vuln.resolution = ""}; - return createOrUpdate(wsName, vuln._id, vuln); + return createOrUpdate(wsName, vuln._id, vuln, 'vulns'); } var modVulnWeb = function(createOrUpdate, wsName, vulnWeb) { diff --git a/server/www/scripts/statusReport/controllers/modalNew.js b/server/www/scripts/statusReport/controllers/modalNew.js index df9e4647382..47b1b2f8edb 100644 --- a/server/www/scripts/statusReport/controllers/modalNew.js +++ b/server/www/scripts/statusReport/controllers/modalNew.js @@ -75,7 +75,7 @@ angular.module('faradayApp') name: "", owned: false, params: "", - parents: [], + parents: [], // a tuple with (parent_id, parent_type) path: "", pname: "", policyviolations: [], @@ -139,7 +139,11 @@ angular.module('faradayApp') var parents = vm.data.parents; vm.data.parents = []; parents.forEach(function(parent) { - vm.data.parents.push(parent._id); + var parent_type = "Service"; + if (Host.prototype.isPrototypeOf(vm.data.parents[0])) { + parent_type = "Host"; + } + vm.data.parents.push({parent_id: parent._id, type:parent_type}); }); $modalInstance.close(vm.data); diff --git a/server/www/scripts/vulns/providers/vuln.js b/server/www/scripts/vulns/providers/vuln.js index c7be5144c42..50613554d68 100644 --- a/server/www/scripts/vulns/providers/vuln.js +++ b/server/www/scripts/vulns/providers/vuln.js @@ -100,8 +100,7 @@ angular.module('faradayApp') }, _update: function(vuln, data) { var deferred = $q.defer(), - self = this, - url = BASEURL + vuln.ws + "/" + vuln._id; + self = this; var now = new Date(), date = now.getTime(); @@ -137,7 +136,6 @@ angular.module('faradayApp') self._save(vuln, true) .then(function(response) { self.set(self.ws, vuln); - self._rev = response.rev; deferred.resolve(); }, function() { deferred.reject(); @@ -168,7 +166,6 @@ angular.module('faradayApp') self = this, vuln = {}; - vuln._id = self._id; vuln.metadata = self.metadata; vuln.obj_id = self.obj_id; vuln.owner = self.owner; @@ -196,13 +193,11 @@ angular.module('faradayApp') save: function() { var deferred = $q.defer(), loadAtt, - self = this, - url = BASEURL + self.ws + "/" + self._id; + self = this; self.populate().then(function(resp) { self._save(resp, false) .then(function(data) { - self._rev = data.rev; deferred.resolve(self); }, function(data, status, headers, config) { deferred.reject(status); diff --git a/server/www/scripts/vulns/providers/vulns.js b/server/www/scripts/vulns/providers/vulns.js index 3de89b6045d..4166c267c13 100644 --- a/server/www/scripts/vulns/providers/vulns.js +++ b/server/www/scripts/vulns/providers/vulns.js @@ -14,8 +14,10 @@ angular.module('faradayApp') delete data.parents; - parents.forEach(function(parent_id) { - data.parent = parent_id; + parents.forEach(function(parent) { + // we iterate parents when creating multiple vulns from new vuln modal. + data.parent = parent.parent_id; + data.parent_type = parent.type if(data.type == "Vulnerability") { var vuln = new Vuln(ws, data); diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 6357f3d3799..3cf5c399a84 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1,5 +1,5 @@ #-*- coding: utf8 -*- -"""Tests for many API endpoints that do not depend on workspace_name""" +'''Tests for many API endpoints that do not depend on workspace_name''' import pytest @@ -24,7 +24,7 @@ class TestListVulnerabilityView(GenericAPITest): #update_fields = ['ip', 'description', 'os'] view_class = VulnerabilityView - def test_(self, test_client, second_workspace, session): + def test_backward_json_compatibility(self, test_client, second_workspace, session): self.factory.create(workspace=second_workspace) session.commit() res = test_client.get(self.url()) @@ -113,3 +113,48 @@ def test_hostnames(self, host_with_hostnames, test_client, session): assert isinstance(res.json['hostnames'], list) assert set(res.json['hostnames']) == set(hostname.name for hostname in host_with_hostnames.hostnames) + + def _create_post_data(self, name, vuln_type, parent_id, parent_type, refs, policyviolations): + return {'metadata': {'update_time': 1508254070.211, 'update_user': '', 'update_action': 0, 'creator': 'UI Web', + 'create_time': 1508254070.211, 'update_controller_action': 'UI Web New', 'owner': ''}, + 'obj_id': '5a60af7f01dde6d3acfa8e9d3bef265c361a49d2', + 'owner': '', + 'parent': parent_id, + 'parent_type': parent_type, + 'type': vuln_type, + 'ws': 'airbnb', + 'confirmed': True, + 'data': '', + 'desc': 'fsdfsd', + 'easeofresolution': '', + 'impact': {'accountability': False, 'availability': False, 'confidentiality': False, + 'integrity': False}, + 'name': name, + 'owned': False, + 'policyviolations': policyviolations, + 'refs': refs, + 'resolution': '', + 'severity': 'high', + 'status': 'opened', + '_attachments': {}, + 'description': '', + 'protocol': '', + 'version': ''} + + def test_create_vuln(self, host_with_hostnames, test_client, session): + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data( + name='New vulns', + vuln_type='Vulnerability', + parent_id=host_with_hostnames.id, + parent_type='Host', + refs=[], + policyviolations=[] + ) + ws_name = host_with_hostnames.workspace.name + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) + assert res.status_code == 201 + assert res.json['name'] == 'New vulns' + assert res.json['type'] == 'Vulnerability' + assert res.json['parent'] == host_with_hostnames.id + assert res.json['parent_type'] == 'Host' From 35fe9c1daed757932e3e1cc3d39ae33aebc958e3 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 17 Oct 2017 18:20:01 -0300 Subject: [PATCH 0348/1506] Remove unnecessary import in Web UI The modal to ask the user to create the Database for Vuln Templates isn't needed anymore. --- server/www/index.html | 1 - 1 file changed, 1 deletion(-) diff --git a/server/www/index.html b/server/www/index.html index adf63d51df8..d971911a736 100644 --- a/server/www/index.html +++ b/server/www/index.html @@ -175,7 +175,6 @@ - From 3b6eb6b9ce5b82ab24bcb08b9cf02565990a025c Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 17 Oct 2017 18:21:09 -0300 Subject: [PATCH 0349/1506] Remove Vuln Templates DB name from config --- server/www/scripts/config/services/config.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/www/scripts/config/services/config.js b/server/www/scripts/config/services/config.js index 741e7c265f2..99ea4fc8463 100644 --- a/server/www/scripts/config/services/config.js +++ b/server/www/scripts/config/services/config.js @@ -9,13 +9,11 @@ angular.module('faradayApp') .then(function(conf) { configSrv.faraday_version = conf.data.ver; configSrv.license_db = conf.data.lic_db; - configSrv.vulnModelsDB = conf.data.vuln_model_db }); configSrv = { faraday_version: null, license_db: null, - vulnModelsDB: null, promise: p } From 62fda8e607bc02bb295fe72863186b75c3a9b6c2 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 17 Oct 2017 18:23:15 -0300 Subject: [PATCH 0350/1506] Remove unnecessary dependency --- server/www/scripts/vulns/providers/vulns.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/vulns/providers/vulns.js b/server/www/scripts/vulns/providers/vulns.js index 3de89b6045d..d732829fdb3 100644 --- a/server/www/scripts/vulns/providers/vulns.js +++ b/server/www/scripts/vulns/providers/vulns.js @@ -4,8 +4,8 @@ angular.module('faradayApp') .factory('vulnsManager', - ['Vuln', 'WebVuln', 'BASEURL', '$q', 'ServerAPI', 'commonsFact', - function(Vuln, WebVuln, BASEURL, $q, ServerAPI, commonsFact) { + ['Vuln', 'WebVuln', '$q', 'ServerAPI', 'commonsFact', + function(Vuln, WebVuln, $q, ServerAPI, commonsFact) { var vulnsManager = {}; vulnsManager.createVuln = function(ws, data) { From 3b71324a63f42a0cba365c40ac736c66256ed4b7 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 17 Oct 2017 18:23:59 -0300 Subject: [PATCH 0351/1506] Add dummy API module for Vuln Templates Response is a JSON that *almost* makes sense. NO actual data is returned. --- server/api/modules/vulnerability_template.py | 45 ++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 server/api/modules/vulnerability_template.py diff --git a/server/api/modules/vulnerability_template.py b/server/api/modules/vulnerability_template.py new file mode 100644 index 00000000000..544272d89f8 --- /dev/null +++ b/server/api/modules/vulnerability_template.py @@ -0,0 +1,45 @@ +# Faraday Penetration Test IDE +# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) +# See the file 'doc/LICENSE' for the license information +import time + +from flask import Blueprint +from marshmallow import fields + +import server.utils.logger + +from server.api.base import AutoSchema, ReadWriteView +from server.models import VulnerabilityTemplate + +vulnerability_template_api = Blueprint('vulnerability_template_api', __name__) +logger = server.utils.logger.get_logger(__name__) + + +class VulnerabilityTemplateSchema(AutoSchema): + _id = fields.Integer(dump_only=True, attribute='id') + _rev = fields.String(default='', dump_only=True) + + + class Meta: + model = VulnerabilityTemplate + fields = ('id', '_id', '_rev') + + +class VulnerabilityTemplateView(ReadWriteView): + route_base = 'vulnerability_template' + model_class = VulnerabilityTemplate + schema_class = VulnerabilityTemplateSchema + + def _envelope_list(self, objects, pagination_metadata=None): + vuln_tpls = [] + for template in objects: + vuln_tpls.append({ + 'id': template['_id'], + 'key': template['_id'], + 'value': template + }) + return { + 'rows': vuln_tpls, + } + +VulnerabilityTemplateView.register(vulnerability_template_api) From f3a38a21e465d821484734f60512c59cdab25f46 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 17 Oct 2017 18:25:32 -0300 Subject: [PATCH 0352/1506] Add functions to ServerAPI factory in the Web UI These functions allow to generate the URL for a GET request in the case that the objects don't belong to a Workspace. In previous versions this was done executing the request in the Provider directly, without using the ServerAPI factory. --- .../www/scripts/commons/providers/server.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index 242d1d78d00..8ffc4a47db7 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -25,6 +25,15 @@ angular.module("faradayApp") return APIURL + "ws/" + wsName + objectName + "/"; }; + // created GET URL for objects that do not belong to a workspace + var createNonWorkspacedGetUrl = function(objectName, objectId) { + if (typeof objectId == 'string' || typeof objectId == "number") { + objectName = objectName + "/" + objectId; + } + + return APIURL + objectName + "/"; + }; + var createNewGetUrl = function(wsName, objectId, objectType) { return APIURL + "ws/" + wsName + "/" + objectType + "/" + objectId; } @@ -182,6 +191,16 @@ angular.module("faradayApp") return get(getUrl, data); } + ServerAPI.getVulnerabilityTemplate = function(objId) { + var url = createNonWorkspacedGetUrl('vulnerability_template', objId); + return get(url); + } + + ServerAPI.getVulnerabilityTemplates = function(data) { + var url = createNonWorkspacedGetUrl('vulnerability_template'); + return get(url); + } + ServerAPI.getService = function(wsName, data, objId) { var getUrl = createGetUrl(wsName, 'services', objId); return get(getUrl); From 52cfbb19455cc4ab3f1eb61858f4a3089a43c8fe Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 17 Oct 2017 18:27:15 -0300 Subject: [PATCH 0353/1506] Fix Web UI components that load Vuln Templates Since the JSON returned by the server is almost empty, the list shows empty fields, but everything seems to be functional. --- server/app.py | 26 +++-- .../scripts/vulndb/providers/vulnModels.js | 103 +++++------------- 2 files changed, 40 insertions(+), 89 deletions(-) diff --git a/server/app.py b/server/app.py index 7255e21fc64..065da42249c 100644 --- a/server/app.py +++ b/server/app.py @@ -64,28 +64,30 @@ def create_app(db_connection_string=None, testing=None): for handler in LOGGING_HANDLERS: app.logger.addHandler(handler) - from server.api.modules.workspaces import workspace_api + from server.modules.info import info_api + from server.api.modules.commandsrun import commandsrun_api + from server.api.modules.credentials import credentials_api from server.api.modules.doc import doc_api - from server.api.modules.vuln_csv import vuln_csv_api - from server.api.modules.vulns import vulns_api from server.api.modules.hosts import host_api from server.api.modules.licenses import license_api - from server.api.modules.commandsrun import commandsrun_api from server.api.modules.services import services_api - from server.api.modules.credentials import credentials_api from server.api.modules.session import session_api - from server.modules.info import info_api - app.register_blueprint(workspace_api) + from server.api.modules.vulns import vulns_api + from server.api.modules.vuln_csv import vuln_csv_api + from server.api.modules.vulnerability_template import vulnerability_template_api + from server.api.modules.workspaces import workspace_api + app.register_blueprint(commandsrun_api) + app.register_blueprint(credentials_api) app.register_blueprint(doc_api) - app.register_blueprint(vuln_csv_api) - app.register_blueprint(vulns_api) app.register_blueprint(host_api) + app.register_blueprint(info_api) app.register_blueprint(license_api) - app.register_blueprint(commandsrun_api) app.register_blueprint(services_api) - app.register_blueprint(credentials_api) - app.register_blueprint(info_api) app.register_blueprint(session_api) + app.register_blueprint(vulns_api) + app.register_blueprint(vuln_csv_api) + app.register_blueprint(vulnerability_template_api) + app.register_blueprint(workspace_api) # We are exposing a RESTful API, so don't redirect a user to a login page in # case of being unauthorized, raise a 403 error instead diff --git a/server/www/scripts/vulndb/providers/vulnModels.js b/server/www/scripts/vulndb/providers/vulnModels.js index 40b7740b34f..64381bb80ae 100644 --- a/server/www/scripts/vulndb/providers/vulnModels.js +++ b/server/www/scripts/vulndb/providers/vulnModels.js @@ -4,54 +4,12 @@ // angular.module('faradayApp'). factory('vulnModelsManager', - ['VulnModel', 'BASEURL', 'configSrv', '$http', '$q', - function(VulnModel, BASEURL, configSrv, $http, $q) { + ['VulnModel', 'BASEURL', '$http', '$q', 'ServerAPI', + function(VulnModel, BASEURL, $http, $q, ServerAPI) { var vulnModelsManager = {}; vulnModelsManager.models = []; vulnModelsManager.totalNumberOfModels = 0; - vulnModelsManager.DBExists = function() { - var deferred = $q.defer(); - var self = this; - - configSrv.promise. - then(function() { - var url = BASEURL + configSrv.vulnModelsDB; - - $http.head(url). - then(function(resp) { - deferred.resolve(true); - }, function(resp) { - deferred.resolve(false); - }); - }, function() { - deferred.reject("Unable to fetch the Vulnerability Models DB name."); - }); - - return deferred.promise; - }; - - vulnModelsManager.createDB = function() { - var deferred = $q.defer(); - var self = this; - - configSrv.promise - .then(function() { - var url = BASEURL + configSrv.vulnModelsDB; - - $http.put(url). - then(function(resp) { - deferred.resolve(true); - }, function(resp) { - deferred.reject(resp); - }); - }, function() { - deferred.reject("Unable to fetch Vulnerability Model DB name."); - }); - - return deferred.promise; - }; - vulnModelsManager.create = function(data, outsider) { if (outsider === undefined) { var outsider = false; }; var deferred = $q.defer(); @@ -104,30 +62,25 @@ angular.module('faradayApp'). var deferred = $q.defer(); var self = this; - configSrv.promise. - then(function() { - var url = BASEURL + configSrv.vulnModelsDB + "/_all_docs?include_docs=true"; - - $http.get(url). - then(function(res) { - var data = res.data; - var vulnModels = []; - - if (data.hasOwnProperty("rows")) { - data.rows.forEach(function(row) { - try { - vulnModels.push(new VulnModel(row.doc)); - } catch(e) { - console.log(e.stack); - } - }); + ServerAPI.getVulnerabilityTemplates() + .then(function(res) { + var data = res.data; + var vulnModels = []; + + if (data.hasOwnProperty("rows")) { + data.rows.forEach(function(row) { + try { + vulnModels.push(new VulnModel(row.doc)); + } catch(e) { + console.log(e.stack); } - - angular.copy(vulnModels, self.models); - deferred.resolve(vulnModels); - }, function(data, status, headers, config) { - deferred.reject("Unable to retrieve vuln models. " + status); }); + } + + angular.copy(vulnModels, self.models); + deferred.resolve(vulnModels); + }, function(data, status, headers, config) { + deferred.reject("Unable to retrieve vuln models. " + status); }); return deferred.promise; @@ -137,17 +90,13 @@ angular.module('faradayApp'). var deferred = $q.defer(); var self = this; - configSrv.promise. - then(function() { - var url = BASEURL + configSrv.vulnModelsDB + "/_all_docs"; - $http.get(url). - then(function(res) { - var data = res.data; - self.updateState(data.total_rows); - deferred.resolve(); - }, function(data, status) { - deferred.reject("Unable to retrieve documents " + status); - }); + ServerAPI.getVulnerabilityTemplates() + .then(function(res) { + var data = res.data; + self.updateState(data.total_rows); + deferred.resolve(); + }, function(data, status) { + deferred.reject("Unable to retrieve documents " + status); }); return deferred.promise; }; From b932b6f0e9a7bd8d4038659fe390657b70d470cb Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 17 Oct 2017 18:38:47 -0300 Subject: [PATCH 0354/1506] Add Test for new vuln web --- server/api/modules/vulns.py | 7 ++--- server/models.py | 12 ++++++++ test_cases/test_api_vulnerability.py | 45 ++++++++++++++++++++++++++-- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 4d4f27af470..de215423ba3 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -82,10 +82,7 @@ def get_type(self, obj): return obj.__class__.__name__ def get_parent_type(self, obj): - if obj.host_id: - return 'Host' - if obj.service_id: - return 'Service' + return obj.parent_type def get_metadata(self, obj): return { @@ -188,7 +185,7 @@ def set_parent(self, data): class VulnerabilityWebSchema(VulnerabilitySchema): method = fields.String(default='') - params = fields.String(default='') + params = fields.String(attribute='parameters', default='') pname = fields.String(dump_only=True, attribute='parameter_name', default='') path = fields.String(default='') response = fields.String(default='') diff --git a/server/models.py b/server/models.py index 73da1e13ff1..bc3aa349bfb 100644 --- a/server/models.py +++ b/server/models.py @@ -367,6 +367,13 @@ def hostnames(self): return self.service.host.hostnames raise ValueError("Vulnerability has no service nor host") + @property + def parent_type(self): + if self.host_id: + return 'Host' + if self.service_id: + return 'Service' + __mapper_args__ = { 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[0] } @@ -393,6 +400,11 @@ def service_id(cls): def service(cls): return relationship('Service', backref='vulnerabilities_web') + @property + def parent_type(self): + if self.service_id: + return 'Service' + @property def hostnames(self): return self.service.host.hostnames diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 3cf5c399a84..1102d5ad8fd 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -13,6 +13,7 @@ ) from server.api.modules.commandsrun import CommandView from server.api.modules.workspaces import WorkspaceView +from test_cases.factories import ServiceFactory @pytest.mark.usefixtures('logged_user') @@ -114,8 +115,8 @@ def test_hostnames(self, host_with_hostnames, test_client, session): assert set(res.json['hostnames']) == set(hostname.name for hostname in host_with_hostnames.hostnames) - def _create_post_data(self, name, vuln_type, parent_id, parent_type, refs, policyviolations): - return {'metadata': {'update_time': 1508254070.211, 'update_user': '', 'update_action': 0, 'creator': 'UI Web', + def _create_post_data_vulnerability(self, name, vuln_type, parent_id, parent_type, refs, policyviolations): + data = {'metadata': {'update_time': 1508254070.211, 'update_user': '', 'update_action': 0, 'creator': 'UI Web', 'create_time': 1508254070.211, 'update_controller_action': 'UI Web New', 'owner': ''}, 'obj_id': '5a60af7f01dde6d3acfa8e9d3bef265c361a49d2', 'owner': '', @@ -140,10 +141,21 @@ def _create_post_data(self, name, vuln_type, parent_id, parent_type, refs, polic 'description': '', 'protocol': '', 'version': ''} + if vuln_type == 'VulnerabilityWeb': + data.update({ + "method": "GET", + "params": "pepe", + "path": "/pepep", + "pname": "pepe", + "query": "queue&dfsa", + "request": "", + "response": "", + "website": "www.pepe.com"}) + return data def test_create_vuln(self, host_with_hostnames, test_client, session): session.commit() # flush host_with_hostnames - raw_data = self._create_post_data( + raw_data = self._create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id=host_with_hostnames.id, @@ -152,9 +164,36 @@ def test_create_vuln(self, host_with_hostnames, test_client, session): policyviolations=[] ) ws_name = host_with_hostnames.workspace.name + vuln_count_previous = session.query(Vulnerability).count() res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) assert res.status_code == 201 + assert vuln_count_previous + 1 == session.query(Vulnerability).count() assert res.json['name'] == 'New vulns' assert res.json['type'] == 'Vulnerability' assert res.json['parent'] == host_with_hostnames.id assert res.json['parent_type'] == 'Host' + + def test_create_vuln_web(self, host_with_hostnames, test_client, session): + service = ServiceFactory.create(host=host_with_hostnames) + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='VulnerabilityWeb', + parent_id=service.id, + parent_type='Service', + refs=[], + policyviolations=[] + ) + ws_name = host_with_hostnames.workspace.name + vuln_count_previous = session.query(Vulnerability).count() + vuln_web_count_previous = session.query(VulnerabilityWeb).count() + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) + assert res.status_code == 201 + assert vuln_web_count_previous + 1 == session.query(VulnerabilityWeb).count() + assert vuln_count_previous == session.query(Vulnerability).count() + assert res.json['name'] == 'New vulns' + assert res.json['type'] == 'VulnerabilityWeb' + assert res.json['parent'] == service.id + assert res.json['parent_type'] == 'Service' + assert res.json['method'] == 'GET' + assert res.json['path'] == '/pepep' From d913493435006dab6dc54bb67209f9d4d435c432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 18 Oct 2017 12:26:50 -0300 Subject: [PATCH 0355/1506] Fix bug importing sevices with not mapped status --- server/importer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/importer.py b/server/importer.py index 5ad11f673a2..2bea18fbabb 100644 --- a/server/importer.py +++ b/server/importer.py @@ -374,9 +374,9 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation 'unknown': 'closed' } couchdb_status = document.get('status', 'open') - if couchdb_status.lower() == 'unkown': - logger.warn('Service with status {0} found! Status will default to closed. Host is {1}'.format(couchdb_status, host.ip)) - service.status = status_mapper[couchdb_status] + if couchdb_status.lower() not in status_mapper: + logger.warn('Service with status {0} found! Status will default to open. Host is {1}'.format(couchdb_status, host.ip)) + service.status = status_mapper.get(couchdb_status, 'open') service.version = document.get('version') service.workspace = workspace From 27da1cea8858ca966531525434c71a6da0c5201b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 18 Oct 2017 12:31:21 -0300 Subject: [PATCH 0356/1506] Add more mappings to vulnerability severity importer. To reduce warnings in the importer, explicitly map "" and "unknown" severities to "unclassified" --- server/importer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/importer.py b/server/importer.py index 2bea18fbabb..e686835346c 100644 --- a/server/importer.py +++ b/server/importer.py @@ -76,6 +76,8 @@ ('low', 'low'), ('info', 'informational'), ('unclassified', 'unclassified'), + ('unknown', 'unclassified'), + ('', 'unclassified'), ]) OBJ_TYPES = [ From 488d24a7531a4141b634c7eab74d988d0ce3434c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 17 Oct 2017 19:19:08 -0300 Subject: [PATCH 0357/1506] Move property parent to models --- server/api/modules/vulns.py | 7 +------ server/models.py | 8 ++++++++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index de215423ba3..ce58454b3fe 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -116,12 +116,7 @@ def get_tags(self, obj): ).all()] def get_parent(self, obj): - # TODO: move this to models and use duck typing? - if getattr(obj, 'service', None): - return obj.service.id - if getattr(obj, 'host', None): - return obj.host.id - return + return obj.parent.id def get_issuetracker(self, obj): return {} diff --git a/server/models.py b/server/models.py index 28770f73017..36bb7d425f8 100644 --- a/server/models.py +++ b/server/models.py @@ -369,6 +369,10 @@ def hostnames(self): return self.service.host.hostnames raise ValueError("Vulnerability has no service nor host") + @property + def parent(self): + return self.host or self.service + @property def parent_type(self): if self.host_id: @@ -407,6 +411,10 @@ def parent_type(self): if self.service_id: return 'Service' + @property + def parent(self): + return self.service + @property def hostnames(self): return self.service.host.hostnames From d589c97531d3c77455c9edb325e58bc4149bbc0c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 18 Oct 2017 14:21:26 -0300 Subject: [PATCH 0358/1506] Fix file field tests --- server/app.py | 12 +++++++++--- test_cases/test_model_fields.py | 15 +++------------ 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/server/app.py b/server/app.py index b004dcc064e..aa5fea334f4 100644 --- a/server/app.py +++ b/server/app.py @@ -68,9 +68,15 @@ def create_app(db_connection_string=None, testing=None): except AttributeError: logger.warn('No storage section or path in the .faraday/server.ini. Setting the default value to .faraday/storage') storage_path = setup_storage_path() - DepotManager.configure('default', { - 'depot.storage_path': storage_path - }) + if not DepotManager.get('default'): + if testing: + DepotManager.configure('default', { + 'depot.storage_path': '/tmp' + }) + else: + DepotManager.configure('default', { + 'depot.storage_path': storage_path + }) if testing: app.config['TESTING'] = testing try: diff --git a/test_cases/test_model_fields.py b/test_cases/test_model_fields.py index 0dc257d1268..a6e91bb7884 100644 --- a/test_cases/test_model_fields.py +++ b/test_cases/test_model_fields.py @@ -1,27 +1,18 @@ import os -import pytest -from depot.manager import DepotManager - from server.fields import FaradayUploadedFile CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) -@pytest.fixture(scope='module') -def setup_depot(request): - DepotManager.configure('default', { - 'depot.storage_path': '/home/lcubo/tmp_images' - }) - -def test_html_content_type_is_not_html(setup_depot): +def test_html_content_type_is_not_html(): with open(os.path.join(CURRENT_PATH, 'data', 'test.html'))as image_data: field = FaradayUploadedFile(image_data.read()) assert field['content_type'] == 'application/octet-stream' assert len(field['files']) == 1 -def test_image_is_detected_correctly(setup_depot): +def test_image_is_detected_correctly(): with open(os.path.join(CURRENT_PATH, 'data', 'faraday.png'))as image_data: field = FaradayUploadedFile(image_data.read()) @@ -31,7 +22,7 @@ def test_image_is_detected_correctly(setup_depot): assert len(field['files']) == 2 -def test_normal_attach_is_not_detected_as_image(setup_depot): +def test_normal_attach_is_not_detected_as_image(): with open(os.path.join(CURRENT_PATH, 'data', 'report_w3af.xml'))as image_data: field = FaradayUploadedFile(image_data.read()) assert field['content_type'] == 'application/octet-stream' From f2b9d6036e457b3d3540fbc94afe9c91d96530f0 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 18 Oct 2017 14:42:25 -0300 Subject: [PATCH 0359/1506] Fix vulnerability backwards compat test. use difference on test to see for human diff --- server/api/modules/vulns.py | 15 +++++++++++++-- test_cases/test_api_vulnerability.py | 8 ++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index ce58454b3fe..392b8cb0bdc 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -43,7 +43,7 @@ class VulnerabilitySchema(AutoSchema): _attachments = fields.Method(load_only=True, deserialize='load_attachments') owned = fields.Boolean(dump_only=True, default=False) owner = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator') - impact = fields.Method(deserialize='load_impact') + impact = fields.Method('get_impact', deserialize='load_impact') policyviolations = PrimaryKeyRelatedField('name', many=True, attribute='policy_violations') desc = fields.String(dump_only=True, attribute='description') @@ -59,7 +59,7 @@ class VulnerabilitySchema(AutoSchema): '_id', 'ports', 'status', 'protocol', 'name', 'version', 'summary' ]), dump_only=True) host = fields.Integer(dump_only=True, attribute='host_id') - status = fields.Method(attribute='status', deserialize='load_status') + status = fields.Method('get_status', deserialize='load_status') type = fields.Method('get_type', deserialize='load_type') obj_id = fields.String(dump_only=True, attribute='id') _attachments = fields.Method('get_attachments') @@ -118,6 +118,17 @@ def get_tags(self, obj): def get_parent(self, obj): return obj.parent.id + def get_status(self, obj): + return obj.status + + def get_impact(self, obj): + return { + 'accountability': obj.impact_accountability, + 'availability': obj.impact_availability, + 'confidentiality': obj.impact_confidentiality, + 'integrity': obj.impact_integrity, + } + def get_issuetracker(self, obj): return {} diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 1102d5ad8fd..75309fe44c2 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -35,7 +35,6 @@ def test_backward_json_compatibility(self, test_client, second_workspace, sessio assert set([u'id', u'key', u'value']) == set(vuln.keys()) object_properties = [ u'status', - u'website', u'issuetracker', u'description', u'parent', @@ -62,16 +61,17 @@ def test_backward_json_compatibility(self, test_client, second_workspace, sessio u'policyviolations', u'request', u'_attachments', - u'params', u'target', u'_id', u'resolution', u'method', - u'metadata' + u'metadata', + u'website', + u'params', ] expected = set(object_properties) result = set(vuln['value'].keys()) - assert expected <= result + assert expected - result == set() def test_handles_vuln_with_no_creator(self, workspace, From 926c1d83913f232e5b3936350670dc14e30343f0 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 18 Oct 2017 16:04:57 -0300 Subject: [PATCH 0360/1506] Add attachment support for vulns --- server/api/modules/vulns.py | 137 +++++++++++++-------------- test_cases/test_api_vulnerability.py | 55 +++++++++-- 2 files changed, 116 insertions(+), 76 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 392b8cb0bdc..040c8dfe302 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -3,16 +3,21 @@ # See the file 'doc/LICENSE' for the license information import time import logging +from base64 import b64encode, b64decode +import ntpath +import os from flask import request, jsonify, abort from flask import Blueprint -from marshmallow import fields, post_load +from marshmallow import fields, post_load, ValidationError +from depot.manager import DepotManager from server.api.base import ( AutoSchema, PaginatedMixin, ReadWriteWorkspacedView, ) +from server.fields import FaradayUploadedFile from server.models import ( db, Tag, @@ -20,7 +25,8 @@ Vulnerability, VulnerabilityWeb, VulnerabilityGeneric, - Host, Service) + Host, Service, File) +from server.utils.database import get_or_create from server.utils.logger import get_logger from server.utils.web import ( gzipped, @@ -29,18 +35,38 @@ filter_request_args ) from server.api.modules.services import ServiceSchema -from server.schemas import PrimaryKeyRelatedField +from server.schemas import PrimaryKeyRelatedField, SelfNestedField from server.dao.vuln import VulnerabilityDAO vulns_api = Blueprint('vulns_api', __name__) logger = logging.getLogger(__name__) + +class EvidenceSchema(AutoSchema): + content_type = fields.Method('get_content_type') + data = fields.Method('get_data') + + class Meta: + model = File + fields = ( + 'content_type', + 'data' + ) + + def get_content_type(self, file_obj): + depot = DepotManager.get() + return depot.get(file_obj.content.get('file_id')).content_type + + def get_data(self, file_obj): + depot = DepotManager.get() + return b64encode(depot.get(file_obj.content.get('file_id')).read()) + + class VulnerabilitySchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') _rev = fields.String(default='') - _attachments = fields.Method(load_only=True, deserialize='load_attachments') owned = fields.Boolean(dump_only=True, default=False) owner = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator') impact = fields.Method('get_impact', deserialize='load_impact') @@ -96,11 +122,20 @@ def get_metadata(self, obj): } def get_attachments(self, obj): - # TODO: retrieve obj attachments - return [] + res = [] + files = db.session.query(File).filter_by(object_id=obj.id, object_type=obj.__class__.__name__).all() + + for file_obj in files: + ret, errors = EvidenceSchema().dump(file_obj) + if errors: + raise ValidationError(errors, data=ret) + res.append(ret) + + return res def get_hostnames(self, obj): # TODO: improve performance here + # TODO: move this to models? if obj.host: return [hostname.name for hostname in obj.host.hostnames] if obj.service: @@ -149,15 +184,6 @@ def load_parent(self, value): self.parent_id = value return value - def load_attachments(self, value): - # TODO: implement attachments - pass - - @post_load - def clean_up(self, data): - data.pop('_attachments') - return data - @post_load def set_impact(self, data): impact = data.pop('impact') @@ -165,7 +191,6 @@ def set_impact(self, data): pass return data - @post_load def set_parent(self, data): # schema guarantees that parent_type exists. @@ -221,22 +246,43 @@ class VulnerabilityView(PaginatedMixin, ReadWriteWorkspacedView): 'VulnerabilityWeb': VulnerabilityWeb, 'VulnerabilityGeneric': VulnerabilityGeneric, } - schema_class = { + schema_class_dict = { 'Vulnerability': VulnerabilitySchema, 'VulnerabilityWeb': VulnerabilityWebSchema } + def post(self, **kwargs): + data = self._parse_data(self._get_schema_class()(strict=True), + request) + attachments = data.pop('_attachments') + obj = self.model_class(**data) + created = self._perform_create(obj, **kwargs) + for filename, attachment in attachments.items(): + faraday_file = FaradayUploadedFile(b64decode(attachment['data'])) + get_or_create( + db.session, + File, + object_id=created.id, + object_type=created.__class__.__name__, + name=os.path.splitext(ntpath.basename(filename))[0], + filename=ntpath.basename(filename), + content=faraday_file, + ) + return self._dump(created), 201 + @property def model_class(self): if request.method == 'POST': return self.model_class_dict[request.json['type']] + # We use Generic to list all vulns from all types return self.model_class_dict['VulnerabilityGeneric'] def _get_schema_class(self): - assert self.schema_class is not None, "You must define schema_class" + assert self.schema_class_dict is not None, "You must define schema_class" if request.method == 'POST': - return self.schema_class[request.json['type']] - return self.schema_class['VulnerabilityWeb'] + return self.schema_class_dict[request.json['type']] + # We use web since it has all the fields + return self.schema_class_dict['VulnerabilityWeb'] def _envelope_list(self, objects, pagination_metadata=None): vulns = [] @@ -250,53 +296,4 @@ def _envelope_list(self, objects, pagination_metadata=None): 'vulnerabilities': vulns, } -VulnerabilityView.register(vulns_api) - - -@vulns_api.route('/ws//vulns', methods=['GET']) -@gzipped -def get_vulnerabilities(workspace=None): - validate_workspace(workspace) - get_logger(__name__).debug("Request parameters: {!r}"\ - .format(request.args)) - - page = get_integer_parameter('page', default=0) - page_size = get_integer_parameter('page_size', default=0) - search = request.args.get('search') - order_by = request.args.get('sort') - order_dir = request.args.get('sort_dir') - - vuln_filter = filter_request_args( - 'page', 'page_size', 'search', 'sort', 'sort_dir') - - vuln_dao = VulnerabilityDAO(workspace) - - result = vuln_dao.list(search=search, - page=page, - page_size=page_size, - order_by=order_by, - order_dir=order_dir, - vuln_filter=vuln_filter) - - return jsonify(result) - - -@vulns_api.route('/ws//vulns/count', methods=['GET']) -@gzipped -def count_vulnerabilities(workspace=None): - validate_workspace(workspace) - get_logger(__name__).debug("Request parameters: {!r}"\ - .format(request.args)) - - field = request.args.get('group_by') - search = request.args.get('search') - vuln_filter = filter_request_args('search', 'group_by') - - vuln_dao = VulnerabilityDAO(workspace) - result = vuln_dao.count(group_by=field, - search=search, - vuln_filter=vuln_filter) - if result is None: - abort(400) - - return jsonify(result) +VulnerabilityView.register(vulns_api) \ No newline at end of file diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 75309fe44c2..3804b24c696 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1,23 +1,24 @@ #-*- coding: utf8 -*- '''Tests for many API endpoints that do not depend on workspace_name''' +import os +from base64 import b64encode import pytest from server.api.modules.vulns import VulnerabilityView from test_cases import factories -from test_api_workspaced_base import ListTestsMixin, API_PREFIX, GenericAPITest +from test_api_workspaced_base import ReadOnlyAPITests from server.models import ( Vulnerability, VulnerabilityWeb, - Workspace, ) -from server.api.modules.commandsrun import CommandView -from server.api.modules.workspaces import WorkspaceView from test_cases.factories import ServiceFactory +CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) + @pytest.mark.usefixtures('logged_user') -class TestListVulnerabilityView(GenericAPITest): +class TestListVulnerabilityView(ReadOnlyAPITests): model = Vulnerability factory = factories.VulnerabilityFactory api_endpoint = 'vulns' @@ -115,7 +116,9 @@ def test_hostnames(self, host_with_hostnames, test_client, session): assert set(res.json['hostnames']) == set(hostname.name for hostname in host_with_hostnames.hostnames) - def _create_post_data_vulnerability(self, name, vuln_type, parent_id, parent_type, refs, policyviolations): + def _create_post_data_vulnerability(self, name, vuln_type, parent_id, + parent_type, refs, policyviolations, + attachments=None): data = {'metadata': {'update_time': 1508254070.211, 'update_user': '', 'update_action': 0, 'creator': 'UI Web', 'create_time': 1508254070.211, 'update_controller_action': 'UI Web New', 'owner': ''}, 'obj_id': '5a60af7f01dde6d3acfa8e9d3bef265c361a49d2', @@ -141,6 +144,7 @@ def _create_post_data_vulnerability(self, name, vuln_type, parent_id, parent_typ 'description': '', 'protocol': '', 'version': ''} + if vuln_type == 'VulnerabilityWeb': data.update({ "method": "GET", @@ -151,6 +155,15 @@ def _create_post_data_vulnerability(self, name, vuln_type, parent_id, parent_typ "request": "", "response": "", "website": "www.pepe.com"}) + + if attachments: + data['_attachments'] = {} + for attachment in attachments: + data['_attachments'][attachment.name] = { + "content_type": "application/x-shellscript", + "data": b64encode(attachment.read()) + } + return data def test_create_vuln(self, host_with_hostnames, test_client, session): @@ -197,3 +210,33 @@ def test_create_vuln_web(self, host_with_hostnames, test_client, session): assert res.json['parent_type'] == 'Service' assert res.json['method'] == 'GET' assert res.json['path'] == '/pepep' + + def test_create_vuln_with_evidence(self, host_with_hostnames, test_client, session): + session.commit() # flush host_with_hostnames + attachments = [ + open(os.path.join(CURRENT_PATH, 'data', 'faraday.png'), 'r'), + open(os.path.join(CURRENT_PATH, 'data', 'test.html'), 'r') + ] + raw_data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='Vulnerability', + parent_id=host_with_hostnames.id, + parent_type='Host', + refs=[], + policyviolations=[], + attachments=attachments, + ) + ws_name = host_with_hostnames.workspace.name + vuln_count_previous = session.query(Vulnerability).count() + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) + + assert res.status_code == 201 + assert len(res.json['_attachments']) == 2 + assert vuln_count_previous + 1 == session.query(Vulnerability).count() + map(lambda fileobj: fileobj.close(), attachments) + + def create_vuln_with_refs(self): + pass + + def create_vuln_with_policyviolations(self): + pass From c312017ee10caaa63569dc573a1c7c72c24ee13e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 18 Oct 2017 16:39:31 -0300 Subject: [PATCH 0361/1506] Update requirements --- requirements_server.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_server.txt b/requirements_server.txt index 154c53f7b91..a747c3978d5 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -18,3 +18,4 @@ twisted>=16.1.1 webargs marshmallow-sqlalchemy filteralchemy +filedepot==0.5.0 From 344cc58fab923063936ce0c92a4f2656e67809d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 18 Oct 2017 17:17:35 -0300 Subject: [PATCH 0362/1506] Fix filteralchemy support for not nullable fields The library has a bug with not nullable fields. It makes them required in the querystring although they should be optional. I fixed on the app code becuase filteralchemy project doesn't seem to be active. TODO: look for other libraries for filtering --- server/api/base.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/api/base.py b/server/api/base.py index 0c40fd9d58b..550f59a85cf 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -10,6 +10,7 @@ from werkzeug.routing import parse_rule from marshmallow import Schema from marshmallow.compat import with_metaclass +from marshmallow_sqlalchemy import ModelConverter from marshmallow_sqlalchemy.schema import ModelSchemaMeta, ModelSchemaOpts from webargs.flaskparser import FlaskParser, parser, abort from webargs.core import ValidationError @@ -384,6 +385,16 @@ class AutoSchema(with_metaclass(ModelSchemaMeta, Schema)): OPTIONS_CLASS = ModelSchemaOpts +class FilterAlchemyModelConverter(ModelConverter): + """Use this to make all fields of a model not required. + + It is used to make filteralchemy support not nullable columns""" + + def _add_column_kwargs(self, kwargs, column): + kwargs['required'] = False + + class FilterSetMeta: """Base Meta class of FilterSet objects""" parser = parser + converter = FilterAlchemyModelConverter() From 7da465915dbf7b2f8715c7fe9a819ba4095e5796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 18 Oct 2017 17:44:21 -0300 Subject: [PATCH 0363/1506] Allow filtering vulns by website and by severity Also changed vuln tests to inherit from ReadOnlyAPITests so it adds generic list and retrieve smoke tests --- server/api/modules/vulns.py | 16 ++++++- test_cases/test_api_vulnerability.py | 63 +++++++++++++++++++++++++++- 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 392b8cb0bdc..9fa2024f6be 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -4,12 +4,15 @@ import time import logging +from filteralchemy import FilterSet, operators from flask import request, jsonify, abort from flask import Blueprint from marshmallow import fields, post_load from server.api.base import ( AutoSchema, + FilterAlchemyMixin, + FilterSetMeta, PaginatedMixin, ReadWriteWorkspacedView, ) @@ -213,8 +216,19 @@ class Meta: 'target', 'resolution', 'method', 'metadata') -class VulnerabilityView(PaginatedMixin, ReadWriteWorkspacedView): +class VulnerabilityFilterSet(FilterSet): + class Meta(FilterSetMeta): + model = VulnerabilityWeb # It has all the fields + fields = ( + 'severity', 'website') + operators = (operators.Equal,) + + +class VulnerabilityView(PaginatedMixin, + FilterAlchemyMixin, + ReadWriteWorkspacedView): route_base = 'vulns' + filterset_class = VulnerabilityFilterSet model_class_dict = { 'Vulnerability': Vulnerability, diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 75309fe44c2..5956c742a17 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -5,7 +5,11 @@ from server.api.modules.vulns import VulnerabilityView from test_cases import factories -from test_api_workspaced_base import ListTestsMixin, API_PREFIX, GenericAPITest +from test_api_workspaced_base import ( + API_PREFIX, + ListTestsMixin, + ReadOnlyAPITests +) from server.models import ( Vulnerability, VulnerabilityWeb, @@ -17,7 +21,7 @@ @pytest.mark.usefixtures('logged_user') -class TestListVulnerabilityView(GenericAPITest): +class TestListVulnerabilityView(ReadOnlyAPITests): model = Vulnerability factory = factories.VulnerabilityFactory api_endpoint = 'vulns' @@ -197,3 +201,58 @@ def test_create_vuln_web(self, host_with_hostnames, test_client, session): assert res.json['parent_type'] == 'Service' assert res.json['method'] == 'GET' assert res.json['path'] == '/pepep' + + @pytest.mark.usefixtures('mock_envelope_list') + def test_filter_by_severity(self, test_client, session, + second_workspace, + vulnerability_factory, + vulnerability_web_factory, + ): + expected_ids = set() + + high_vulns = vulnerability_factory.create_batch( + 5, workspace=second_workspace, severity='high') + high_vulns_web = vulnerability_web_factory.create_batch( + 5, workspace=second_workspace, severity='high') + session.commit() + + expected_ids.update(vuln.id for vuln in high_vulns) + expected_ids.update(vuln.id for vuln in high_vulns_web) + + vulnerability_factory.create_batch( + 5, workspace=second_workspace, severity='medium') + vulnerability_web_factory.create_batch( + 5, workspace=second_workspace, severity='medium') + + res = test_client.get(self.url( + workspace=second_workspace) + '?severity=high') + assert res.status_code == 200 + for vuln in res.json['data']: + assert vuln['severity'] == 'high' + assert set(vuln['_id'] for vuln in res.json['data']) == expected_ids + + @pytest.mark.usefixtures('mock_envelope_list') + def test_filter_by_website(self, test_client, session, + second_workspace, + vulnerability_factory, + vulnerability_web_factory, + ): + + # Vulns that shouldn't be shown + vulnerability_factory.create_batch(5, workspace=second_workspace) + vulnerability_web_factory.create_batch( + 5, workspace=second_workspace, website='other.com') + + # Vulns that must be shown + expected_vulns = vulnerability_web_factory.create_batch( + 5, workspace=second_workspace, website='faradaysec.com') + session.commit() + expected_ids = {vuln.id for vuln in expected_vulns} + + res = test_client.get(self.url( + workspace=second_workspace) + '?website=faradaysec.com') + assert res.status_code == 200 + + for vuln in res.json['data']: + assert vuln['website'] == 'faradaysec.com' + assert set(vuln['_id'] for vuln in res.json['data']) == expected_ids From db4a5a97d4c885a1a99c4b516b9743e16fa6c9d9 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 18 Oct 2017 18:25:25 -0300 Subject: [PATCH 0364/1506] Uverride _perform_create instead of post --- server/api/base.py | 23 +++++++++++++++++------ server/api/modules/vulns.py | 19 ++++++++++--------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 550f59a85cf..634a320b470 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -256,11 +256,12 @@ class CreateMixin(object): def post(self, **kwargs): data = self._parse_data(self._get_schema_class()(strict=True), flask.request) - obj = self.model_class(**data) - created = self._perform_create(obj, **kwargs) + + created = self._perform_create(data, **kwargs) return self._dump(created), 201 - def _perform_create(self, obj): + def _perform_create(self, data, **kwargs): + obj = self.model_class(**data) # assert not db.session.new with db.session.no_autoflush: # Required because _validate_uniqueness does a select. Doing this @@ -274,10 +275,20 @@ def _perform_create(self, obj): class CreateWorkspacedMixin(CreateMixin): """Add POST // route""" - def _perform_create(self, obj, workspace_name): + def _perform_create(self, data, workspace_name): assert not db.session.new - obj.workspace = self._get_workspace(workspace_name) - return super(CreateWorkspacedMixin, self)._perform_create(obj) + workspace = self._get_workspace(workspace_name) + obj = self.model_class(**data) + obj.workspace = workspace + # assert not db.session.new + with db.session.no_autoflush: + # Required because _validate_uniqueness does a select. Doing this + # outside a no_autoflush block would result in a premature create. + self._validate_uniqueness(obj) + db.session.add(obj) + db.session.commit() + + return obj class UpdateMixin(object): diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index ae5532c9b6c..145e62e0c73 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -69,6 +69,7 @@ class VulnerabilitySchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') _rev = fields.String(default='') + _attachments = fields.Method('get_attachments') owned = fields.Boolean(dump_only=True, default=False) owner = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator') impact = fields.Method('get_impact', deserialize='load_impact') @@ -90,7 +91,6 @@ class VulnerabilitySchema(AutoSchema): status = fields.Method('get_status', deserialize='load_status') type = fields.Method('get_type', deserialize='load_type') obj_id = fields.String(dump_only=True, attribute='id') - _attachments = fields.Method('get_attachments') target = fields.String(default='') # TODO: review this attribute class Meta: @@ -264,24 +264,25 @@ class VulnerabilityView(PaginatedMixin, 'VulnerabilityWeb': VulnerabilityWebSchema } - def post(self, **kwargs): + def _perform_create(self, data, **kwargs): data = self._parse_data(self._get_schema_class()(strict=True), request) attachments = data.pop('_attachments') - obj = self.model_class(**data) - created = self._perform_create(obj, **kwargs) + references = data.pop('references') + obj = super(VulnerabilityView, self)._perform_create(data, **kwargs) + for filename, attachment in attachments.items(): faraday_file = FaradayUploadedFile(b64decode(attachment['data'])) get_or_create( db.session, File, - object_id=created.id, - object_type=created.__class__.__name__, - name=os.path.splitext(os.path.basename(filename))[0], - filename=os.path.basename(filename), + object_id=obj.id, + object_type=obj.__class__.__name__, + name = os.path.splitext(os.path.basename(filename))[0], + filename = os.path.basename(filename), content=faraday_file, ) - return self._dump(created), 201 + return obj @property def model_class(self): From 4c699af279d6294c4b0b07fd9f57c52d5cc322ff Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 18 Oct 2017 19:38:40 -0300 Subject: [PATCH 0365/1506] Clean up imports --- server/api/modules/vulns.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 145e62e0c73..974019e1bfa 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -1,13 +1,13 @@ # Faraday Penetration Test IDE # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information +import os import time import logging from base64 import b64encode, b64decode from filteralchemy import FilterSet, operators -import os -from flask import request, jsonify, abort +from flask import request from flask import Blueprint from marshmallow import fields, post_load, ValidationError @@ -29,22 +29,14 @@ VulnerabilityGeneric, Host, Service, File) from server.utils.database import get_or_create -from server.utils.logger import get_logger -from server.utils.web import ( - gzipped, - validate_workspace, - get_integer_parameter, - filter_request_args -) + from server.api.modules.services import ServiceSchema -from server.schemas import PrimaryKeyRelatedField, SelfNestedField -from server.dao.vuln import VulnerabilityDAO +from server.schemas import PrimaryKeyRelatedField vulns_api = Blueprint('vulns_api', __name__) logger = logging.getLogger(__name__) - class EvidenceSchema(AutoSchema): content_type = fields.Method('get_content_type') data = fields.Method('get_data') From ac3fc742c1c59e9544430fee49004b3bb6ade7fa Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 18 Oct 2017 19:43:48 -0300 Subject: [PATCH 0366/1506] Fix js code to send parent_type --- server/www/scripts/vulns/providers/vuln.js | 22 +++++++--------------- server/www/scripts/vulns/providers/web.js | 8 -------- 2 files changed, 7 insertions(+), 23 deletions(-) diff --git a/server/www/scripts/vulns/providers/vuln.js b/server/www/scripts/vulns/providers/vuln.js index 50613554d68..4e6ae1bd96e 100644 --- a/server/www/scripts/vulns/providers/vuln.js +++ b/server/www/scripts/vulns/providers/vuln.js @@ -37,6 +37,7 @@ angular.module('faradayApp') this.owner = ""; this.owned = ""; this.parent = ""; + this.parent_type = ""; this.refs = ""; this.resolution = ""; this.service = ""; @@ -70,24 +71,14 @@ angular.module('faradayApp') set: function(ws, data) { var self = this; - // new vuln - if(data._id === undefined) { - var id = CryptoJS.SHA1(data.name + "." + data.desc).toString(); - self._id = data.parent + "." + id; - self.obj_id = id; - } else { - self._id = data._id; - self.obj_id = data.obj_id; - if(data._rev !== undefined) self._rev = data._rev; - if(data.metadata !== undefined) self.metadata = data.metadata; - if(data.target !== undefined) self.target = data.target; - if(data.hostnames !== undefined) self.hostnames = data.hostnames; - if(data.service !== undefined) self.service = data.service; - } - + if(data.metadata !== undefined) self.metadata = data.metadata; + if(data.target !== undefined) self.target = data.target; + if(data.hostnames !== undefined) self.hostnames = data.hostnames; + if(data.service !== undefined) self.service = data.service; if(data.owner !== undefined) self.owner = data.owner; self.ws = ws; if(data.parent !== undefined) self.parent = data.parent; + if(data.parent_type !== undefined) self.parent_type = data.parent_type; self.public_properties.forEach(function(property) { if(data[property] !== undefined) self[property] = data[property]; @@ -170,6 +161,7 @@ angular.module('faradayApp') vuln.obj_id = self.obj_id; vuln.owner = self.owner; vuln.parent = self.parent; + vuln.parent_type = self.parent_type; vuln.type = self.type; vuln.ws = self.ws; diff --git a/server/www/scripts/vulns/providers/web.js b/server/www/scripts/vulns/providers/web.js index f4434b6aa21..e4f81fc7a59 100644 --- a/server/www/scripts/vulns/providers/web.js +++ b/server/www/scripts/vulns/providers/web.js @@ -30,14 +30,6 @@ angular.module('faradayApp') Vuln.prototype.set.call(self, ws, data); - // new vuln - if(data._id === undefined) { - var id = CryptoJS.SHA1(data.name + "." + data.website + "." + data.path + "." + data.desc).toString(); - - self._id = data.parent + "." + id; - self.obj_id = id; - } - self.type = "VulnerabilityWeb"; public_properties.forEach(function(property) { From 980d7871ccc2491d65faeb4e845adadfa2f5e945 Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 18 Oct 2017 19:56:08 -0300 Subject: [PATCH 0367/1506] Fix Vuln Template importer The field CWE was being ignored and therefore got lost in the import process. Now it is added as a reference in the template. --- server/importer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/importer.py b/server/importer.py index e686835346c..af62ca37193 100644 --- a/server/importer.py +++ b/server/importer.py @@ -876,6 +876,9 @@ def run(self): vuln_template.severity = new_severity references = document['references'] if isinstance(document['references'], list) else [x.strip() for x in document['references'].split(',')] + cwe_field = document.get('cwe') + if cwe_field not in references: + references.append(cwe_field) for ref_doc in references: get_or_create(session, ReferenceTemplate, From 459bcc8b7b1944aa8c1028930c989771b219a00c Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 18 Oct 2017 19:57:54 -0300 Subject: [PATCH 0368/1506] Fix Vulnerability Template Schema Now the vuln templates API returns the proper JSON. --- server/api/modules/vulnerability_template.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/server/api/modules/vulnerability_template.py b/server/api/modules/vulnerability_template.py index 544272d89f8..5cae10682cf 100644 --- a/server/api/modules/vulnerability_template.py +++ b/server/api/modules/vulnerability_template.py @@ -10,6 +10,7 @@ from server.api.base import AutoSchema, ReadWriteView from server.models import VulnerabilityTemplate +from server.schemas import PrimaryKeyRelatedField vulnerability_template_api = Blueprint('vulnerability_template_api', __name__) logger = server.utils.logger.get_logger(__name__) @@ -18,11 +19,13 @@ class VulnerabilityTemplateSchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') _rev = fields.String(default='', dump_only=True) - + cwe = fields.String(dump_only=True, default='') # this field was deprecated and the legacy data is added to references on import + exploitation = fields.String(dump_only=True, attribute='severity') + references = PrimaryKeyRelatedField('name', many=True, attribute='references') class Meta: model = VulnerabilityTemplate - fields = ('id', '_id', '_rev') + fields = ('id', '_id', '_rev', 'cwe', 'description', 'exploitation', 'name', 'references', 'resolution') class VulnerabilityTemplateView(ReadWriteView): @@ -36,7 +39,8 @@ def _envelope_list(self, objects, pagination_metadata=None): vuln_tpls.append({ 'id': template['_id'], 'key': template['_id'], - 'value': template + 'value': {'rev': ''}, + 'doc': template }) return { 'rows': vuln_tpls, From 61cfb6efc333c1dd5a0f4007b61395d8da8e63f3 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 19 Oct 2017 14:41:44 -0300 Subject: [PATCH 0369/1506] Add reference to vuln --- server/api/base.py | 1 + server/api/modules/vulns.py | 6 +++- server/models.py | 54 +++++++++++++++------------- test_cases/test_api_vulnerability.py | 19 ++++++++-- 4 files changed, 51 insertions(+), 29 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 634a320b470..04bc8d089ea 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -278,6 +278,7 @@ class CreateWorkspacedMixin(CreateMixin): def _perform_create(self, data, workspace_name): assert not db.session.new workspace = self._get_workspace(workspace_name) + self.workspace = workspace obj = self.model_class(**data) obj.workspace = workspace # assert not db.session.new diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 974019e1bfa..d19d493ac81 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -27,7 +27,7 @@ Vulnerability, VulnerabilityWeb, VulnerabilityGeneric, - Host, Service, File) + Host, Service, File, Reference) from server.utils.database import get_or_create from server.api.modules.services import ServiceSchema @@ -263,6 +263,10 @@ def _perform_create(self, data, **kwargs): references = data.pop('references') obj = super(VulnerabilityView, self)._perform_create(data, **kwargs) + for reference in references: + instance, _ = get_or_create(db.session, Reference, name=reference, workspace=self.workspace) + obj.references.append(instance) + for filename, attachment in attachments.items(): faraday_file = FaradayUploadedFile(b64decode(attachment['data'])) get_or_create( diff --git a/server/models.py b/server/models.py index 36bb7d425f8..75fb86dc74b 100644 --- a/server/models.py +++ b/server/models.py @@ -16,10 +16,8 @@ String, Text, UniqueConstraint, - Unicode, event ) -from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import relationship, backref from sqlalchemy.sql import select, text, table from sqlalchemy import func @@ -31,13 +29,14 @@ _EngineConnector ) from depot.fields.sqlalchemy import UploadedFileField + +import server.config from server.fields import FaradayUploadedFile from flask_security import ( RoleMixin, UserMixin, ) - -import server.config +from server.utils.database import get_or_create class SQLAlchemy(OriginalSQLAlchemy): @@ -339,6 +338,11 @@ class VulnerabilityGeneric(VulnerabilityABC): ) workspace = relationship('Workspace', backref='vulnerabilities') + references = relationship( + "Reference", + secondary="reference_vulnerability_association" + ) + __mapper_args__ = { 'polymorphic_on': type } @@ -473,33 +477,33 @@ class Reference(Metadata): name = Column(Text, nullable=False) workspace_id = Column( - Integer, - ForeignKey('workspace.id'), - index=True, - nullable=False - ) + Integer, + ForeignKey('workspace.id'), + index=True, + nullable=False + ) workspace = relationship( - 'Workspace', - backref='references', - foreign_keys=[workspace_id], - ) - - vulnerability_id = Column( - Integer, - ForeignKey(VulnerabilityGeneric.id), - index=True - ) - vulnerability = relationship( - 'VulnerabilityGeneric', - backref='references', - foreign_keys=[vulnerability_id], - ) + 'Workspace', + backref='references', + foreign_keys=[workspace_id], + ) __table_args__ = ( - UniqueConstraint('name', 'vulnerability_id', 'workspace_id', name='uix_reference_name_vulnerability_workspace'), + UniqueConstraint('name', 'workspace_id', name='uix_reference_name_vulnerability_workspace'), ) +class ReferenceVulnerabilityAssociation(db.Model): + + __tablename__ = 'reference_vulnerability_association' + + vulnerability_id = Column(Integer, ForeignKey('vulnerability.id'), primary_key=True) + reference_id = Column(Integer, ForeignKey('reference.id'), primary_key=True) + + reference = relationship("Reference", backref="reference_associations", foreign_keys=[reference_id]) + vulnerability = relationship("Vulnerability", backref="vulnerability_associations", foreign_keys=[vulnerability_id]) + + class PolicyViolationTemplate(Metadata): __tablename__ = 'policy_violation_template' id = Column(Integer, primary_key=True) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 5f4d15e89e1..710673201ec 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -294,8 +294,21 @@ def test_create_vuln_with_evidence(self, host_with_hostnames, test_client, sessi assert vuln_count_previous + 1 == session.query(Vulnerability).count() map(lambda fileobj: fileobj.close(), attachments) - def create_vuln_with_refs(self): - pass + def test_create_vuln_with_refs(self, host_with_hostnames, test_client, session): + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='Vulnerability', + parent_id=host_with_hostnames.id, + parent_type='Host', + refs=['CVE-2017-0002', 'CVE-2017-0012'], + policyviolations=[] + ) + ws_name = host_with_hostnames.workspace.name + vuln_count_previous = session.query(Vulnerability).count() + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) + assert res.status_code == 201 + assert vuln_count_previous + 1 == session.query(Vulnerability).count() - def create_vuln_with_policyviolations(self): + def test_create_vuln_with_policyviolations(self): pass From 19c3f37d5d518041f89fae6cdab1474c220bbb20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 19 Oct 2017 14:57:04 -0300 Subject: [PATCH 0370/1506] Add many fields to vulnerability filterset To make the status report compatible --- Jenkinsfile | 2 ++ server/api/modules/vulns.py | 20 ++++++++++++++++++-- test_cases/test_api_vulnerability.py | 7 +++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 65893ed1f6b..da8316eaa5e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,6 +24,8 @@ node { pip install -r $WORKSPACE/requirements_server.txt pip install -r $WORKSPACE/requirements_extras.txt pip install -r $WORKSPACE/requirements_dev.txt + pip uninstall -y filteralchemy + pip install -e git+https://github.com/sh4r3m4n/filteralchemy@dev#egg=Flask_Classful deactivate """ } diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index ae5532c9b6c..bd4880dea9d 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -240,12 +240,28 @@ class Meta: 'target', 'resolution', 'method', 'metadata') +# Use this override for filterset fields that filter by en exact match by +# default, and not by a similar one (like operator) +_strict_filtering = {'default_operator': operators.Equal} + class VulnerabilityFilterSet(FilterSet): class Meta(FilterSetMeta): model = VulnerabilityWeb # It has all the fields + # TODO migration: Check if we should add fields creator, owner, command, + # impact, type, service, issuetracker, tags, date, target, host, + # easeofresolution, evidence, policy violations, hostnames, target fields = ( - 'severity', 'website') - operators = (operators.Equal,) + "status", "website", "parameter_name", "query_string", "path", + "data", "severity", "confirmed", "name", "request", "response", + "parameters", "resolution", "method", "ease_of_resolution", + "description") + strict_fields = ( + "severity", "confirmed", "method" + ) + default_operator = operators.ILike + column_overrides = { + field: _strict_filtering for field in strict_fields} + operators = (operators.ILike, operators.Equal) class VulnerabilityView(PaginatedMixin, diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 5f4d15e89e1..9727ebe8dba 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -244,6 +244,13 @@ def test_filter_by_severity(self, test_client, session, assert vuln['severity'] == 'high' assert set(vuln['_id'] for vuln in res.json['data']) == expected_ids + # This shouldn't show any vulns since by default severity filter is + # exact + res = test_client.get(self.url( + workspace=second_workspace) + '?severity=h%25') + assert res.status_code == 200 + assert len(res.json['data']) == 0 + @pytest.mark.usefixtures('mock_envelope_list') def test_filter_by_website(self, test_client, session, second_workspace, From 1659cf0c6e147eb572380865ecb976aa2ae8f8dc Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 19 Oct 2017 15:43:52 -0300 Subject: [PATCH 0371/1506] Fix reference and policy violations tests. Add verification to avoid duplicate references and pvs --- server/api/modules/vulns.py | 11 ++++++--- server/models.py | 30 +++++++++++++----------- server/schemas.py | 2 +- test_cases/factories.py | 2 -- test_cases/test_api_vulnerability.py | 34 ++++++++++++++++++++++------ 5 files changed, 53 insertions(+), 26 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 1bae7de8a2c..e3831c57efd 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -27,7 +27,7 @@ Vulnerability, VulnerabilityWeb, VulnerabilityGeneric, - Host, Service, File, Reference) + Host, Service, File, Reference, PolicyViolation) from server.utils.database import get_or_create from server.api.modules.services import ServiceSchema @@ -66,9 +66,9 @@ class VulnerabilitySchema(AutoSchema): owner = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator') impact = fields.Method('get_impact', deserialize='load_impact') policyviolations = PrimaryKeyRelatedField('name', many=True, - attribute='policy_violations') + attribute='policy_violations', default=[]) desc = fields.String(dump_only=True, attribute='description') - refs = PrimaryKeyRelatedField('name', many=True, attribute='references') + refs = PrimaryKeyRelatedField('name', many=True, attribute='references', default=[]) issuetracker = fields.Method('get_issuetracker') parent = fields.Method('get_parent', deserialize='load_parent') parent_type = fields.Method('get_parent_type') @@ -277,12 +277,17 @@ def _perform_create(self, data, **kwargs): request) attachments = data.pop('_attachments') references = data.pop('references') + policyviolations = data.pop('policy_violations') obj = super(VulnerabilityView, self)._perform_create(data, **kwargs) for reference in references: instance, _ = get_or_create(db.session, Reference, name=reference, workspace=self.workspace) obj.references.append(instance) + for policyviolation in policyviolations: + instance, _ = get_or_create(db.session, PolicyViolation, name=policyviolation, workspace=self.workspace) + obj.policy_violations.append(instance) + for filename, attachment in attachments.items(): faraday_file = FaradayUploadedFile(b64decode(attachment['data'])) get_or_create( diff --git a/server/models.py b/server/models.py index 75fb86dc74b..dcd5fd4b48c 100644 --- a/server/models.py +++ b/server/models.py @@ -343,6 +343,11 @@ class VulnerabilityGeneric(VulnerabilityABC): secondary="reference_vulnerability_association" ) + policy_violations = relationship( + "PolicyViolation", + secondary="policy_violation_vulnerability_association" + ) + __mapper_args__ = { 'polymorphic_on': type } @@ -501,7 +506,18 @@ class ReferenceVulnerabilityAssociation(db.Model): reference_id = Column(Integer, ForeignKey('reference.id'), primary_key=True) reference = relationship("Reference", backref="reference_associations", foreign_keys=[reference_id]) - vulnerability = relationship("Vulnerability", backref="vulnerability_associations", foreign_keys=[vulnerability_id]) + vulnerability = relationship("Vulnerability", backref="reference_vulnerability_associations", foreign_keys=[vulnerability_id]) + + +class PolicyViolationVulnerabilityAssociation(db.Model): + __tablename__ = 'policy_violation_vulnerability_association' + + vulnerability_id = Column(Integer, ForeignKey('vulnerability.id'), primary_key=True) + policy_violation_id = Column(Integer, ForeignKey('policy_violation.id'), primary_key=True) + + policy_violation = relationship("PolicyViolation", backref="policy_violation_associations", foreign_keys=[policy_violation_id]) + vulnerability = relationship("Vulnerability", backref="policy_violationvulnerability_associations", + foreign_keys=[vulnerability_id]) class PolicyViolationTemplate(Metadata): @@ -545,21 +561,9 @@ class PolicyViolation(Metadata): foreign_keys=[workspace_id], ) - vulnerability_id = Column( - Integer, - ForeignKey(VulnerabilityGeneric.id), - index=True - ) - vulnerability = relationship( - 'VulnerabilityGeneric', - backref='policy_violations', - foreign_keys=[vulnerability_id] - ) - __table_args__ = ( UniqueConstraint( 'name', - 'vulnerability_id', 'workspace_id', name='uix_policy_violation_template_name_vulnerability_workspace'), ) diff --git a/server/schemas.py b/server/schemas.py index c4b3f3776f6..23033bc3fbc 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -26,7 +26,7 @@ def _serialize(self, value, attr, obj): return ret else: if value is None: - return None + return [] return getattr(value, self.field_name) diff --git a/test_cases/factories.py b/test_cases/factories.py index ef97ee511fd..4c212d1cb9e 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -121,7 +121,6 @@ class Meta: class PolicyViolationFactory(WorkspaceObjectFactory): name = FuzzyText() - vulnerability = None class Meta: model = PolicyViolation @@ -130,7 +129,6 @@ class Meta: class ReferenceFactory(WorkspaceObjectFactory): name = FuzzyText() - vulnerability = None class Meta: model = Reference diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 7e73b962bfc..080c6ef3eae 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -15,7 +15,7 @@ from server.models import ( Vulnerability, VulnerabilityWeb, -) + Reference, PolicyViolation) from test_cases.factories import ServiceFactory CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) @@ -87,12 +87,14 @@ def test_handles_vuln_with_no_creator(self, session.commit() res = test_client.get(self.url(vuln)) assert res.status_code == 200 - assert res.json['owner'] is None + assert res.json['owner'] == [] def test_shows_policy_violations(self, workspace, test_client, session, policy_violation_factory): pvs = policy_violation_factory.create_batch( - 5, workspace=workspace, vulnerability=self.first_object) + 5, workspace=workspace) + for pv in pvs: + self.first_object.policy_violations.append(pv) session.commit() res = test_client.get(self.url(self.first_object)) assert res.status_code == 200 @@ -102,7 +104,9 @@ def test_shows_policy_violations(self, workspace, test_client, session, def test_shows_refs(self, workspace, test_client, session, reference_factory): refs = reference_factory.create_batch( - 5, workspace=workspace, vulnerability=self.first_object) + 5, workspace=workspace) + for ref in refs: + self.first_object.references.append(ref) session.commit() res = test_client.get(self.url(self.first_object)) assert res.status_code == 200 @@ -308,14 +312,30 @@ def test_create_vuln_with_refs(self, host_with_hostnames, test_client, session): vuln_type='Vulnerability', parent_id=host_with_hostnames.id, parent_type='Host', - refs=['CVE-2017-0002', 'CVE-2017-0012'], + refs=['CVE-2017-0002', 'CVE-2017-0012', 'CVE-2017-0012'], policyviolations=[] ) ws_name = host_with_hostnames.workspace.name vuln_count_previous = session.query(Vulnerability).count() res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) assert res.status_code == 201 + assert session.query(Reference).count() == 2 assert vuln_count_previous + 1 == session.query(Vulnerability).count() - def test_create_vuln_with_policyviolations(self): - pass + def test_create_vuln_with_policyviolations(self, host_with_hostnames, test_client, session): + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='Vulnerability', + parent_id=host_with_hostnames.id, + parent_type='Host', + refs=[], + policyviolations=['PCI DSS Credir card not encrypted', + 'PCI DSS Credir card not encrypted'], + ) + ws_name = host_with_hostnames.workspace.name + vuln_count_previous = session.query(Vulnerability).count() + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) + assert res.status_code == 201 + assert session.query(PolicyViolation).count() == 1 + assert vuln_count_previous + 1 == session.query(Vulnerability).count() From a9d995401c947b4296b8e77f50735f2b80f17de6 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 19 Oct 2017 16:16:01 -0300 Subject: [PATCH 0372/1506] Fix bug on PrimaryKeyRelatedField --- server/schemas.py | 4 +--- test_cases/test_api_vulnerability.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/server/schemas.py b/server/schemas.py index 23033bc3fbc..cbc6eb2e87f 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -25,9 +25,7 @@ def _serialize(self, value, attr, obj): ret.append(getattr(item, self.field_name)) return ret else: - if value is None: - return [] - return getattr(value, self.field_name) + return getattr(value, self.field_name, None) class SelfNestedField(fields.Field): diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 080c6ef3eae..4e924756c94 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -87,7 +87,7 @@ def test_handles_vuln_with_no_creator(self, session.commit() res = test_client.get(self.url(vuln)) assert res.status_code == 200 - assert res.json['owner'] == [] + assert res.json['owner'] is None def test_shows_policy_violations(self, workspace, test_client, session, policy_violation_factory): From 9992bc5369b4f56b901e9400c87f61be64d3dbb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 19 Oct 2017 17:49:12 -0300 Subject: [PATCH 0373/1506] Fix some issues with the marshmallow shemas - Fix compatibility issues with dev version marshmallow Related to https://github.com/marshmallow-code/marshmallow/issues/328 - Set parent and parent_type required - Improve support for partial loading schemas (although we don't use them yet) - Fix potential bug in PrimaryKeyRelatedField It only happens with the field name is in `dir(NoneType)` --- server/api/modules/vulns.py | 29 ++++++++++++++++++----------- server/schemas.py | 4 +++- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index e3831c57efd..078b3f290b3 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -61,7 +61,7 @@ class VulnerabilitySchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') _rev = fields.String(default='') - _attachments = fields.Method('get_attachments') + _attachments = fields.Method('get_attachments', 'set_attachments', default=[]) owned = fields.Boolean(dump_only=True, default=False) owner = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator') impact = fields.Method('get_impact', deserialize='load_impact') @@ -70,8 +70,8 @@ class VulnerabilitySchema(AutoSchema): desc = fields.String(dump_only=True, attribute='description') refs = PrimaryKeyRelatedField('name', many=True, attribute='references', default=[]) issuetracker = fields.Method('get_issuetracker') - parent = fields.Method('get_parent', deserialize='load_parent') - parent_type = fields.Method('get_parent_type') + parent = fields.Method('get_parent', deserialize='load_parent', required=True) + parent_type = fields.String(required=True) tags = fields.Method('get_tags') easeofresolution = fields.String(dump_only=True, attribute='ease_of_resolution') hostnames = PrimaryKeyRelatedField('name', many=True) @@ -101,9 +101,6 @@ class Meta: def get_type(self, obj): return obj.__class__.__name__ - def get_parent_type(self, obj): - return obj.parent_type - def get_metadata(self, obj): return { "command_id": "e1a042dd0e054c1495e1c01ced856438", @@ -127,6 +124,9 @@ def get_attachments(self, obj): return res + def set_attachments(self, obj): + return obj + def get_hostnames(self, obj): # TODO: improve performance here # TODO: move this to models? @@ -147,6 +147,9 @@ def get_tags(self, obj): def get_parent(self, obj): return obj.parent.id + def load_parent(self, obj): + return obj + def get_status(self, obj): return obj.status @@ -180,7 +183,7 @@ def load_parent(self, value): @post_load def set_impact(self, data): - impact = data.pop('impact') + impact = data.pop('impact', None) if impact: pass return data @@ -189,8 +192,11 @@ def set_impact(self, data): def set_parent(self, data): # schema guarantees that parent_type exists. parent_class = None - parent_type = data.pop('parent_type') - parent_id = data.pop('parent') + parent_type = data.pop('parent_type', None) + parent_id = data.pop('parent', None) + if not (parent_type and parent_id): + # Probably a partial load, since they are required + return if parent_type == 'Host': parent_class = Host parent_field = 'host_id' @@ -198,11 +204,12 @@ def set_parent(self, data): parent_class = Service parent_field = 'service_id' if not parent_class: - raise Exception('Bad data') + print('parent_type', parent_type) + raise ValidationError('Unknown parent type') parent = db.session.query(parent_class).filter_by(id=parent_id).first() if not parent: - raise Exception('Parent not found') + raise ValidationError('Parent not found') data[parent_field] = parent.id return data diff --git a/server/schemas.py b/server/schemas.py index cbc6eb2e87f..c4b3f3776f6 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -25,7 +25,9 @@ def _serialize(self, value, attr, obj): ret.append(getattr(item, self.field_name)) return ret else: - return getattr(value, self.field_name, None) + if value is None: + return None + return getattr(value, self.field_name) class SelfNestedField(fields.Field): From 4ae537f18b30949862d7f1f014b0584e826e3b7e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 19 Oct 2017 18:01:59 -0300 Subject: [PATCH 0374/1506] Add impact with tests --- server/api/modules/vulns.py | 8 +++++-- test_cases/test_api_vulnerability.py | 35 +++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 078b3f290b3..df9380a7137 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -165,7 +165,7 @@ def get_issuetracker(self, obj): return {} def load_impact(self, value): - pass + return value def load_status(self, value): if value == 'opened': @@ -185,7 +185,10 @@ def load_parent(self, value): def set_impact(self, data): impact = data.pop('impact', None) if impact: - pass + data['impact_accountability'] = impact['accountability'] + data['impact_availability'] = impact['availability'] + data['impact_confidentiality'] = impact['confidentiality'] + data['impact_integrity'] = impact['integrity'] return data @post_load @@ -306,6 +309,7 @@ def _perform_create(self, data, **kwargs): filename = os.path.basename(filename), content=faraday_file, ) + db.session.commit() return obj @property diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 4e924756c94..1513807ce76 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -126,7 +126,10 @@ def test_hostnames(self, host_with_hostnames, test_client, session): def _create_post_data_vulnerability(self, name, vuln_type, parent_id, parent_type, refs, policyviolations, - attachments=None): + attachments=None, impact=None): + if not impact: + impact = {'accountability': False, 'availability': False, 'confidentiality': False, + 'integrity': False} data = {'metadata': {'update_time': 1508254070.211, 'update_user': '', 'update_action': 0, 'creator': 'UI Web', 'create_time': 1508254070.211, 'update_controller_action': 'UI Web New', 'owner': ''}, 'obj_id': '5a60af7f01dde6d3acfa8e9d3bef265c361a49d2', @@ -139,8 +142,7 @@ def _create_post_data_vulnerability(self, name, vuln_type, parent_id, 'data': '', 'desc': 'fsdfsd', 'easeofresolution': '', - 'impact': {'accountability': False, 'availability': False, 'confidentiality': False, - 'integrity': False}, + 'impact': impact, 'name': name, 'owned': False, 'policyviolations': policyviolations, @@ -339,3 +341,30 @@ def test_create_vuln_with_policyviolations(self, host_with_hostnames, test_clien assert res.status_code == 201 assert session.query(PolicyViolation).count() == 1 assert vuln_count_previous + 1 == session.query(Vulnerability).count() + + def test_create_vuln_imapct_verification(self, host_with_hostnames, test_client, session): + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='Vulnerability', + parent_id=host_with_hostnames.id, + parent_type='Host', + refs=[], + policyviolations=[], + impact={ + 'accountability': True, + 'availability': True, + 'confidentiality': True, + 'integrity': True + } + ) + ws_name = host_with_hostnames.workspace.name + vuln_count_previous = session.query(Vulnerability).count() + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) + assert res.status_code == 201 + assert vuln_count_previous + 1 == session.query(Vulnerability).count() + assert res.json['name'] == 'New vulns' + assert res.json['impact'] == {u'accountability': True, + u'availability': True, + u'confidentiality': True, + u'integrity': True} \ No newline at end of file From 35f924e8f124feb08ed1f9e4658f2442ec84114a Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 19 Oct 2017 18:17:41 -0300 Subject: [PATCH 0375/1506] Add support for non workspaced items in Server API The AngularJS factory only had support for objects that belong to a Workspace. Added methods for objs that do not belong to a ws (for example, Licenses, Tags, Vuln Templates). --- .../www/scripts/commons/providers/server.js | 85 ++++++++++++++++--- 1 file changed, 74 insertions(+), 11 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index 880feed87f9..3cb6ac31bd8 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -25,7 +25,7 @@ angular.module("faradayApp") return APIURL + "ws/" + wsName + objectName + "/"; }; - // created GET URL for objects that do not belong to a workspace + // create GET URL for objects that do not belong to a workspace var createNonWorkspacedGetUrl = function(objectName, objectId) { if (typeof objectId == 'string' || typeof objectId == "number") { objectName = objectName + "/" + objectId; @@ -42,16 +42,28 @@ angular.module("faradayApp") return APIURL + "ws/" + wsName + "/" + objectType + "/"; }; + // create POST URL for objects that do not belong to a workspace + var createNonWorkspacedPostUrl = function(objectId, objectType) { + return APIURL + objectType + "/"; + }; + var createPutUrl = function(wsName, objectId, objectType) { return APIURL + "ws/" + wsName + "/" + objectType + "/" + objectId + "/"; }; + // create PUT URL for objects that do not belong to a workspace + var createNonWorkspacedPutUrl = function(objectId, objectType) { + return APIURL + objectType + "/" + objectId + "/"; + }; + var createDbUrl = function(wsName) { return APIURL + "ws/" + wsName; } var createDeleteUrl = createPutUrl; + var createNonWorkspacedDeleteUrl = createNonWorkspacedPutUrl; + var serverComm = function(method, url, data) { var success = function (response) { return response; @@ -127,19 +139,31 @@ angular.module("faradayApp") } var modVulnWeb = function(createOrUpdate, wsName, vulnWeb) { - if (typeof vulnWeb.owner === "undefined") {vuln.owner = ""}; - if (typeof vuln.description === "undefined") {vuln.description = ""}; - if (typeof vulnWeb.protocol === "undefined") {vuln.protocol = ""}; - if (typeof vulnWeb.status === "undefined") {vuln.status = ""}; - if (typeof vulnWeb.version === "undefined") {vuln.version = ""}; - if (typeof vulnWeb.confirmed === "undefined") {vuln.confirmed = false}; - if (typeof vulnWeb.data === "undefined") {vuln.data = ""}; - if (typeof vulnWeb.severity === "undefined") {vuln.severity = "info"}; - if (typeof vulnWeb.resolution === "undefined") {vuln.resolution = ""}; - if (typeof vulnWeb.params === "undefined") {vuln.parmas = ""}; + if (typeof vulnWeb.owner === "undefined") {vulnWeb.owner = ""}; + if (typeof vulnWeb.description === "undefined") {vulnWeb.description = ""}; + if (typeof vulnWeb.protocol === "undefined") {vulnWeb.protocol = ""}; + if (typeof vulnWeb.status === "undefined") {vulnWeb.status = ""}; + if (typeof vulnWeb.version === "undefined") {vulnWeb.version = ""}; + if (typeof vulnWeb.confirmed === "undefined") {vulnWeb.confirmed = false}; + if (typeof vulnWeb.data === "undefined") {vulnWeb.data = ""}; + if (typeof vulnWeb.severity === "undefined") {vulnWeb.severity = "info"}; + if (typeof vulnWeb.resolution === "undefined") {vulnWeb.resolution = ""}; + if (typeof vulnWeb.params === "undefined") {vulnWeb.params = ""}; return createOrUpdate(wsName, vulnWeb._id, vulnWeb); } + var modVulnerabilityTemplate = function(createOrUpdate, vulnerabilityTemplate) { + var data_name = 'vulnerability_template'; + if(typeof vulnerabilityTemplate.cwe === "undefined") {vulnerabilityTemplate.cwe = ""}; + if(typeof vulnerabilityTemplate.description === "undefined") {vulnerabilityTemplate.description = ""}; + if(typeof vulnerabilityTemplate.exploitation === "undefined") {vulnerabilityTemplate.exploitation = "informational"}; + if(typeof vulnerabilityTemplate.name === "undefined") {vulnerabilityTemplate.name = ""}; + if(typeof vulnerabilityTemplate.references === "undefined") {vulnerabilityTemplate.references = ""}; + if(typeof vulnerabilityTemplate.resolution === "undefined") {vulnerabilityTemplate.resolution = ""}; + vulnerabilityTemplate.type = data_name; + return createOrUpdate(vulnerabilityTemplate._id, vulnerabilityTemplate, data_name); + } + var modNote = function(createOrUpdate, wsName, note) { if (typeof note.owner === "undefined") {note.owner = ""}; if (typeof note.description === "undefined") {note.description = ""}; @@ -161,21 +185,42 @@ angular.module("faradayApp") return send_data(postUrl, data, false, "POST"); } + var createNonWorkspacedObject = function(id, data, collectionName) { + var postUrl = createNonWorkspacedPostUrl(id, collectionName); + console.log(collectionName); + return send_data(postUrl, data, false, "POST"); + }; + var updateObject = function(wsName, id, data, collectionName) { var postUrl = createPostUrl(wsName, id, collectionName); return send_data(postUrl, data, true, "PUT"); } + var updateNonWorkspacedObject = function(id, data, collectionName) { + var postUrl = createNonWorkspacedPostUrl(id, collectionName); + return send_data(postUrl, data, true, "PUT"); + }; + var saveInServer = function(wsName, objectId, data, collectionName) { var postUrl = createPostUrl(wsName, objectId, collectionName); return send_data(postUrl, data, false, "PUT"); } + var saveNonWorkspacedInServer = function(objectId, data, collectionName) { + var postUrl = createNonWorkspacedPostUrl(objectId, collectionName); + return send_data(postUrl, data, false, "PUT"); + }; + var updateInServer = function(wsName, objectId, data, collectionName) { var postUrl = createPostUrl(wsName, objectId, collectionName); return send_data(postUrl, objectId, true, "PUT"); } + var updateNonWorkspacedInServer = function(objectId, data, collectionName) { + var postUrl = createPostUrl(objectId, collectionName); + return send_data(postUrl, objectId, true, "PUT"); + }; + ServerAPI.getHost = function(wsName, objId) { var url = createGetUrl(wsName, 'hosts', objId); return get(url); @@ -333,6 +378,14 @@ angular.module("faradayApp") return modVulnWeb(updateObject, wsName, vulnWeb); } + ServerAPI.createVulnerabilityTemplate = function(vulnerabilityTemplate) { + return modVulnerabilityTemplate(createNonWorkspacedObject, vulnerabilityTemplate); + }; + + ServerAPI.updateVulnerabilityTemplate = function(vulnerabilityTemplate) { + return modVulnerabilityTemplate(updateNonWorkspacedObject, vulnerabilityTemplate); + }; + ServerAPI.createNote = function(wsName, note) { return modNote(createObject, wsName, note); } @@ -392,6 +445,16 @@ angular.module("faradayApp") } } + ServerAPI.deleteVulnerabilityTemplate = function(vulnId, rev) { + var deleteUrl = createNonWorkspacedDeleteUrl(vulnId, rev); + if (typeof rev === "undefined") { + return _delete(deleteUrl, false) + } + else { + return _delete(deleteUrl, true); + } + } + ServerAPI.deleteNote = function(wsName, noteId, rev) { var deleteUrl = createDeleteUrl(wsName, noteId, rev); if (typeof rev === "undefined") { From a879ed89778e021c390db1060944849e5ed93637 Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 19 Oct 2017 18:19:38 -0300 Subject: [PATCH 0376/1506] Fix new Vuln Template modal validation The partial wasn't correctly validating the input. Fixed that. --- .../www/scripts/vulndb/partials/modalNew.html | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/server/www/scripts/vulndb/partials/modalNew.html b/server/www/scripts/vulndb/partials/modalNew.html index 91531007ada..e7d053513ea 100644 --- a/server/www/scripts/vulndb/partials/modalNew.html +++ b/server/www/scripts/vulndb/partials/modalNew.html @@ -5,7 +5,7 @@
    - +

    New Vulnerability Model

    @@ -18,7 +18,7 @@

    New Vulnerability Model

    -

    +

    Vulnerability Model name is required

    @@ -49,21 +49,27 @@

    New Vulnerability Model

    - - -
    +
    +
    +

    + Vulnerability Model exploitation is required +

    +

    All fields marked * are required

    - +
    From 0fd36e534e1a284e10ead969e5cd972f9ee9ee77 Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 19 Oct 2017 18:22:05 -0300 Subject: [PATCH 0377/1506] Fix indentation TOC. --- .../www/scripts/vulndb/providers/vulnModels.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/server/www/scripts/vulndb/providers/vulnModels.js b/server/www/scripts/vulndb/providers/vulnModels.js index 64381bb80ae..57c61991c8e 100644 --- a/server/www/scripts/vulndb/providers/vulnModels.js +++ b/server/www/scripts/vulndb/providers/vulnModels.js @@ -17,18 +17,18 @@ angular.module('faradayApp'). try { var vulnModel = new VulnModel(data); - vulnModel.save(). - then(function(resp) { + vulnModel.save() + .then(function(resp) { if (outsider) { deferred.resolve(resp); } else { - vulnModelsManager.get(). - then(function() { - self.updateState(self.totalNumberOfModels + 1); - deferred.resolve(self); - }, function(reason) { - deferred.reject(reason); - }); + vulnModelsManager.get() + .then(function() { + self.updateState(self.totalNumberOfModels + 1); + deferred.resolve(self); + }, function(reason) { + deferred.reject(reason); + }); }}, function(reason) { deferred.reject(reason); }); From eac8595fbe6eee1d06a1539566814430b28369df Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 19 Oct 2017 18:22:42 -0300 Subject: [PATCH 0378/1506] Remove calls to the Vuln Templates DB in Couch Use the newly added ServerAPI methods. --- .../www/scripts/vulndb/providers/vulnModel.js | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/server/www/scripts/vulndb/providers/vulnModel.js b/server/www/scripts/vulndb/providers/vulnModel.js index a040d97495b..15e645fdc03 100644 --- a/server/www/scripts/vulndb/providers/vulnModel.js +++ b/server/www/scripts/vulndb/providers/vulnModel.js @@ -3,17 +3,17 @@ // See the file 'doc/LICENSE' for the license information angular.module('faradayApp'). - factory('VulnModel', ['BASEURL', 'configSrv', '$http', '$q', - function(BASEURL, configSrv, $http, $q) { + factory('VulnModel', ['BASEURL', 'configSrv', 'ServerAPI', '$http', '$q', + function(BASEURL, configSrv, ServerAPI, $http, $q) { function VulnModel(data) { this._id = ""; this._rev = ""; + this.cwe = ""; + this.description = ""; this.exploitation = ""; - this.references = []; this.name = ""; + this.references = []; this.resolution = ""; - this.cwe = ""; - this.description = ""; if (data) { if(data.name === undefined || data.name === "") { throw new Error("Unable to create a Vulnerability Model whithout a name"); @@ -93,26 +93,27 @@ angular.module('faradayApp'). delete this._id; delete this._rev; - configSrv.promise. - then(function() { - var url = BASEURL + configSrv.vulnModelsDB; - - $http.post(url, self). - then(function(data) { - self._id = data.id; - self._rev = data.rev; - deferred.resolve(self); - }, function(res) { - try { - deferred.reject("Unable to save the Vuln Model. " + res.data.reason); - } catch(err) { - deferred.reject(err); + ServerAPI.createVulnerabilityTemplate(self) + .then(function(data) { + self._id = data.id; + self._rev = data.rev; + deferred.resolve(self); + }, function(res) { + try { + var msg = ''; + for(var item in res.data.messages) { + if(res.data.messages.hasOwnProperty(item)) { + msg += item.charAt(0).toUpperCase() + item.slice(1) + ": "; + msg += res.data.messages[item][0]; } - }); - }, function(reason) { - deferred.reject(reason); + } + deferred.reject("Unable to save the Vuln Model. " + msg); + } catch(err) { + deferred.reject(err); + } }); + return deferred.promise; } }; From 3240f988610c1f7d6db63786b8bf9f8d411a9073 Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 19 Oct 2017 18:23:30 -0300 Subject: [PATCH 0379/1506] Fix Vuln Templates API Now it returns all the data that is present in the DB. Also adds filtering support, etc. Also validates exploitation field, which was previously causing issues. --- server/api/modules/vulnerability_template.py | 33 +++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/server/api/modules/vulnerability_template.py b/server/api/modules/vulnerability_template.py index 5cae10682cf..9956fbeef84 100644 --- a/server/api/modules/vulnerability_template.py +++ b/server/api/modules/vulnerability_template.py @@ -4,11 +4,24 @@ import time from flask import Blueprint +from filteralchemy import ( + FilterSet, + operators, +) from marshmallow import fields +from marshmallow.validate import ( + OneOf, +) import server.utils.logger -from server.api.base import AutoSchema, ReadWriteView +from server.api.base import ( + AutoSchema, + FilterAlchemyMixin, + FilterSetMeta, + PaginatedMixin, + ReadWriteView, +) from server.models import VulnerabilityTemplate from server.schemas import PrimaryKeyRelatedField @@ -19,8 +32,8 @@ class VulnerabilityTemplateSchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') _rev = fields.String(default='', dump_only=True) - cwe = fields.String(dump_only=True, default='') # this field was deprecated and the legacy data is added to references on import - exploitation = fields.String(dump_only=True, attribute='severity') + cwe = fields.String(dump_only=True, default='') # deprecated field, the legacy data is added to refs on import + exploitation = fields.String(attribute='severity', validate=OneOf(VulnerabilityTemplate.SEVERITIES), required=True) references = PrimaryKeyRelatedField('name', many=True, attribute='references') class Meta: @@ -28,10 +41,21 @@ class Meta: fields = ('id', '_id', '_rev', 'cwe', 'description', 'exploitation', 'name', 'references', 'resolution') -class VulnerabilityTemplateView(ReadWriteView): +class VulnerabilityTemplateFilterSet(FilterSet): + class Meta(FilterSetMeta): + model = VulnerabilityTemplate # It has all the fields + fields = ( + 'severity') + operators = (operators.Equal,) + + +class VulnerabilityTemplateView(PaginatedMixin, + FilterAlchemyMixin, + ReadWriteView): route_base = 'vulnerability_template' model_class = VulnerabilityTemplate schema_class = VulnerabilityTemplateSchema + filterset_class = VulnerabilityTemplateFilterSet def _envelope_list(self, objects, pagination_metadata=None): vuln_tpls = [] @@ -47,3 +71,4 @@ def _envelope_list(self, objects, pagination_metadata=None): } VulnerabilityTemplateView.register(vulnerability_template_api) + From 24b9dca84dd1b870beacb4d7bca37f832fa0e7e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 19 Oct 2017 19:03:18 -0300 Subject: [PATCH 0380/1506] Add implementation of MutableField This helper field will make some schemas development easier --- server/schemas.py | 23 ++++++++++++++++ test_cases/test_marshmallow_fields.py | 39 ++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/server/schemas.py b/server/schemas.py index c4b3f3776f6..a5c39a15be4 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -50,3 +50,26 @@ def _serialize(self, value, attr, obj): def _deserialize(self, value, attr, data): raise NotImplementedError("Only dump is implemented for now") + + +class MutableField(fields.Field): + """ + A field that enables the use of different fields for read and write. + + This is useful in many cases, like for example when you want to use a + Nested field to show the data but an Integer field (that uses to be a + primary key) for writing/deserializing. + """ + + # TODO: inherit required and other properties from the child fields + + def __init__(self, read_field, write_field, **kwargs): + self.read_field = read_field + self.write_field = write_field + super(MutableField, self).__init__(**kwargs) + + def _serialize(self, value, attr, obj): + return self.read_field._serialize(value, attr, obj) + + def _deserialize(self, value, attr, data): + return self.write_field._deserialize(value, attr, data) diff --git a/test_cases/test_marshmallow_fields.py b/test_cases/test_marshmallow_fields.py index d79cae44d22..b14fb9da47f 100644 --- a/test_cases/test_marshmallow_fields.py +++ b/test_cases/test_marshmallow_fields.py @@ -2,9 +2,10 @@ import datetime import pytest from collections import namedtuple -from marshmallow import Schema, fields +from marshmallow import Schema, fields, ValidationError from server.schemas import ( JSTimestampField, + MutableField, PrimaryKeyRelatedField, SelfNestedField, ) @@ -92,3 +93,39 @@ def test_single_with_none_value(self): "user": None, "first_name": "other" } + + +Blogpost2 = namedtuple('Blogpost', ['id', 'title', 'user']) + + +class Blogpost2Schema(Schema): + id = fields.Integer() + title = fields.String() + user = MutableField(fields.Nested(UserSchema, only=('username',)), + fields.String()) + +class TestMutableField: + + serialized_data = {"id": 1, "title": "test", "user": {"username": "john"}} + loaded_data = {"id": 1, "title": "test", "user": "john"} + + @pytest.fixture(autouse=True) + def load_data(self): + self.user = User('john', []) # I don't care for the user's blogposts + self.blogpost = Blogpost2(1, 'test', self.user) + + def serialize(self, obj=None, schema=Blogpost2Schema): + return schema(strict=True).dump(obj or self.blogpost).data + + def load(self, data, schema=Blogpost2Schema): + return schema(strict=True).load(data).data + + def test_serialize(self): + assert self.serialize() == self.serialized_data + + def test_deserialize(self): + assert self.load(self.loaded_data) == self.loaded_data + + def test_deserialize_fails(self): + with pytest.raises(ValidationError): + self.load(self.serialized_data) From 3eb13c8f04140b94446eac4070fdccf1c214dc23 Mon Sep 17 00:00:00 2001 From: micabot Date: Fri, 20 Oct 2017 14:34:09 -0300 Subject: [PATCH 0381/1506] Add non-functional test case for Vuln Templates --- test_cases/factories.py | 17 +++++++++++++++++ test_cases/test_api_non_workspaced_base.py | 6 ++++++ 2 files changed, 23 insertions(+) diff --git a/test_cases/factories.py b/test_cases/factories.py index 4c212d1cb9e..e703ce60565 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -1,6 +1,7 @@ import random import factory import datetime +import unicodedata import pytz from factory.fuzzy import ( @@ -25,6 +26,7 @@ User, Vulnerability, VulnerabilityCode, + VulnerabilityTemplate, VulnerabilityWeb, Workspace, ) @@ -43,6 +45,8 @@ ) ) +all_unicode = ''.join(unichr(i) for i in xrange(65536)) +UNICODE_LETTERS = ''.join(c for c in all_unicode if unicodedata.category(c) == 'Lu' or unicodedata.category(c) == 'Ll') class FaradayFactory(factory.alchemy.SQLAlchemyModelFactory): @@ -231,6 +235,19 @@ class Meta: sqlalchemy_session = db.session +class VulnerabilityTemplateFactory(FaradayFactory): + # name = FuzzyText(chars=UNICODE_LETTERS) + # description = FuzzyText(chars=UNICODE_LETTERS) + name = FuzzyText() + description = FuzzyText() + severity = FuzzyChoice(VulnerabilityTemplate.SEVERITIES) + creator = factory.SubFactory(UserFactory) + + class Meta: + model = VulnerabilityTemplate + sqlalchemy_session = db.session + + class CredentialFactory(HasParentHostOrService, WorkspaceObjectFactory): host = factory.SubFactory(HostFactory) service = factory.SubFactory(ServiceFactory) diff --git a/test_cases/test_api_non_workspaced_base.py b/test_cases/test_api_non_workspaced_base.py index 9f47c6db5a2..0ff529db904 100644 --- a/test_cases/test_api_non_workspaced_base.py +++ b/test_cases/test_api_non_workspaced_base.py @@ -143,3 +143,9 @@ class ReadWriteTestsMixin(ListTestsMixin, class ReadWriteAPITests(ReadWriteTestsMixin, GenericAPITest): pass + + +class ReadOnlyAPITests(ListTestsMixin, + RetrieveTestsMixin, + GenericAPITest): + pass \ No newline at end of file From 82e1a205125fc59c24e46a4267d3b718a41f5030 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 20 Oct 2017 18:12:28 -0300 Subject: [PATCH 0382/1506] Add tests for vulnerability template --- server/api/modules/vulnerability_template.py | 3 +- test_cases/test_api_non_workspaced_base.py | 6 +- test_cases/test_api_vulnerability.py | 5 +- test_cases/test_api_vulnerability_template.py | 107 ++++++++++++++++++ 4 files changed, 114 insertions(+), 7 deletions(-) create mode 100644 test_cases/test_api_vulnerability_template.py diff --git a/server/api/modules/vulnerability_template.py b/server/api/modules/vulnerability_template.py index 9956fbeef84..820cb92e736 100644 --- a/server/api/modules/vulnerability_template.py +++ b/server/api/modules/vulnerability_template.py @@ -70,5 +70,4 @@ def _envelope_list(self, objects, pagination_metadata=None): 'rows': vuln_tpls, } -VulnerabilityTemplateView.register(vulnerability_template_api) - +VulnerabilityTemplateView.register(vulnerability_template_api) \ No newline at end of file diff --git a/test_cases/test_api_non_workspaced_base.py b/test_cases/test_api_non_workspaced_base.py index 0ff529db904..fe00954e4a0 100644 --- a/test_cases/test_api_non_workspaced_base.py +++ b/test_cases/test_api_non_workspaced_base.py @@ -49,7 +49,11 @@ def test_list_retrieves_all_items_from(self, test_client, session): res = test_client.get(self.url()) assert res.status_code == 200 - assert len(res.json) == OBJECT_COUNT + if 'rows' in res.json: + assert len(res.json['rows']) == OBJECT_COUNT + else: + assert len(res.json) == OBJECT_COUNT + class RetrieveTestsMixin: diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 1513807ce76..fd68f6de023 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1,5 +1,4 @@ #-*- coding: utf8 -*- -'''Tests for many API endpoints that do not depend on workspace_name''' import os from base64 import b64encode @@ -7,9 +6,7 @@ from server.api.modules.vulns import VulnerabilityView from test_cases import factories -from test_api_workspaced_base import ( - API_PREFIX, - ListTestsMixin, +from test_api_non_workspaced_base import ( ReadOnlyAPITests ) from server.models import ( diff --git a/test_cases/test_api_vulnerability_template.py b/test_cases/test_api_vulnerability_template.py new file mode 100644 index 00000000000..e841d1b75f6 --- /dev/null +++ b/test_cases/test_api_vulnerability_template.py @@ -0,0 +1,107 @@ +#-*- coding: utf8 -*- +import os + +import pytest + +from server.api.modules.vulnerability_template import VulnerabilityTemplateView +from test_cases import factories +from test_api_non_workspaced_base import ( + ReadOnlyAPITests +) +from server.models import ( + VulnerabilityTemplate, +) +from test_cases.factories import VulnerabilityTemplateFactory + +CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) + + +@pytest.mark.usefixtures('logged_user') +class TestListVulnerabilityTemplateView(ReadOnlyAPITests): + model = VulnerabilityTemplate + factory = factories.VulnerabilityTemplateFactory + api_endpoint = 'vulnerability_template' + #unique_fields = ['ip'] + #update_fields = ['ip', 'description', 'os'] + view_class = VulnerabilityTemplateView + + def test_backwards_json_compatibility(self, test_client, session): + self.factory.create() + session.commit() + res = test_client.get(self.url()) + assert res.status_code == 200 + assert 'rows' in res.json + for vuln in res.json['rows']: + assert set([u'id', u'key', u'value', u'doc']) == set(vuln.keys()) + object_properties = [ + u'exploitation', + u'references', + u'name', + u'cwe', + u'_rev', + u'_id', + u'resolution', + u'description' + ] + + expected = set(object_properties) + result = set(vuln['doc'].keys()) + assert expected - result == set() + + def _create_post_data_vulnerability_template(self, references): + data = { + "exploitation":"high", + "references":references, + "name":"name", + "resolution": "resolution", + "cwe":"swe", + "description":"desc"} + return data + + def test_create_new_vulnerability_template(self, session, test_client): + vuln_count_previous = session.query(VulnerabilityTemplate).count() + raw_data = self._create_post_data_vulnerability_template(references=[]) + res = test_client.post('/v2/vulnerability_template/', data=raw_data) + assert res.status_code == 201 + assert res.json['_id'] == 6 + assert vuln_count_previous + 1 == session.query(VulnerabilityTemplate).count() + + def test_update_vulnerability_template(self, session, test_client): + template = self.factory.create() + session.commit() + raw_data = self._create_post_data_vulnerability_template(references=[]) + res = test_client.put('/v2/vulnerability_template/{0}/'.format(template.id), data=raw_data) + assert res.status_code == 200 + updated_template = session.query(VulnerabilityTemplate).filter_by(id=template.id).first() + assert updated_template.name == raw_data['name'] + assert updated_template.severity == raw_data['exploitation'] + assert updated_template.resolution == raw_data['resolution'] + assert updated_template.description == raw_data['description'] + assert updated_template.references == [] + + def test_update_vulnerabiliy_template_change_refs(self, session, test_client): + template = self.factory.create() + raw_data = self._create_post_data_vulnerability_template(references=['new_ref', 'another_ref']) + res = test_client.put('/v2/vulnerability_template/{0}/'.format(template.id), data=raw_data) + assert res.status_code == 200 + updated_template = session.query(VulnerabilityTemplate).filter_by(id=template.id).first() + assert updated_template.name == raw_data['name'] + assert updated_template.severity == raw_data['exploitation'] + assert updated_template.resolution == raw_data['resolution'] + assert updated_template.description == raw_data['description'] + assert updated_template.references == [] + + def test_create_new_vulnerability_template_with_references(self, session, test_client): + vuln_count_previous = session.query(VulnerabilityTemplate).count() + raw_data = self._create_post_data_vulnerability_template(references=['ref1', 'ref2']) + res = test_client.post('/v2/vulnerability_template/', data=raw_data) + assert res.status_code == 201 + assert res.json['_id'] == 6 + assert vuln_count_previous + 1 == session.query(VulnerabilityTemplate).count() + + def test_delete_vuln_template(self, session, test_client): + template = self.factory.create() + vuln_count_previous = session.query(VulnerabilityTemplate).count() + res = test_client.delete('/v2/vulnerability_template/{0}/'.format(template.id)) + assert res.status_code == 204 + assert vuln_count_previous - 1 == session.query(VulnerabilityTemplate).count() \ No newline at end of file From 0895a6673787abc4301a7285d946fff713505d4d Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 20 Oct 2017 19:18:45 -0300 Subject: [PATCH 0383/1506] Update js code to use the new api endpoints --- .../www/scripts/commons/providers/server.js | 4 +- .../www/scripts/vulndb/providers/vulnModel.js | 41 ++++++------------- 2 files changed, 15 insertions(+), 30 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index 3cb6ac31bd8..c92203f1e90 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -197,7 +197,7 @@ angular.module("faradayApp") } var updateNonWorkspacedObject = function(id, data, collectionName) { - var postUrl = createNonWorkspacedPostUrl(id, collectionName); + var postUrl = createNonWorkspacedPutUrl(id, collectionName); return send_data(postUrl, data, true, "PUT"); }; @@ -446,7 +446,7 @@ angular.module("faradayApp") } ServerAPI.deleteVulnerabilityTemplate = function(vulnId, rev) { - var deleteUrl = createNonWorkspacedDeleteUrl(vulnId, rev); + var deleteUrl = createNonWorkspacedDeleteUrl(vulnId, 'vulnerability_template'); if (typeof rev === "undefined") { return _delete(deleteUrl, false) } diff --git a/server/www/scripts/vulndb/providers/vulnModel.js b/server/www/scripts/vulndb/providers/vulnModel.js index 15e645fdc03..9346136bac0 100644 --- a/server/www/scripts/vulndb/providers/vulnModel.js +++ b/server/www/scripts/vulndb/providers/vulnModel.js @@ -47,19 +47,12 @@ angular.module('faradayApp'). var deferred = $q.defer(); var self = this; - configSrv.promise. - then(function() { - var url = BASEURL + configSrv.vulnModelsDB + "/" + self._id + "?rev=" + self._rev; - - $http.delete(url). - then(function(resp) { - deferred.resolve(resp); - }, function(data, status, headers, config) { - deferred.reject("Unable to delete Vuln Model from DB. " + status) - }); - }, function(reason) { - deferred.reject(reason); - }); + ServerAPI.deleteVulnerabilityTemplate(self._id) + .then(function(resp) { + deferred.resolve(resp); + }, function(data, status, headers, config) { + deferred.reject("Unable to delete Vuln Model from DB. " + status) + }); return deferred.promise; }, @@ -68,21 +61,13 @@ angular.module('faradayApp'). var deferred = $q.defer(); var self = this; - configSrv.promise. - then(function() { - var url = BASEURL + configSrv.vulnModelsDB + "/" + self._id; - - $http.put(url, data). - then(function(res) { - self.set(res.data); - self._rev = res.data.rev; - deferred.resolve(self); - }, function(res) { - deferred.reject("Unable to update the Vuln Model. " + res.data.reason); - }); - }, function(reason) { - deferred.reject(reason); - }); + ServerAPI.updateVulnerabilityTemplate(self) + .then(function(res) { + self.set(res.data); + deferred.resolve(self); + }, function(res) { + deferred.reject("Unable to update the Vuln Model. " + res.data.reason); + }); return deferred.promise; }, From 1b0496e658966a762ce934bcdb7b32e1bb0d24be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 20 Oct 2017 19:33:25 -0300 Subject: [PATCH 0384/1506] First attempt on references association proxy --- server/models.py | 8 +++++-- test_cases/models/test_vulnerability.py | 29 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/server/models.py b/server/models.py index dcd5fd4b48c..78faf01d26a 100644 --- a/server/models.py +++ b/server/models.py @@ -23,6 +23,7 @@ from sqlalchemy import func from sqlalchemy.orm import column_property from sqlalchemy.schema import DDL +from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.ext.declarative import declared_attr from flask_sqlalchemy import ( SQLAlchemy as OriginalSQLAlchemy, @@ -338,11 +339,14 @@ class VulnerabilityGeneric(VulnerabilityABC): ) workspace = relationship('Workspace', backref='vulnerabilities') - references = relationship( + reference_instances = relationship( "Reference", - secondary="reference_vulnerability_association" + secondary="reference_vulnerability_association", + collection_class=set ) + references = association_proxy('reference_instances', 'name') + policy_violations = relationship( "PolicyViolation", secondary="policy_violation_vulnerability_association" diff --git a/test_cases/models/test_vulnerability.py b/test_cases/models/test_vulnerability.py index bb97b3be32a..f0db9ad5e7d 100644 --- a/test_cases/models/test_vulnerability.py +++ b/test_cases/models/test_vulnerability.py @@ -1,3 +1,6 @@ +import pytest +from server.models import Reference + def test_standard_host_vuln_hostnames(vulnerability_factory, host_with_hostnames, session): @@ -28,3 +31,29 @@ def test_code_vuln(vulnerability_code, session): session.commit() # Source code vulnerabilities have no hostnames assert vulnerability_code.hostnames == [] + + +class TestReferences: + @pytest.fixture(autouse=True) + def load_data(self, vulnerability_factory, session): + self.vuln = vulnerability_factory.create() + self.vuln2 = vulnerability_factory.create( + workspace=self.vuln.workspace) + self.vuln_different_ws = vulnerability_factory.create() + self.vulns = [self.vuln, self.vuln2, self.vuln_different_ws] + session.commit() + assert self.vuln.workspace_id != self.vuln_different_ws.workspace_id + + def test_empty_references(self): + for vuln in self.vulns: + assert isinstance(vuln.reference_instances, set) + assert len(vuln.references) == 0 + + def test_add_references(self, session): + self.vuln.references.add('CVE-2017-1234') + assert session.new + session.commion() + assert Reference.query.count() == 1 + ref = Reference.query.first() + assert ref.name == 'CVE-2017-1234' + assert ref.workspace_id == self.vuln.workspace_id From 296239d241651ac51677cd90e2b7bdfa1b3bfce2 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 23 Oct 2017 11:46:53 -0300 Subject: [PATCH 0385/1506] Add vuln type check --- server/api/base.py | 22 ++++++++++++++++++++++ server/api/modules/vulns.py | 7 +++++-- test_cases/test_api_vulnerability.py | 19 ++++++++++++++++++- 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 04bc8d089ea..7f52d4ca5b1 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -28,6 +28,22 @@ def output_json(data, code, headers=None): return response +class InvalidUsage(Exception): + status_code = 400 + + def __init__(self, message, status_code=None, payload=None): + Exception.__init__(self) + self.message = message + if status_code is not None: + self.status_code = status_code + self.payload = payload + + def to_dict(self): + rv = dict(self.payload or ()) + rv['message'] = self.message + return rv + + # TODO: Require @view decorator to enable custom routes class GenericView(FlaskView): """Abstract class to provide helpers. Inspired in Django REST @@ -102,6 +118,12 @@ def handle_unprocessable_entity(err): 'messages': messages, }), 400 + @app.errorhandler(InvalidUsage) + def handle_invalid_usage(error): + response = flask.jsonify(error.to_dict()) + response.status_code = error.status_code + return response + class GenericWorkspacedView(GenericView): """Abstract class for a view that depends on the workspace, that is diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index df9380a7137..ec5d18ee568 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -18,7 +18,7 @@ FilterSetMeta, PaginatedMixin, ReadWriteWorkspacedView, -) + InvalidUsage) from server.fields import FaradayUploadedFile from server.models import ( db, @@ -322,7 +322,10 @@ def model_class(self): def _get_schema_class(self): assert self.schema_class_dict is not None, "You must define schema_class" if request.method == 'POST': - return self.schema_class_dict[request.json['type']] + requested_type = request.json['type'] + if requested_type not in self.schema_class_dict: + raise InvalidUsage('Invalid vulnerability type.') + return self.schema_class_dict[requested_type] # We use web since it has all the fields return self.schema_class_dict['VulnerabilityWeb'] diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index fd68f6de023..c9caf12a8df 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -364,4 +364,21 @@ def test_create_vuln_imapct_verification(self, host_with_hostnames, test_client, assert res.json['impact'] == {u'accountability': True, u'availability': True, u'confidentiality': True, - u'integrity': True} \ No newline at end of file + u'integrity': True} + + def test_create_vuln_with_invalid_type(self, host_with_hostnames, test_client, session): + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='VulnerabilitySarasa', + parent_id=host_with_hostnames.id, + parent_type='Host', + refs=[], + policyviolations=[] + ) + ws_name = host_with_hostnames.workspace.name + vuln_count_previous = session.query(Vulnerability).count() + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) + assert res.status_code == 400 + assert vuln_count_previous == session.query(Vulnerability).count() + assert res.json['message'] == 'Invalid vulnerability type.' From b502d0d6a86c1d3eca1135f561e79737ffb307e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 23 Oct 2017 12:43:06 -0300 Subject: [PATCH 0386/1506] Make references addition work properly --- server/models.py | 55 ++++++++++++++++++++++++- test_cases/models/test_vulnerability.py | 38 ++++++++++++++++- 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/server/models.py b/server/models.py index 78faf01d26a..f36a952cf99 100644 --- a/server/models.py +++ b/server/models.py @@ -23,7 +23,7 @@ from sqlalchemy import func from sqlalchemy.orm import column_property from sqlalchemy.schema import DDL -from sqlalchemy.ext.associationproxy import association_proxy +from sqlalchemy.ext.associationproxy import association_proxy, _AssociationSet from sqlalchemy.ext.declarative import declared_attr from flask_sqlalchemy import ( SQLAlchemy as OriginalSQLAlchemy, @@ -313,6 +313,50 @@ class VulnerabilityTemplate(VulnerabilityABC): ) +class CustomAssociationSet(_AssociationSet): + """ + A custom associacion set that passes the creator method the both + the value and the instance of the parent object + """ + + # def __init__(self, lazy_collection, creator, getter, setter, parent): + def __init__(self, lazy_collection, creator, value_attr, parent): + """I have to override this method because the proxy_factory + class takes different arguments than the hardcoded + _AssociationSet one. + In particular, the getter and the setter aren't passed, but + since I have an instance of the parent (AssociationProxy + instance) I do the logic here. + The value_attr argument isn't relevant to this implementation + """ + + if parent.getset_factory: + getter, setter = parent.getset_factory( + parent.collection_class, parent) + else: + getter, setter = parent._default_getset(parent.collection_class) + + super(CustomAssociationSet, self).__init__( + lazy_collection, creator, getter, setter, parent) + + def _create(self, value): + parent_instance = self.lazy_collection.ref() + return self.creator(value, parent_instance) + + +def _reference_creator(name, vulnerability): + """Get or create a vulnerability with the corresponding name. + This must be worspace aware""" + reference = Reference.query.filter( + Reference.workspace_id == vulnerability.workspace_id, + Reference.name == name + ).first() + if reference is None: + # Doesn't exist + reference = Reference(name, vulnerability.workspace_id) + return reference + + class VulnerabilityGeneric(VulnerabilityABC): STATUSES = [ 'open', @@ -345,7 +389,9 @@ class VulnerabilityGeneric(VulnerabilityABC): collection_class=set ) - references = association_proxy('reference_instances', 'name') + references = association_proxy('reference_instances', 'name', + proxy_factory=CustomAssociationSet, + creator=_reference_creator) policy_violations = relationship( "PolicyViolation", @@ -501,6 +547,11 @@ class Reference(Metadata): UniqueConstraint('name', 'workspace_id', name='uix_reference_name_vulnerability_workspace'), ) + def __init__(self, name, workspace_id): + self.name = name + self.workspace_id = workspace_id + super(Reference, self).__init__() + class ReferenceVulnerabilityAssociation(db.Model): diff --git a/test_cases/models/test_vulnerability.py b/test_cases/models/test_vulnerability.py index f0db9ad5e7d..d8bdae6e3a4 100644 --- a/test_cases/models/test_vulnerability.py +++ b/test_cases/models/test_vulnerability.py @@ -44,6 +44,13 @@ def load_data(self, vulnerability_factory, session): session.commit() assert self.vuln.workspace_id != self.vuln_different_ws.workspace_id + @pytest.fixture + def reference(self, session): + reference = Reference('CVE-2017-1234', self.vuln.workspace_id) + session.add(reference) + session.commit() + return reference + def test_empty_references(self): for vuln in self.vulns: assert isinstance(vuln.reference_instances, set) @@ -52,8 +59,37 @@ def test_empty_references(self): def test_add_references(self, session): self.vuln.references.add('CVE-2017-1234') assert session.new - session.commion() + session.commit() assert Reference.query.count() == 1 ref = Reference.query.first() assert ref.name == 'CVE-2017-1234' assert ref.workspace_id == self.vuln.workspace_id + + # Re-adding the reference shouldn't do nothing + self.vuln.references.add('CVE-2017-1234') + assert Reference.query.count() == 1 + + def test_add_existing_reference(self, session, reference): + for vuln in [self.vuln, self.vuln2]: + vuln.references.add('CVE-2017-1234') + session.commit() + assert Reference.query.count() == 1 + assert len(vuln.references) == 1 + assert list(vuln.reference_instances)[0].id == reference.id + + @pytest.mark.parametrize('orphan_vuln', [True, False], + ids=['with_orphan_reference', + 'with_used_reference']) + def test_add_existing_from_other_workspace(self, session, reference, + orphan_vuln): + if not orphan_vuln: + self.vuln.references.add(reference.name) + session.commit() + self.vuln_different_ws.references.add(reference.name) + session.commit() + assert Reference.query.count() == 2 + assert len(self.vuln_different_ws.references) == 1 + new_reference = list(self.vuln_different_ws.reference_instances)[0] + assert (new_reference.workspace_id == + self.vuln_different_ws.workspace_id) + assert new_reference.id != reference.id From 7fdeb223d8c8f33ee540ab6df8286a167d8fd3c8 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 23 Oct 2017 13:47:40 -0300 Subject: [PATCH 0387/1506] Add credentials count for host --- server/api/modules/hosts.py | 2 +- server/models.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index dcd870720bc..898e8b326d1 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -32,7 +32,6 @@ class HostSchema(AutoSchema): _rev = fields.String(default='') ip = fields.String(default='') description = fields.String(required=True) # Explicitly set required=True - credentials = fields.Function(lambda host: len(host.credentials)) default_gateway = fields.List(fields.String, attribute="default_gateway_ip") metadata = fields.Method('get_metadata') name = fields.String(dump_only=True, attribute='ip', default='') @@ -41,6 +40,7 @@ class HostSchema(AutoSchema): owner = PrimaryKeyRelatedField('username', attribute='creator', dump_only=True) services = fields.Integer(attribute='service_count', dump_only=True) vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) + credentials = fields.Integer(attribute='vulnerability_count', dump_only=True) def get_metadata(self, obj): diff --git a/server/models.py b/server/models.py index dcd5fd4b48c..8ff16bfcb23 100644 --- a/server/models.py +++ b/server/models.py @@ -194,6 +194,7 @@ class Host(Metadata): service_count = _make_generic_count_property('host', 'service') vulnerability_count = _make_generic_count_property('host', 'vulnerability') + credentials_count = _make_generic_count_property('host', 'credential') __table_args__ = ( UniqueConstraint(ip, workspace_id, name='uix_host_ip_workspace'), From 6b03e979202cacf7692b82a1332a400a31390a44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 23 Oct 2017 15:11:59 -0300 Subject: [PATCH 0388/1506] Modify vulns API to use the new references implementation. I had to fix some issues with the workspace_id propagation --- server/api/modules/vulns.py | 9 ++-- server/models.py | 14 +++--- test_cases/models/test_vulnerability.py | 66 +++++++++++++++++++++++-- test_cases/test_api_vulnerability.py | 4 +- 4 files changed, 78 insertions(+), 15 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index df9380a7137..f766f38035e 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -68,7 +68,7 @@ class VulnerabilitySchema(AutoSchema): policyviolations = PrimaryKeyRelatedField('name', many=True, attribute='policy_violations', default=[]) desc = fields.String(dump_only=True, attribute='description') - refs = PrimaryKeyRelatedField('name', many=True, attribute='references', default=[]) + refs = fields.List(fields.String(), attribute='references') issuetracker = fields.Method('get_issuetracker') parent = fields.Method('get_parent', deserialize='load_parent', required=True) parent_type = fields.String(required=True) @@ -286,13 +286,14 @@ def _perform_create(self, data, **kwargs): data = self._parse_data(self._get_schema_class()(strict=True), request) attachments = data.pop('_attachments') + + # This will be set after setting the workspace references = data.pop('references') + policyviolations = data.pop('policy_violations') obj = super(VulnerabilityView, self)._perform_create(data, **kwargs) + obj.references = references - for reference in references: - instance, _ = get_or_create(db.session, Reference, name=reference, workspace=self.workspace) - obj.references.append(instance) for policyviolation in policyviolations: instance, _ = get_or_create(db.session, PolicyViolation, name=policyviolation, workspace=self.workspace) diff --git a/server/models.py b/server/models.py index f36a952cf99..03948545c60 100644 --- a/server/models.py +++ b/server/models.py @@ -347,13 +347,15 @@ def _create(self, value): def _reference_creator(name, vulnerability): """Get or create a vulnerability with the corresponding name. This must be worspace aware""" + assert (vulnerability.workspace and vulnerability.workspace.id + is not None), "Unknown workspace id" reference = Reference.query.filter( - Reference.workspace_id == vulnerability.workspace_id, + Reference.workspace == vulnerability.workspace, Reference.name == name ).first() if reference is None: # Doesn't exist - reference = Reference(name, vulnerability.workspace_id) + reference = Reference(name, vulnerability.workspace.id) return reference @@ -547,10 +549,10 @@ class Reference(Metadata): UniqueConstraint('name', 'workspace_id', name='uix_reference_name_vulnerability_workspace'), ) - def __init__(self, name, workspace_id): - self.name = name - self.workspace_id = workspace_id - super(Reference, self).__init__() + def __init__(self, name=None, workspace_id=None, **kwargs): + super(Reference, self).__init__(name=name, + workspace_id=workspace_id, + **kwargs) class ReferenceVulnerabilityAssociation(db.Model): diff --git a/test_cases/models/test_vulnerability.py b/test_cases/models/test_vulnerability.py index d8bdae6e3a4..0b39d8941e0 100644 --- a/test_cases/models/test_vulnerability.py +++ b/test_cases/models/test_vulnerability.py @@ -1,5 +1,8 @@ import pytest -from server.models import Reference +from server.models import ( + Reference, + ReferenceVulnerabilityAssociation +) def test_standard_host_vuln_hostnames(vulnerability_factory, host_with_hostnames, @@ -75,7 +78,7 @@ def test_add_existing_reference(self, session, reference): session.commit() assert Reference.query.count() == 1 assert len(vuln.references) == 1 - assert list(vuln.reference_instances)[0].id == reference.id + assert vuln.reference_instances.pop().id == reference.id @pytest.mark.parametrize('orphan_vuln', [True, False], ids=['with_orphan_reference', @@ -89,7 +92,64 @@ def test_add_existing_from_other_workspace(self, session, reference, session.commit() assert Reference.query.count() == 2 assert len(self.vuln_different_ws.references) == 1 - new_reference = list(self.vuln_different_ws.reference_instances)[0] + new_reference = self.vuln_different_ws.reference_instances.pop() assert (new_reference.workspace_id == self.vuln_different_ws.workspace_id) assert new_reference.id != reference.id + + def test_remove_reference(self, session, reference): + self.vuln.references.add(reference.name) + session.commit() + self.vuln.references.remove(reference.name) + session.commit() + assert ReferenceVulnerabilityAssociation.query.filter_by( + vulnerability=self.vuln, + reference=reference + ).count() == 0 + + @pytest.mark.skip('not implemented yet') + def test_removes_orphan(self): + pass + + @pytest.mark.parametrize('previous_refs,new_refs',[ + (set(), set()), + (set(), {'CVE-2017-1234'}), + ({'CVE-2017-1234'}, set()), + ({'CVE-2017-1234'}, {'CVE-2017-1234'}), + ({'CVE-2017-1234'}, {'CVE-2017-4321'}), + ({'CVE-2017-1234'}, {'CVE-2017-1234', 'CVE-2017-4321'}), + ], ids=[ + '{} -> {}', + '{} -> {a}', + '{a} -> {}', + '{a} -> {a}', + '{a} -> {b}', + '{a} -> {a, b}', + ]) + def test_direct_assignation(self, session, previous_refs, new_refs): + for ref in previous_refs: + self.vuln.references.add(ref) + session.commit() + self.vuln.references = new_refs + session.commit() + session.refresh(self.vuln) + assert self.vuln.references == new_refs + + def test_create_workspace_and_vuln_with_refs( + self, session, vulnerability_factory, reference): + # This sould raise an error since the workspace_id won't be propagated + # to the references + vuln = vulnerability_factory.create() + with pytest.raises(AssertionError): + vuln.references = {'CVE-2017-1234'} + + def test_create_vuln_with_refs(self, session, vulnerability_factory): + vuln = vulnerability_factory.build() + session.add(vuln.workspace) + session.commit() + assert vuln.workspace.id + vuln.references = {'CVE-2017-1234'} + session.add(vuln) + session.commit() + assert len(vuln.references) == 1 + assert Reference.query.count() == 1 diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 1513807ce76..e5c08011ee6 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -106,7 +106,7 @@ def test_shows_refs(self, workspace, test_client, session, refs = reference_factory.create_batch( 5, workspace=workspace) for ref in refs: - self.first_object.references.append(ref) + self.first_object.reference_instances.add(ref) session.commit() res = test_client.get(self.url(self.first_object)) assert res.status_code == 200 @@ -367,4 +367,4 @@ def test_create_vuln_imapct_verification(self, host_with_hostnames, test_client, assert res.json['impact'] == {u'accountability': True, u'availability': True, u'confidentiality': True, - u'integrity': True} \ No newline at end of file + u'integrity': True} From 11e5b8f4211b7d8e0ccfab3db4ba6e9b01251247 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 23 Oct 2017 15:59:47 -0300 Subject: [PATCH 0389/1506] Fix: when host has hostnames add them as relations --- server/api/modules/hosts.py | 18 +++++++++++++++--- server/models.py | 3 +++ server/www/scripts/hosts/controllers/host.js | 8 ++++---- test_cases/test_api_hosts.py | 10 +++++++++- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 898e8b326d1..4ea83f1b75f 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -9,6 +9,7 @@ from filteralchemy import FilterSet, operators from sqlalchemy.orm import undefer +from server.utils.database import get_or_create from server.utils.logger import get_logger from server.utils.web import gzipped, validate_workspace,\ get_integer_parameter, filter_request_args @@ -21,7 +22,7 @@ FilterSetMeta, ) from server.schemas import PrimaryKeyRelatedField -from server.models import Host, Service +from server.models import Host, Service, db, Hostname host_api = Blueprint('host_api', __name__) @@ -41,7 +42,8 @@ class HostSchema(AutoSchema): services = fields.Integer(attribute='service_count', dump_only=True) vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) credentials = fields.Integer(attribute='vulnerability_count', dump_only=True) - + hostnames = PrimaryKeyRelatedField('name', many=True, + attribute="hostnames", default=[]) def get_metadata(self, obj): return { @@ -59,7 +61,8 @@ class Meta: model = Host fields = ('id', '_id', '_rev', 'ip', 'description', 'credentials', 'default_gateway', 'metadata', - 'name', 'os', 'owned', 'owner', 'services', 'vulns' + 'name', 'os', 'owned', 'owner', 'services', 'vulns', + 'hostnames' ) @@ -92,6 +95,15 @@ def service_list(self, workspace_name, host_id): services = self._get_object(host_id, workspace_name).services return ServiceSchema(many=True).dump(services).data + def _perform_create(self, data, **kwargs): + hostnames = data.pop('hostnames', []) + host = super(HostsView, self)._perform_create(data, **kwargs) + for name in hostnames: + get_or_create(db.session, Hostname, name=name['key'], host=host, workspace=host.workspace) + db.session.commit() + return host + + def _get_base_query(self, workspace_name): """Get services_count in one query and not deferred, that doe one query per host""" diff --git a/server/models.py b/server/models.py index 05023a89925..5fe6eee4b24 100644 --- a/server/models.py +++ b/server/models.py @@ -219,6 +219,9 @@ class Hostname(Metadata): UniqueConstraint(name, host_id, workspace_id, name='uix_hostname_host_workspace'), ) + def __str__(self): + return self.name + class Service(Metadata): STATUSES = [ diff --git a/server/www/scripts/hosts/controllers/host.js b/server/www/scripts/hosts/controllers/host.js index 7d304965f7a..230d0eeadc3 100644 --- a/server/www/scripts/hosts/controllers/host.js +++ b/server/www/scripts/hosts/controllers/host.js @@ -100,7 +100,7 @@ angular.module('faradayApp') }; $scope.newHostnames = function($event){ - $scope.interface.hostnames.push({key:''}); + $scope.host.hostnames.push({key:''}); $event.preventDefault(); } @@ -109,9 +109,9 @@ angular.module('faradayApp') timestamp = date.getTime()/1000.0; // The objectToArray transform is necessary to call updateHost correctly - // If I don't restore the object after the call hostnames won't be shown in the interface - var old_hostnames = $scope.interface.hostnames; - $scope.interface.hostnames = commons.objectToArray($scope.interface.hostnames.filter(Boolean)); + // If I don't restore the object after the call hostnames won't be shown in the host + var old_hostnames = $scope.host.hostnames; + $scope.host.hostnames = commons.objectToArray($scope.host.hostnames.filter(Boolean)); $scope.hostdata = $scope.host; $scope.hostdata.metadata['update_time'] = timestamp; diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index f9f571d78ca..42d84ca779c 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -282,7 +282,15 @@ def test_host_services_vuln_count_verification(self, test_client, session, assert res.status_code == 200 assert res.json[0]['vulns'] == 1 - + def test_create_host_with_hostnames(self, test_client): + raw_data = { + "ip": "192.168.0.21", + "hostnames": [{"key": "google.com"}], + "mac": "00:00:00:00:00:00", "description": "", + "os": "", "owned": False, "owner": "" + } + res = test_client.post(self.url(), data=raw_data) + assert res.status_code == 201 class TestHostAPIGeneric(ReadWriteAPITests, PaginationTestsMixin): From f88356e25f7a445efd309f47360214856c6565a7 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 23 Oct 2017 16:01:59 -0300 Subject: [PATCH 0390/1506] Fix vuln tests --- test_cases/test_api_vulnerability.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 88e6b2a3e27..aa8399c4e2f 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -6,7 +6,7 @@ from server.api.modules.vulns import VulnerabilityView from test_cases import factories -from test_api_non_workspaced_base import ( +from test_api_workspaced_base import ( ReadOnlyAPITests ) from server.models import ( From 214058d6fb39c88e9297fa16372bf3cae7b31d09 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 23 Oct 2017 16:06:16 -0300 Subject: [PATCH 0391/1506] Fix how refs and pvs are added to vulns on import When the vulns are imported from CouchDB, the References and Policy violations must be added as a list to the Vuln object. --- server/importer.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/server/importer.py b/server/importer.py index af62ca37193..fc28c36b442 100644 --- a/server/importer.py +++ b/server/importer.py @@ -436,7 +436,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation ) vulnerability.confirmed = document.get('confirmed', False) or False vulnerability.data = document.get('data') - vulnerability.easeofresolution = document.get('easeofresolution') + vulnerability.ease_of_resolution = document.get('easeofresolution') if document.get('easeofresolution') else None vulnerability.resolution = document.get('resolution') vulnerability.owned = document.get('owned', False) @@ -463,8 +463,8 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation status = status_map[document.get('status', 'opened')] vulnerability.status = status - self.add_references(document, vulnerability, workspace) - self.add_policy_violations(document, vulnerability, workspace) + vulnerability.references.extend(self.add_references(document, vulnerability, workspace)) + vulnerability.policy_violations.extend(self.add_policy_violations(document, vulnerability, workspace)) # need the vuln ID before creating Tags for it session.commit() @@ -502,24 +502,28 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation yield vulnerability def add_policy_violations(self, document, vulnerability, workspace): + policy_violations = list() for policy_violation in document.get('policyviolations', []): - get_or_create( + pv, _ = get_or_create( session, PolicyViolation, name=policy_violation, - workspace=workspace, - vulnerability=vulnerability + workspace=workspace ) + policy_violations.append(pv) + return policy_violations def add_references(self, document, vulnerability, workspace): + references = list() for ref in document.get('refs', []): - get_or_create( + reference, _ = get_or_create( session, Reference, name=ref, - workspace=workspace, - vulnerability=vulnerability + workspace=workspace ) + references.append(reference) + return references class CommandImporter(object): From 38d153e9e13f69cecfe3f99ba56df9666cce361e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 23 Oct 2017 17:17:57 -0300 Subject: [PATCH 0392/1506] Support new policy violations implementation (using AssociationProxy) --- server/api/modules/vulns.py | 18 ++-- server/models.py | 56 +++++++---- test_cases/models/test_vulnerability.py | 121 +++++++++++++++--------- test_cases/test_api_vulnerability.py | 2 +- 4 files changed, 120 insertions(+), 77 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 1500c5f7a38..1c1d86cfca1 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -27,7 +27,7 @@ Vulnerability, VulnerabilityWeb, VulnerabilityGeneric, - Host, Service, File, Reference, PolicyViolation) + Host, Service, File) from server.utils.database import get_or_create from server.api.modules.services import ServiceSchema @@ -65,9 +65,9 @@ class VulnerabilitySchema(AutoSchema): owned = fields.Boolean(dump_only=True, default=False) owner = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator') impact = fields.Method('get_impact', deserialize='load_impact') - policyviolations = PrimaryKeyRelatedField('name', many=True, - attribute='policy_violations', default=[]) desc = fields.String(dump_only=True, attribute='description') + policyviolations = fields.List(fields.String, + attribute='policy_violations') refs = fields.List(fields.String(), attribute='references') issuetracker = fields.Method('get_issuetracker') parent = fields.Method('get_parent', deserialize='load_parent', required=True) @@ -289,15 +289,11 @@ def _perform_create(self, data, **kwargs): # This will be set after setting the workspace references = data.pop('references') - policyviolations = data.pop('policy_violations') + obj = super(VulnerabilityView, self)._perform_create(data, **kwargs) obj.references = references - - - for policyviolation in policyviolations: - instance, _ = get_or_create(db.session, PolicyViolation, name=policyviolation, workspace=self.workspace) - obj.policy_violations.append(instance) + obj.policy_violations = policyviolations for filename, attachment in attachments.items(): faraday_file = FaradayUploadedFile(b64decode(attachment['data'])) @@ -306,8 +302,8 @@ def _perform_create(self, data, **kwargs): File, object_id=obj.id, object_type=obj.__class__.__name__, - name = os.path.splitext(os.path.basename(filename))[0], - filename = os.path.basename(filename), + name=os.path.splitext(os.path.basename(filename))[0], + filename=os.path.basename(filename), content=faraday_file, ) db.session.commit() diff --git a/server/models.py b/server/models.py index 5fe6eee4b24..ead22583bd5 100644 --- a/server/models.py +++ b/server/models.py @@ -348,19 +348,27 @@ def _create(self, value): return self.creator(value, parent_instance) -def _reference_creator(name, vulnerability): - """Get or create a vulnerability with the corresponding name. - This must be worspace aware""" - assert (vulnerability.workspace and vulnerability.workspace.id - is not None), "Unknown workspace id" - reference = Reference.query.filter( - Reference.workspace == vulnerability.workspace, - Reference.name == name - ).first() - if reference is None: - # Doesn't exist - reference = Reference(name, vulnerability.workspace.id) - return reference +def _build_associationproxy_creator(model_class_name): + def creator(name, vulnerability): + """Get or create a reference/policyviolation with the + corresponding name. This must be worspace aware""" + + # Ugly hack to avoid the fact that Reference is defined after + # Vulnerability + model_class = globals()[model_class_name] + + assert (vulnerability.workspace and vulnerability.workspace.id + is not None), "Unknown workspace id" + child = model_class.query.filter( + getattr(model_class, 'workspace') == vulnerability.workspace, + getattr(model_class, 'name') == name, + ).first() + if child is None: + # Doesn't exist + child = model_class(name, vulnerability.workspace.id) + return child + + return creator class VulnerabilityGeneric(VulnerabilityABC): @@ -395,15 +403,22 @@ class VulnerabilityGeneric(VulnerabilityABC): collection_class=set ) - references = association_proxy('reference_instances', 'name', - proxy_factory=CustomAssociationSet, - creator=_reference_creator) + references = association_proxy( + 'reference_instances', 'name', + proxy_factory=CustomAssociationSet, + creator=_build_associationproxy_creator('Reference')) - policy_violations = relationship( + policy_violation_instances = relationship( "PolicyViolation", - secondary="policy_violation_vulnerability_association" + secondary="policy_violation_vulnerability_association", + collection_class=set ) + policy_violations = association_proxy( + 'policy_violation_instances', 'name', + proxy_factory=CustomAssociationSet, + creator=_build_associationproxy_creator('PolicyViolation')) + __mapper_args__ = { 'polymorphic_on': type } @@ -629,6 +644,11 @@ class PolicyViolation(Metadata): name='uix_policy_violation_template_name_vulnerability_workspace'), ) + def __init__(self, name=None, workspace_id=None, **kwargs): + super(PolicyViolation, self).__init__(name=name, + workspace_id=workspace_id, + **kwargs) + class Credential(Metadata): __tablename__ = 'credential' diff --git a/test_cases/models/test_vulnerability.py b/test_cases/models/test_vulnerability.py index 0b39d8941e0..80a73a41c98 100644 --- a/test_cases/models/test_vulnerability.py +++ b/test_cases/models/test_vulnerability.py @@ -1,9 +1,12 @@ import pytest from server.models import ( Reference, - ReferenceVulnerabilityAssociation + ReferenceVulnerabilityAssociation, + PolicyViolation, + PolicyViolationVulnerabilityAssociation, ) + def test_standard_host_vuln_hostnames(vulnerability_factory, host_with_hostnames, session): @@ -12,6 +15,7 @@ def test_standard_host_vuln_hostnames(vulnerability_factory, session.commit() assert set(vuln.hostnames) == set(host_with_hostnames.hostnames) + def test_standard_vuln_service_hostnames(vulnerability_factory, service_factory, host_with_hostnames, @@ -21,6 +25,7 @@ def test_standard_vuln_service_hostnames(vulnerability_factory, session.commit() assert set(vuln.hostnames) == set(host_with_hostnames.hostnames) + def test_web_vuln_hostnames(vulnerability_web_factory, service_factory, host_with_hostnames, @@ -30,6 +35,7 @@ def test_web_vuln_hostnames(vulnerability_web_factory, session.commit() assert set(vuln.hostnames) == set(host_with_hostnames.hostnames) + def test_code_vuln(vulnerability_code, session): session.commit() # Source code vulnerabilities have no hostnames @@ -37,6 +43,13 @@ def test_code_vuln(vulnerability_code, session): class TestReferences: + + field_name = 'references' + instances_field_name = 'reference_instances' + model = Reference + intermediate_model = ReferenceVulnerabilityAssociation + intermediate_field = 'reference' + @pytest.fixture(autouse=True) def load_data(self, vulnerability_factory, session): self.vuln = vulnerability_factory.create() @@ -48,70 +61,76 @@ def load_data(self, vulnerability_factory, session): assert self.vuln.workspace_id != self.vuln_different_ws.workspace_id @pytest.fixture - def reference(self, session): - reference = Reference('CVE-2017-1234', self.vuln.workspace_id) - session.add(reference) + def child(self, session): + child = self.model('CVE-2017-1234', self.vuln.workspace_id) + session.add(child) session.commit() - return reference + return child + + def childs(self, vuln=None, instance=False): + vuln = vuln or self.vuln + field = self.instances_field_name if instance else self.field_name + return getattr(vuln, field) def test_empty_references(self): for vuln in self.vulns: - assert isinstance(vuln.reference_instances, set) + assert isinstance(self.childs(vuln, True), set) assert len(vuln.references) == 0 def test_add_references(self, session): - self.vuln.references.add('CVE-2017-1234') + self.childs().add('CVE-2017-1234') assert session.new session.commit() - assert Reference.query.count() == 1 - ref = Reference.query.first() + assert self.model.query.count() == 1 + ref = self.model.query.first() assert ref.name == 'CVE-2017-1234' assert ref.workspace_id == self.vuln.workspace_id # Re-adding the reference shouldn't do nothing - self.vuln.references.add('CVE-2017-1234') - assert Reference.query.count() == 1 + self.childs().add('CVE-2017-1234') + assert self.model.query.count() == 1 - def test_add_existing_reference(self, session, reference): + def test_add_existing_child(self, session, child): for vuln in [self.vuln, self.vuln2]: - vuln.references.add('CVE-2017-1234') + self.childs(vuln).add('CVE-2017-1234') session.commit() - assert Reference.query.count() == 1 - assert len(vuln.references) == 1 - assert vuln.reference_instances.pop().id == reference.id + assert self.model.query.count() == 1 + assert len(self.childs(vuln)) == 1 + assert self.childs(vuln, True).pop().id == child.id @pytest.mark.parametrize('orphan_vuln', [True, False], - ids=['with_orphan_reference', - 'with_used_reference']) - def test_add_existing_from_other_workspace(self, session, reference, + ids=['with_orphan_child', + 'with_used_child']) + def test_add_existing_from_other_workspace(self, session, child, orphan_vuln): if not orphan_vuln: - self.vuln.references.add(reference.name) + self.childs().add(child.name) session.commit() - self.vuln_different_ws.references.add(reference.name) + self.childs(self.vuln_different_ws).add(child.name) session.commit() - assert Reference.query.count() == 2 - assert len(self.vuln_different_ws.references) == 1 - new_reference = self.vuln_different_ws.reference_instances.pop() - assert (new_reference.workspace_id == + assert self.model.query.count() == 2 + assert len(self.childs(self.vuln_different_ws)) == 1 + new_child = self.childs(self.vuln_different_ws, True).pop() + assert (new_child.workspace_id == self.vuln_different_ws.workspace_id) - assert new_reference.id != reference.id + assert new_child.id != child.id - def test_remove_reference(self, session, reference): - self.vuln.references.add(reference.name) + def test_remove_reference(self, session, child): + self.childs().add(child.name) session.commit() - self.vuln.references.remove(reference.name) + self.childs().remove(child.name) session.commit() - assert ReferenceVulnerabilityAssociation.query.filter_by( - vulnerability=self.vuln, - reference=reference - ).count() == 0 + filters = { + 'vulnerability': self.vuln, + self.intermediate_field: child + } + assert self.intermediate_model.query.filter_by(**filters).count() == 0 @pytest.mark.skip('not implemented yet') def test_removes_orphan(self): pass - @pytest.mark.parametrize('previous_refs,new_refs',[ + @pytest.mark.parametrize('previous_childs,new_childs', [ (set(), set()), (set(), {'CVE-2017-1234'}), ({'CVE-2017-1234'}, set()), @@ -126,30 +145,38 @@ def test_removes_orphan(self): '{a} -> {b}', '{a} -> {a, b}', ]) - def test_direct_assignation(self, session, previous_refs, new_refs): - for ref in previous_refs: - self.vuln.references.add(ref) + def test_direct_assignation(self, session, previous_childs, new_childs): + for ref in previous_childs: + self.childs().add(ref) session.commit() - self.vuln.references = new_refs + setattr(self.vuln, self.field_name, new_childs) session.commit() session.refresh(self.vuln) - assert self.vuln.references == new_refs + assert self.childs() == new_childs - def test_create_workspace_and_vuln_with_refs( - self, session, vulnerability_factory, reference): + def test_create_workspace_and_vuln_with_childs( + self, session, vulnerability_factory, child): # This sould raise an error since the workspace_id won't be propagated - # to the references + # to the childs vuln = vulnerability_factory.create() with pytest.raises(AssertionError): - vuln.references = {'CVE-2017-1234'} + setattr(vuln, self.field_name, {'CVE-2017-1234'}) - def test_create_vuln_with_refs(self, session, vulnerability_factory): + def test_create_vuln_with_childs(self, session, vulnerability_factory): vuln = vulnerability_factory.build() session.add(vuln.workspace) session.commit() assert vuln.workspace.id - vuln.references = {'CVE-2017-1234'} + setattr(vuln, self.field_name, {'CVE-2017-1234'}) session.add(vuln) session.commit() - assert len(vuln.references) == 1 - assert Reference.query.count() == 1 + assert len(self.childs(vuln)) == 1 + assert self.model.query.count() == 1 + + +class TestPolicyViolations(TestReferences): + field_name = 'policy_violations' + instances_field_name = 'policy_violation_instances' + model = PolicyViolation + intermediate_model = PolicyViolationVulnerabilityAssociation + intermediate_field = 'policy_violation' diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index aa8399c4e2f..8f48888805d 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -91,7 +91,7 @@ def test_shows_policy_violations(self, workspace, test_client, session, pvs = policy_violation_factory.create_batch( 5, workspace=workspace) for pv in pvs: - self.first_object.policy_violations.append(pv) + self.first_object.policy_violation_instances.add(pv) session.commit() res = test_client.get(self.url(self.first_object)) assert res.status_code == 200 From 1fda13a235ead78e3e29f80c2c6b3e90a00390a5 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 23 Oct 2017 16:37:57 -0300 Subject: [PATCH 0393/1506] Fix a bug when the host as default_gateway_ip data --- server/api/modules/hosts.py | 2 ++ test_cases/test_api_hosts.py | 11 +++++++++++ test_cases/test_api_vulnerability_template.py | 1 + 3 files changed, 14 insertions(+) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 4ea83f1b75f..6551145a8ec 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -97,7 +97,9 @@ def service_list(self, workspace_name, host_id): def _perform_create(self, data, **kwargs): hostnames = data.pop('hostnames', []) + default_gateway_ip = data.pop('default_gateway_ip', [None]) host = super(HostsView, self)._perform_create(data, **kwargs) + host.default_gateway_ip = default_gateway_ip[0] for name in hostnames: get_or_create(db.session, Hostname, name=name['key'], host=host, workspace=host.workspace) db.session.commit() diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 42d84ca779c..651651adbb1 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -292,6 +292,17 @@ def test_create_host_with_hostnames(self, test_client): res = test_client.post(self.url(), data=raw_data) assert res.status_code == 201 + def test_create_host_with_default_gateway(self, test_client): + raw_data = { + "ip": "192.168.0.21", + "default_gateway": "192.168.0.1", + "mac": "00:00:00:00:00:00", "description": "", + "os": "", "owned": False, "owner": "" + } + res = test_client.post(self.url(), data=raw_data) + assert res.status_code == 201 + assert res.json['default_gateway'] == [u'192.168.0.1'] + class TestHostAPIGeneric(ReadWriteAPITests, PaginationTestsMixin): model = Host diff --git a/test_cases/test_api_vulnerability_template.py b/test_cases/test_api_vulnerability_template.py index e841d1b75f6..c3f2eae64a6 100644 --- a/test_cases/test_api_vulnerability_template.py +++ b/test_cases/test_api_vulnerability_template.py @@ -81,6 +81,7 @@ def test_update_vulnerability_template(self, session, test_client): def test_update_vulnerabiliy_template_change_refs(self, session, test_client): template = self.factory.create() + session.commit() raw_data = self._create_post_data_vulnerability_template(references=['new_ref', 'another_ref']) res = test_client.put('/v2/vulnerability_template/{0}/'.format(template.id), data=raw_data) assert res.status_code == 200 From 873f8e6282ae10e3823b4a0584b1b27ab3b33e81 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 23 Oct 2017 18:01:37 -0300 Subject: [PATCH 0394/1506] Delete programming --- server/api/modules/commandsrun.py | 22 ------ server/api/modules/credentials.py | 18 ----- server/api/modules/hosts.py | 26 ------- server/api/modules/services.py | 30 -------- server/api/modules/workspaces.py | 118 ------------------------------ 5 files changed, 214 deletions(-) diff --git a/server/api/modules/commandsrun.py b/server/api/modules/commandsrun.py index 066fe95381d..ac66836d8da 100644 --- a/server/api/modules/commandsrun.py +++ b/server/api/modules/commandsrun.py @@ -60,25 +60,3 @@ def _envelope_list(self, objects, pagination_metadata=None): } CommandView.register(commandsrun_api) - - -@gzipped -@commandsrun_api.route('/ws//commands', methods=['GET']) -def list_commands(workspace=None): - validate_workspace(workspace) - get_logger(__name__).debug( - "Request parameters: {!r}".format(flask.request.args)) - - page = get_integer_parameter('page', default=0) - page_size = get_integer_parameter('page_size', default=0) - - commands_filter = filter_request_args( - 'page', 'page_size') - - dao = CommandDAO(workspace) - result = dao.list( - page=page, - page_size=page_size, - command_filter=commands_filter) - - return flask.jsonify(result) diff --git a/server/api/modules/credentials.py b/server/api/modules/credentials.py index e673c39432c..d56b37e601c 100644 --- a/server/api/modules/credentials.py +++ b/server/api/modules/credentials.py @@ -74,21 +74,3 @@ def _envelope_list(self, objects, pagination_metadata=None): } CredentialView.register(credentials_api) - - -@gzipped -@credentials_api.route('/ws//credentials', methods=['GET']) -def list_credentials(workspace=None): - - validate_workspace(workspace) - - get_logger(__name__).debug( - "Request parameters: {!r}".format( - flask.request.args)) - - cred_filter = filter_request_args() - - dao = CredentialDAO(workspace) - result = dao.list(cred_filter=cred_filter) - - return flask.jsonify(result) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 6551145a8ec..2c80a81a603 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -127,29 +127,3 @@ def _envelope_list(self, objects, pagination_metadata=None): } HostsView.register(host_api) - - -@gzipped -@host_api.route('/ws//hosts', methods=['GET']) -def list_hosts(workspace=None): - validate_workspace(workspace) - get_logger(__name__).debug("Request parameters: {!r}"\ - .format(flask.request.args)) - - page = get_integer_parameter('page', default=0) - page_size = get_integer_parameter('page_size', default=0) - search = flask.request.args.get('search') - order_by = flask.request.args.get('sort') - order_dir = flask.request.args.get('sort_dir') - - host_filter = filter_request_args('page', 'page_size', 'search', 'sort', 'sort_dir') - - dao = HostDAO(workspace) - result = dao.list(search=search, - page=page, - page_size=page_size, - order_by=order_by, - order_dir=order_dir, - host_filter=host_filter) - - return flask.jsonify(result) diff --git a/server/api/modules/services.py b/server/api/modules/services.py index 30c4e9037f3..26add482150 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -73,33 +73,3 @@ def _envelope_list(self, objects, pagination_metadata=None): } ServiceView.register(services_api) - - -@services_api.route('/ws//services', methods=['GET']) -@gzipped -def list_services(workspace=None): - validate_workspace(workspace) - get_logger(__name__).debug("Request parameters: {!r}"\ - .format(flask.request.args)) - - services_dao = ServiceDAO(workspace) - - services = services_dao.list(service_filter=flask.request.args) - - return flask.jsonify(services) - -@services_api.route('/ws//services/count', methods=['GET']) -@gzipped -def count_services(workspace=None): - validate_workspace(workspace) - get_logger(__name__).debug("Request parameters: {!r}"\ - .format(flask.request.args)) - - field = flask.request.args.get('group_by') - - services_dao = ServiceDAO(workspace) - result = services_dao.count(group_by=field) - if result is None: - flask.abort(400) - - return flask.jsonify(result) diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index 445bac07782..e9a375519be 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -70,121 +70,3 @@ class WorkspaceView(ReadWriteView): schema_class = WorkspaceSchema WorkspaceView.register(workspace_api) - - -@workspace_api.route('/ws', methods=['GET']) -@gzipped -def workspace_list(): - get_logger(__name__).debug("Request parameters: {!r}" - .format(flask.request.args)) - - page = get_integer_parameter('page', default=0) - page_size = get_integer_parameter('page_size', default=0) - search = flask.request.args.get('search') - order_by = flask.request.args.get('sort') - order_dir = flask.request.args.get('sort_dir') - - ws_filter = filter_request_args('page', 'page_size', 'search', 'sort', 'sort_dir') - - ws_dao = WorkspaceDAO() - - result = ws_dao.list(search=search, - page=page, - page_size=page_size, - order_by=order_by, - order_dir=order_dir, - workspace_filter=ws_filter) - - return flask.jsonify(result) - - -@workspace_api.route('/ws//summary', methods=['GET']) -@gzipped -def workspace_summary(workspace=None): - validate_workspace(workspace) - - services_count = ServiceDAO(workspace).count() - vuln_count = VulnerabilityDAO(workspace).count(vuln_filter=flask.request.args) - host_count = HostDAO(workspace).count() - - response = { - 'stats': { - 'services': services_count.get('total_count', 0), - 'total_vulns': vuln_count.get('total_count', 0), - 'web_vulns': vuln_count.get('web_vuln_count', 0), - 'std_vulns': vuln_count.get('vuln_count', 0), - 'hosts': host_count.get('total_count', 0), - } - } - - return flask.jsonify(response) - - -@workspace_api.route('/ws/', methods=['GET']) -@gzipped -def workspace(workspace): - validate_workspace(workspace) - workspaces = list_workspaces_as_user( - flask.request.cookies, get_basic_auth())['workspaces'] - if not workspaces: - return flask.abort(404) - ws = get_workspace(workspace, flask.request.cookies, get_basic_auth()) if workspace in workspaces else None - # TODO: When the workspace DAO is ready, we have to remove this next line - if not ws.get('fdate') and ws.get('duration'): - ws['fdate'] = ws.get('duration').get('end') - if not ws.get('description'): - ws['description'] = '' - return flask.jsonify(ws) - - -@workspace_api.route('/ws/', methods=['PUT']) -@gzipped -def workspace_create_or_update(workspace): - # only admins can create workspaces - validate_admin_perm() - - try: - document = json.loads(flask.request.data) - except ValueError: - return build_bad_request_response('invalid json') - if not document.get('name', None): - return build_bad_request_response('workspace name needed') - if document.get('name') != workspace: - return build_bad_request_response('workspace name and route parameter don\'t match') - if document.get('name').startswith('_'): - return build_bad_request_response('database cannot start with an underscore') - document['_id'] = document.get('name') # document dictionary does not have id, add it - workspace_exists = db.session.query(Workspace).filter_by(name=workspace).first() - if not workspace_exists: - new_workspace = Workspace(name=workspace) - db.session.add(new_workspace) - db.session.commit() - res = True - else: - response = flask.jsonify({'error': "Workspace {0} already exists.".format(workspace)}) - response.status_code = 409 - return response - - if not res: - response = flask.jsonify({'error': "There was an error {0} the workspace".format("updating" if is_update_request else "creating")}) - response.status_code = 500 - return response - - return flask.jsonify({'ok': True}) - - -@workspace_api.route('/ws/', methods=['DELETE']) -@gzipped -def workspace_delete(workspace): - # only admins can delete workspaces - validate_admin_perm() - validate_workspace(workspace) - - db_manager = get_manager() - - if not db_manager.delete_workspace(workspace): - response = flask.jsonify({'error': "There was an error deleting the workspace"}) - response.status_code = 500 - return response - - return flask.jsonify({'ok': True}) From 742f661d3ebb9e7faccf6a5568f24da3e8cae29e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 23 Oct 2017 18:13:40 -0300 Subject: [PATCH 0395/1506] Remove unused and possibly buggy line It is a bad practice to set attributes to the view instance based on the current request, since the class is reused in next requests --- server/api/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/api/base.py b/server/api/base.py index 7f52d4ca5b1..6a5989212ba 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -300,7 +300,6 @@ class CreateWorkspacedMixin(CreateMixin): def _perform_create(self, data, workspace_name): assert not db.session.new workspace = self._get_workspace(workspace_name) - self.workspace = workspace obj = self.model_class(**data) obj.workspace = workspace # assert not db.session.new From 7ebe6acbdde194130c583f4d1d3a293a0aef5966 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 24 Oct 2017 15:41:54 -0300 Subject: [PATCH 0396/1506] Fix importer to work with new references/policyviolations --- server/importer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/importer.py b/server/importer.py index fc28c36b442..42f68a26c5a 100644 --- a/server/importer.py +++ b/server/importer.py @@ -463,8 +463,10 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation status = status_map[document.get('status', 'opened')] vulnerability.status = status - vulnerability.references.extend(self.add_references(document, vulnerability, workspace)) - vulnerability.policy_violations.extend(self.add_policy_violations(document, vulnerability, workspace)) + vulnerability.reference_instances.update( + self.add_references(document, vulnerability, workspace)) + vulnerability.policy_violation_instances.update( + self.add_policy_violations(document, vulnerability, workspace)) # need the vuln ID before creating Tags for it session.commit() From 0e98fb00bb21161d5147feece0db434aeb5051bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 24 Oct 2017 15:42:23 -0300 Subject: [PATCH 0397/1506] Add security TODO in count endpoint --- server/api/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/api/base.py b/server/api/base.py index 6a5989212ba..aec554a8803 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -371,6 +371,9 @@ def count(self, **kwargs): 'total_count': 0 } group_by = flask.request.args.get('group_by', None) + # TODO migration: whitelist fields to avoid leaking a confidential + # field's value. + # Example: /users/count/?group_by=password if not group_by or group_by not in inspect(self.model_class).attrs: abort(404) From 185fa216aab2a7a748a27b8185d32d35458411e8 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 24 Oct 2017 16:34:03 -0300 Subject: [PATCH 0398/1506] Some fixes on credentials --- server/api/modules/credentials.py | 11 ++--- server/api/modules/vulnerability_template.py | 1 + server/events.py | 0 .../www/scripts/commons/providers/server.js | 6 +-- test_cases/test_api_credentials.py | 47 ++++++++++++++++++- test_cases/test_model_events.py | 0 6 files changed, 55 insertions(+), 10 deletions(-) create mode 100644 server/events.py create mode 100644 test_cases/test_model_events.py diff --git a/server/api/modules/credentials.py b/server/api/modules/credentials.py index d56b37e601c..29422b798d9 100644 --- a/server/api/modules/credentials.py +++ b/server/api/modules/credentials.py @@ -9,9 +9,6 @@ from server.api.base import AutoSchema, ReadWriteWorkspacedView from server.models import Credential -from server.utils.logger import get_logger -from server.utils.web import gzipped, validate_workspace, filter_request_args -from server.dao.credential import CredentialDAO credentials_api = Blueprint('credentials_api', __name__) @@ -22,10 +19,11 @@ class CredentialSchema(AutoSchema): _rev = fields.String(default='', dump_only=True) metadata = fields.Method('get_metadata') owned = fields.Boolean(default=False) - owner = fields.String(dump_only=True, attribute='creator.username') + owner = fields.String(dump_only=True, attribute='creator.username', default='') username = fields.String(default='') password = fields.String(default='') description = fields.String(default='') + couchdbid = fields.String(default='') # backwards compatibility parent = fields.Method('get_parent') @@ -53,11 +51,12 @@ class Meta: fields = ('id', '_id', "_rev", 'parent', 'username', 'description', 'name', 'password', + 'owner', 'owned', 'couchdbid', 'metadata') class CredentialView(ReadWriteWorkspacedView): - route_base = 'credentials' + route_base = 'credential' model_class = Credential schema_class = CredentialSchema @@ -70,7 +69,7 @@ def _envelope_list(self, objects, pagination_metadata=None): 'value': credential }) return { - 'credentials': credentials, + 'rows': credentials, } CredentialView.register(credentials_api) diff --git a/server/api/modules/vulnerability_template.py b/server/api/modules/vulnerability_template.py index 820cb92e736..52852e90137 100644 --- a/server/api/modules/vulnerability_template.py +++ b/server/api/modules/vulnerability_template.py @@ -70,4 +70,5 @@ def _envelope_list(self, objects, pagination_metadata=None): 'rows': vuln_tpls, } + VulnerabilityTemplateView.register(vulnerability_template_api) \ No newline at end of file diff --git a/server/events.py b/server/events.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index c92203f1e90..da353d06dd1 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -173,7 +173,7 @@ angular.module("faradayApp") var modCredential = function(createOrUpdate, wsName, credential) { if (typeof credential.owner === "undefined") {credential.owner = ""}; if (typeof credential.description === "undefined") {credential.description = ""}; - return createOrUpdate(wsName, credential._id, credential); + return createOrUpdate(wsName, credential._id, credential, 'credential'); } var modCommand = function(createOrUpdate, wsName, command) { @@ -261,8 +261,8 @@ angular.module("faradayApp") return get(getUrl, data); } - ServerAPI.getCredentials = function(wsName, data) { - var getUrl = createGetUrl(wsName, 'credentials'); + ServerAPI.getCredentials = function(wsName, data, objId) { + var getUrl = createGetUrl(wsName, 'credential', objId); return get(getUrl, data); } diff --git a/test_cases/test_api_credentials.py b/test_cases/test_api_credentials.py index 3495dbed32a..5b6cf437f36 100644 --- a/test_cases/test_api_credentials.py +++ b/test_cases/test_api_credentials.py @@ -10,5 +10,50 @@ class TestCredentialsAPIGeneric(ReadOnlyAPITests): model = Credential factory = factories.CredentialFactory view_class = CredentialView - api_endpoint = 'credentials' + api_endpoint = 'credential' update_fields = ['username', 'password'] + + def test_get_list_backwards_compatibility(self, test_client, session, second_workspace): + cred = self.factory.create(workspace=second_workspace) + session.commit() + res = test_client.get(self.url()) + assert res.status_code == 200 + assert 'rows' in res.json + for vuln in res.json['rows']: + assert set([u'id', u'key', u'value']) == set(vuln.keys()) + object_properties = [ + u'_id', + u'couchdbid', + u'description', + u'metadata', + u'name', + u'owned', + u'owner', + u'password', + u'username', + ] + expected = set(object_properties) + result = set(vuln['value'].keys()) + assert expected - result == set() + + def test_create_from_raw_data_host_as_parent(self, session, test_client, + workspace, host_factory): + host = host_factory.create(workspace=workspace) + session.commit() + raw_data = { + "_id":"1.e5069bb0718aa519852e6449448eedd717f1b90d", + "name":"name", + "username":"username", + "metadata":{"update_time":1508794240799,"update_user":"", + "update_action":0,"creator":"UI Web", + "create_time":1508794240799,"update_controller_action":"", + "owner":""}, + "password":"pass", + "type":"Cred", + "owner":"", + "description":"", + "parent_id": host.id, + "parent_type": "Host" + } + res = test_client.post(self.url(), data=raw_data) + assert res.status_code == 201 diff --git a/test_cases/test_model_events.py b/test_cases/test_model_events.py new file mode 100644 index 00000000000..e69de29bb2d From 51b011567a35f099f1e76187a785a0b059f6b544 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 24 Oct 2017 16:34:49 -0300 Subject: [PATCH 0399/1506] Add workspace parent verification --- server/api/modules/vulns.py | 7 +++- server/events.py | 15 +++++++ server/models.py | 54 +++++++++++++++++++------ test_cases/factories.py | 12 +++--- test_cases/models/test_vulnerability.py | 11 ++--- test_cases/test_api_hosts.py | 17 ++++---- test_cases/test_api_services.py | 2 +- test_cases/test_api_vulnerability.py | 2 +- test_cases/test_api_workspaced_base.py | 2 +- test_cases/test_model_events.py | 18 +++++++++ 10 files changed, 105 insertions(+), 35 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 1c1d86cfca1..6cd8936e74c 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -71,7 +71,7 @@ class VulnerabilitySchema(AutoSchema): refs = fields.List(fields.String(), attribute='references') issuetracker = fields.Method('get_issuetracker') parent = fields.Method('get_parent', deserialize='load_parent', required=True) - parent_type = fields.String(required=True) + parent_type = fields.Method('get_parent_type', required=True) tags = fields.Method('get_tags') easeofresolution = fields.String(dump_only=True, attribute='ease_of_resolution') hostnames = PrimaryKeyRelatedField('name', many=True) @@ -80,7 +80,7 @@ class VulnerabilitySchema(AutoSchema): '_id', 'ports', 'status', 'protocol', 'name', 'version', 'summary' ]), dump_only=True) host = fields.Integer(dump_only=True, attribute='host_id') - status = fields.Method('get_status', deserialize='load_status') + status = fields.Method('get_status', deserialize='load_status') # TODO: this breaks enum validation. type = fields.Method('get_type', deserialize='load_type') obj_id = fields.String(dump_only=True, attribute='id') target = fields.String(default='') # TODO: review this attribute @@ -147,6 +147,9 @@ def get_tags(self, obj): def get_parent(self, obj): return obj.parent.id + def get_parent_type(self, obj): + return obj.parent.__class__.__name__ + def load_parent(self, obj): return obj diff --git a/server/events.py b/server/events.py index e69de29bb2d..cfd855a0860 100644 --- a/server/events.py +++ b/server/events.py @@ -0,0 +1,15 @@ +import sys +import inspect + +from sqlalchemy import event + + +def after_insert_check_child_has_same_workspace(mapper, connection, inserted_instance): + if inserted_instance.parent: + assert inserted_instance.workspace == inserted_instance.parent.workspace + + +# register the workspace verification for all objs that has workspace_id +for name, obj in inspect.getmembers(sys.modules['server.models']): + if inspect.isclass(obj) and getattr(obj, 'workspace_id', None): + event.listen(obj, 'after_insert', after_insert_check_child_has_same_workspace) diff --git a/server/models.py b/server/models.py index ead22583bd5..624f12fe22c 100644 --- a/server/models.py +++ b/server/models.py @@ -161,6 +161,10 @@ class SourceCode(Metadata): UniqueConstraint(filename, workspace_id, name='uix_source_code_filename_workspace'), ) + @property + def parent(self): + return + class Host(Metadata): __tablename__ = 'host' @@ -201,6 +205,10 @@ class Host(Metadata): UniqueConstraint(ip, workspace_id, name='uix_host_ip_workspace'), ) + @property + def parent(self): + return + class Hostname(Metadata): __tablename__ = 'hostname' @@ -222,6 +230,10 @@ class Hostname(Metadata): def __str__(self): return self.name + @property + def parent(self): + return self.host + class Service(Metadata): STATUSES = [ @@ -267,6 +279,10 @@ class Service(Metadata): UniqueConstraint(port, protocol, host_id, workspace_id, name='uix_service_port_protocol_host_workspace'), ) + @property + def parent(self): + return self.host + class VulnerabilityABC(Metadata): # revisar plugin nexpose, netspark para terminar de definir uniques. asegurar que se carguen bien @@ -308,6 +324,10 @@ class VulnerabilityABC(Metadata): name='check_vulnerability_risk'), ) + @property + def parent(self): + raise NotImplementedError('ABC property called') + class VulnerabilityTemplate(VulnerabilityABC): __tablename__ = 'vulnerability_template' @@ -453,13 +473,6 @@ def hostnames(self): def parent(self): return self.host or self.service - @property - def parent_type(self): - if self.host_id: - return 'Host' - if self.service_id: - return 'Service' - __mapper_args__ = { 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[0] } @@ -486,11 +499,6 @@ def service_id(cls): def service(cls): return relationship('Service', backref='vulnerabilities_web') - @property - def parent_type(self): - if self.service_id: - return 'Service' - @property def parent(self): return self.service @@ -525,6 +533,10 @@ class VulnerabilityCode(VulnerabilityGeneric): def hostnames(self): return [] + @property + def parent(self): + return self.source_code + class ReferenceTemplate(Metadata): __tablename__ = 'reference_template' @@ -573,6 +585,11 @@ def __init__(self, name=None, workspace_id=None, **kwargs): workspace_id=workspace_id, **kwargs) + @property + def parent(self): + # TODO: fix this propery + return + class ReferenceVulnerabilityAssociation(db.Model): @@ -649,6 +666,11 @@ def __init__(self, name=None, workspace_id=None, **kwargs): workspace_id=workspace_id, **kwargs) + @property + def parent(self): + # TODO: Fix this property + return + class Credential(Metadata): __tablename__ = 'credential' @@ -697,6 +719,10 @@ class Credential(Metadata): ), ) + @property + def parent(self): + return self.host or self.service + class Command(Metadata): __tablename__ = 'command' @@ -726,6 +752,10 @@ class Command(Metadata): foreign_keys=[entity_metadata_id] ) + @property + def parent(self): + return + def _make_vuln_count_property(type_=None): query = (select([func.count(text('vulnerability.id'))]). diff --git a/test_cases/factories.py b/test_cases/factories.py index e703ce60565..4c29d639829 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -144,7 +144,7 @@ class ServiceFactory(WorkspaceObjectFactory): description = FuzzyText() port = FuzzyInteger(1, 65535) protocol = FuzzyChoice(['TCP', 'UDP']) - host = factory.SubFactory(HostFactory) + host = factory.SubFactory(HostFactory, workspace=factory.SelfAttribute('..workspace')) status = FuzzyChoice(Service.STATUSES) creator = factory.SubFactory(UserFactory) @@ -208,8 +208,8 @@ def _after_postgeneration(cls, obj, create, results=None): class VulnerabilityFactory(HasParentHostOrService, VulnerabilityGenericFactory): - host = factory.SubFactory(HostFactory) - service = factory.SubFactory(ServiceFactory) + host = factory.SubFactory(HostFactory, workspace=factory.SelfAttribute('..workspace')) + service = factory.SubFactory(ServiceFactory, workspace=factory.SelfAttribute('..workspace')) class Meta: model = Vulnerability @@ -219,7 +219,7 @@ class Meta: class VulnerabilityWebFactory(VulnerabilityGenericFactory): method = FuzzyChoice(['GET', 'POST', 'PUT', 'PATCH' 'DELETE']) parameter_name = FuzzyText() - service = factory.SubFactory(ServiceFactory) + service = factory.SubFactory(ServiceFactory, workspace=factory.SelfAttribute('..workspace')) class Meta: model = VulnerabilityWeb @@ -249,8 +249,8 @@ class Meta: class CredentialFactory(HasParentHostOrService, WorkspaceObjectFactory): - host = factory.SubFactory(HostFactory) - service = factory.SubFactory(ServiceFactory) + host = factory.SubFactory(HostFactory, workspace=factory.SelfAttribute('..workspace')) + service = None # factory.SubFactory(ServiceFactory) username = FuzzyText() password = FuzzyText() diff --git a/test_cases/models/test_vulnerability.py b/test_cases/models/test_vulnerability.py index 80a73a41c98..fa6a44868bd 100644 --- a/test_cases/models/test_vulnerability.py +++ b/test_cases/models/test_vulnerability.py @@ -11,8 +11,9 @@ def test_standard_host_vuln_hostnames(vulnerability_factory, host_with_hostnames, session): vuln = vulnerability_factory.create(host=host_with_hostnames, - service=None) + service=None, workspace=host_with_hostnames.workspace) session.commit() + assert set(vuln.hostnames) == set(host_with_hostnames.hostnames) @@ -20,8 +21,8 @@ def test_standard_vuln_service_hostnames(vulnerability_factory, service_factory, host_with_hostnames, session): - service = service_factory.create(host=host_with_hostnames) - vuln = vulnerability_factory.create(service=service, host=None) + service = service_factory.create(host=host_with_hostnames, workspace=host_with_hostnames.workspace) + vuln = vulnerability_factory.create(service=service, host=None, workspace=host_with_hostnames.workspace) session.commit() assert set(vuln.hostnames) == set(host_with_hostnames.hostnames) @@ -30,8 +31,8 @@ def test_web_vuln_hostnames(vulnerability_web_factory, service_factory, host_with_hostnames, session): - service = service_factory.create(host=host_with_hostnames) - vuln = vulnerability_web_factory.create(service=service) + service = service_factory.create(host=host_with_hostnames, workspace=host_with_hostnames.workspace) + vuln = vulnerability_web_factory.create(service=service, workspace=host_with_hostnames.workspace) session.commit() assert set(vuln.hostnames) == set(host_with_hostnames.hostnames) diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 651651adbb1..0a57eea8f3f 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -258,9 +258,9 @@ def test_host_with_open_vuln_count_verification(self, test_client, session, service_factory): host = host_factory.create(workspace=workspace) - service = service_factory.create(host=host) - vulnerability_factory.create(service=service, host=None) - vulnerability_factory.create(service=None, host=host) + service = service_factory.create(host=host, workspace=workspace) + vulnerability_factory.create(service=service, host=None, workspace=workspace) + vulnerability_factory.create(service=None, host=host, workspace=workspace) session.commit() @@ -274,8 +274,8 @@ def test_host_services_vuln_count_verification(self, test_client, session, workspace, host_factory, vulnerability_factory, service_factory): host = host_factory.create(workspace=workspace) - service = service_factory.create(host=host) - vulnerability_factory.create(service=service, host=None) + service = service_factory.create(host=host, workspace=workspace) + vulnerability_factory.create(service=service, host=None, workspace=workspace) session.commit() res = test_client.get(self.url() + str(host.id) + "/" + 'services/') @@ -286,8 +286,11 @@ def test_create_host_with_hostnames(self, test_client): raw_data = { "ip": "192.168.0.21", "hostnames": [{"key": "google.com"}], - "mac": "00:00:00:00:00:00", "description": "", - "os": "", "owned": False, "owner": "" + "mac": "00:00:00:00:00:00", + "description": "", + "os": "", + "owned": False, + "owner": "" } res = test_client.post(self.url(), data=raw_data) assert res.status_code == 201 diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index 816aec0c3a0..ddd2d789e07 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -22,7 +22,7 @@ class TestListServiceView(ReadOnlyAPITests): #update_fields = ['ip', 'description', 'os'] view_class = ServiceView - def test_(self, test_client, second_workspace, session): + def test_service_list_backwards_compatibility(self, test_client, second_workspace, session): self.factory.create(workspace=second_workspace) session.commit() res = test_client.get(self.url()) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 8f48888805d..7f0c9c0aa8c 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -194,7 +194,7 @@ def test_create_vuln(self, host_with_hostnames, test_client, session): assert res.json['parent_type'] == 'Host' def test_create_vuln_web(self, host_with_hostnames, test_client, session): - service = ServiceFactory.create(host=host_with_hostnames) + service = ServiceFactory.create(host=host_with_hostnames, workspace=host_with_hostnames.workspace) session.commit() # flush host_with_hostnames raw_data = self._create_post_data_vulnerability( name='New vulns', diff --git a/test_cases/test_api_workspaced_base.py b/test_cases/test_api_workspaced_base.py index 16d971ccdff..a244ec227d7 100644 --- a/test_cases/test_api_workspaced_base.py +++ b/test_cases/test_api_workspaced_base.py @@ -4,7 +4,7 @@ import pytest from sqlalchemy.orm.util import was_deleted -from server.models import db, Workspace +from server.models import db, Workspace, Credential from test_api_pagination import PaginationTestsMixin as \ OriginalPaginationTestsMixin diff --git a/test_cases/test_model_events.py b/test_cases/test_model_events.py index e69de29bb2d..9f1842378f9 100644 --- a/test_cases/test_model_events.py +++ b/test_cases/test_model_events.py @@ -0,0 +1,18 @@ +from server import events +from test_cases.factories import HostFactory, ServiceFactory + + +def test_child_parent_verification_event(session, workspace, second_workspace): + host = HostFactory.build(workspace=workspace) + ServiceFactory.build(host=host, workspace=second_workspace) + try: + session.commit() + except AssertionError: + return True + return False + +def test_child_parent_verification_event(session, workspace): + host = HostFactory.build(workspace=workspace) + ServiceFactory.build(host=host, workspace=workspace) + session.commit() + return True \ No newline at end of file From b97cb8fbd79c2f76c839f435696a7f865c08bf15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 24 Oct 2017 16:56:31 -0300 Subject: [PATCH 0400/1506] Fix bug with filteralchemy integration I forgot to do a call to the parent method on my custom ModelConverter that was causing unexpected bugs. Added tests to it. --- server/api/base.py | 2 ++ server/api/modules/vulns.py | 8 ++++--- test_cases/test_api_vulnerability.py | 34 +++++++++++++++++++++++++--- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index aec554a8803..cf9b8e886c9 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -427,6 +427,8 @@ class FilterAlchemyModelConverter(ModelConverter): It is used to make filteralchemy support not nullable columns""" def _add_column_kwargs(self, kwargs, column): + super(FilterAlchemyModelConverter, self)._add_column_kwargs(kwargs, + column) kwargs['required'] = False diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 1c1d86cfca1..3be8512648b 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -246,12 +246,14 @@ class Meta: # default, and not by a similar one (like operator) _strict_filtering = {'default_operator': operators.Equal} + class VulnerabilityFilterSet(FilterSet): class Meta(FilterSetMeta): model = VulnerabilityWeb # It has all the fields - # TODO migration: Check if we should add fields creator, owner, command, - # impact, type, service, issuetracker, tags, date, target, host, - # easeofresolution, evidence, policy violations, hostnames, target + # TODO migration: Check if we should add fields creator, owner, + # command, impact, type, service, issuetracker, tags, date, target, + # host, easeofresolution, evidence, policy violations, hostnames, + # target fields = ( "status", "website", "parameter_name", "query_string", "path", "data", "severity", "confirmed", "name", "request", "response", diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 8f48888805d..a519e4db1a9 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -247,10 +247,38 @@ def test_filter_by_severity(self, test_client, session, assert vuln['severity'] == 'high' assert set(vuln['_id'] for vuln in res.json['data']) == expected_ids - # This shouldn't show any vulns since by default severity filter is - # exact + @pytest.mark.usefixtures('mock_envelope_list') + def test_filter_by_invalid_severity(self, test_client): + res = test_client.get(self.url() + '?severity=invalid') + assert res.status_code == 400 + + @pytest.mark.usefixtures('mock_envelope_list') + def test_filter_by_method(self, test_client, session, second_workspace, + vulnerability_factory, + vulnerability_web_factory): + + # Vulns that shouldn't be shown + vulnerability_factory.create_batch(5, workspace=second_workspace) + vulnerability_web_factory.create_batch(5, workspace=second_workspace, + method='POSTT') + + # Vulns that must be shown + expected_vulns = vulnerability_web_factory.create_batch( + 5, workspace=second_workspace, method='POST') + + session.commit() + expected_ids = {vuln.id for vuln in expected_vulns} + + # This shouldn't show any vulns with POSTT method + res = test_client.get(self.url( + workspace=second_workspace) + '?method=POST') + assert res.status_code == 200 + assert set(vuln['_id'] for vuln in res.json['data']) == expected_ids + + # This shouldn't show any vulns since by default method filter is + # an exact match, not a like statement res = test_client.get(self.url( - workspace=second_workspace) + '?severity=h%25') + workspace=second_workspace) + '?method=%25POST%25') assert res.status_code == 200 assert len(res.json['data']) == 0 From de50a242e20a6ff38047231d7837a4abe8acdb60 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 24 Oct 2017 17:15:11 -0300 Subject: [PATCH 0401/1506] Fix credential test --- server/api/modules/credentials.py | 34 +++++++++++++------ test_cases/test_api_credentials.py | 2 +- test_cases/test_api_vulnerability_template.py | 1 + 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/server/api/modules/credentials.py b/server/api/modules/credentials.py index 29422b798d9..28f85de4074 100644 --- a/server/api/modules/credentials.py +++ b/server/api/modules/credentials.py @@ -5,11 +5,10 @@ import flask from flask import Blueprint -from marshmallow import fields +from marshmallow import fields, post_load from server.api.base import AutoSchema, ReadWriteWorkspacedView -from server.models import Credential - +from server.models import Credential, Host, Service, db credentials_api = Blueprint('credentials_api', __name__) @@ -24,15 +23,14 @@ class CredentialSchema(AutoSchema): password = fields.String(default='') description = fields.String(default='') couchdbid = fields.String(default='') # backwards compatibility - - parent = fields.Method('get_parent') + parent_type = fields.Method('get_parent_type', required=True) + parent = fields.Method('get_parent', required=True) def get_parent(self, obj): - if getattr(obj, 'service', None): - return obj.service.id - if getattr(obj, 'host', None): - return obj.host.id - return + return obj.parent.id + + def get_parent_type(self, obj): + return obj.parent.__class__.__name__ def get_metadata(self, obj): return { @@ -52,8 +50,23 @@ class Meta: 'username', 'description', 'name', 'password', 'owner', 'owned', 'couchdbid', + 'parent', 'parent_type', 'metadata') + @post_load + def set_parent(self, data): + parent_type = data.pop('parent_type', None) + parent_id = data.pop('parent', None) + if parent_type == 'Host': + parent_class = Host + parent_field = 'host_id' + if parent_type == 'Service': + parent_class = Service + parent_field = 'service_id' + parent = db.session.query(parent_class).filter_by(id=parent_id).first() + data[parent_field] = parent.id + return data + class CredentialView(ReadWriteWorkspacedView): route_base = 'credential' @@ -72,4 +85,5 @@ def _envelope_list(self, objects, pagination_metadata=None): 'rows': credentials, } + CredentialView.register(credentials_api) diff --git a/test_cases/test_api_credentials.py b/test_cases/test_api_credentials.py index 5b6cf437f36..9c425903752 100644 --- a/test_cases/test_api_credentials.py +++ b/test_cases/test_api_credentials.py @@ -52,7 +52,7 @@ def test_create_from_raw_data_host_as_parent(self, session, test_client, "type":"Cred", "owner":"", "description":"", - "parent_id": host.id, + "parent": host.id, "parent_type": "Host" } res = test_client.post(self.url(), data=raw_data) diff --git a/test_cases/test_api_vulnerability_template.py b/test_cases/test_api_vulnerability_template.py index c3f2eae64a6..f2ad689907b 100644 --- a/test_cases/test_api_vulnerability_template.py +++ b/test_cases/test_api_vulnerability_template.py @@ -104,5 +104,6 @@ def test_delete_vuln_template(self, session, test_client): template = self.factory.create() vuln_count_previous = session.query(VulnerabilityTemplate).count() res = test_client.delete('/v2/vulnerability_template/{0}/'.format(template.id)) + assert res.status_code == 204 assert vuln_count_previous - 1 == session.query(VulnerabilityTemplate).count() \ No newline at end of file From 08327869fb7d43d96d39a1900f370ffe8b59cd4f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 24 Oct 2017 18:07:36 -0300 Subject: [PATCH 0402/1506] Updates on web ui to create credentials --- .../www/scripts/commons/providers/server.js | 4 +-- .../credentials/controllers/credentials.js | 22 +++++++-------- .../scripts/credentials/partials/list.html | 2 +- .../credentials/providers/credential.js | 28 ++++++++----------- 4 files changed, 25 insertions(+), 31 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index da353d06dd1..aae51b9a62a 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -296,8 +296,8 @@ angular.module("faradayApp") return get(getUrl, payload); } - ServerAPI.getObj = function(wsName, objID) { - var getUrl = createNewGetUrl(wsName, objID) + ServerAPI.getObj = function(wsName, objID, objectType) { + var getUrl = createNewGetUrl(wsName, objID, objectType) + "/"; return get(getUrl); } diff --git a/server/www/scripts/credentials/controllers/credentials.js b/server/www/scripts/credentials/controllers/credentials.js index 60960e621a7..4f9b2faef89 100644 --- a/server/www/scripts/credentials/controllers/credentials.js +++ b/server/www/scripts/credentials/controllers/credentials.js @@ -29,10 +29,10 @@ angular.module('faradayApp') if($routeParams.hId !== undefined){ // Load all host information needed. - $scope.parentObject.type = 'Host'; + $scope.parentObject.parent_type = 'Host'; $scope.parentObject.id = $routeParams.hId; - ServerAPI.getObj($scope.workspace, $scope.parentObject.id).then(function (response) { + ServerAPI.getObj($scope.workspace, $scope.parentObject.id, 'hosts').then(function (response) { $scope.parentObject.nameHost = response['data']['name']; deferred.resolve(); }); @@ -42,16 +42,16 @@ angular.module('faradayApp') if($routeParams.sId !== undefined){ // Load all service information needed. - $scope.parentObject.type = 'Service'; + $scope.parentObject.parent_type = 'Service'; $scope.parentObject.id = $routeParams.sId; - ServerAPI.getObj($scope.workspace, $scope.parentObject.id).then(function (response) { + ServerAPI.getObj($scope.workspace, $scope.parentObject.id, 'services').then(function (response) { $scope.parentObject.nameService = response['data']['name']; // and also, load all host information needed. var hostId = response['data']['_id'].split('.')[0]; - ServerAPI.getObj($scope.workspace, hostId).then(function (response) { + ServerAPI.getObj($scope.workspace, hostId, 'hosts').then(function (response) { $scope.parentObject.nameHost = response['data']['name']; deferred.resolve(); }); @@ -77,16 +77,16 @@ angular.module('faradayApp') var getAndLoadCredentials = function() { // Load all credentials, we dont have a parent. - if($scope.parentObject.type === undefined){ + if($scope.parentObject.parent_type === undefined){ ServerAPI.getCredentials($scope.workspace).then(function(response){ loadCredentials(response.data.rows); }); } else { // Load all credentials, filtered by host internal id or service internal id. - if ($scope.parentObject.type === 'Host') + if ($scope.parentObject.parent_type === 'Host') var data = {'host_id': $scope.parentObject.id}; - else if ($scope.parentObject.type === 'Service') + else if ($scope.parentObject.parent_type === 'Service') var data = {'service_id': $scope.parentObject.id}; ServerAPI.getCredentials($scope.workspace, data).then(function(response){ @@ -144,10 +144,10 @@ angular.module('faradayApp') return $q.all(confirmations); }; - var createCredential = function(credentialData, parent_id){ + var createCredential = function(credentialData, parent_id, parent_type){ // Add parent id, create credential and save to server. try { - var credentialObj = new credential(credentialData, parent_id); + var credentialObj = new credential(credentialData, parent_id, parent_type); credentialObj.create($scope.workspace).then(function(){ $scope.credentials.push(credentialObj); @@ -188,7 +188,7 @@ angular.module('faradayApp') }); modal.result .then(function(data) { - createCredential(data, $scope.parentObject.id); + createCredential(data, $scope.parentObject.id, $scope.parentObject.parent_type); }); }; diff --git a/server/www/scripts/credentials/partials/list.html b/server/www/scripts/credentials/partials/list.html index 98a7bf449cf..f265b0bd350 100644 --- a/server/www/scripts/credentials/partials/list.html +++ b/server/www/scripts/credentials/partials/list.html @@ -31,7 +31,7 @@

    Edit - diff --git a/server/www/scripts/credentials/providers/credential.js b/server/www/scripts/credentials/providers/credential.js index 00f0d9569bd..f35a960f821 100644 --- a/server/www/scripts/credentials/providers/credential.js +++ b/server/www/scripts/credentials/providers/credential.js @@ -10,7 +10,6 @@ angular.module('faradayApp') // All credentials need this properties minimum for build object. var _credentialFields = { - '_id': 'string', 'name': 'string', 'username': 'string', 'password': 'string', @@ -19,31 +18,31 @@ angular.module('faradayApp') // Only this properties will be saved to server. var _credentialFieldsSaveToServer = { - '_id': 'string', - '_rev': 'string', 'name': 'string', 'username': 'string', 'metadata': 'string', 'password': 'string', 'type': 'string', + 'parent_type': 'string', + 'parent': 'string' }; var Credential; - Credential = function(data, parent){ + Credential = function(data, parent, parent_type){ if(data) { - this.set(data, parent); + this.set(data, parent, parent_type); } }; Credential.prototype = { // Build object. - set: function(data, parent) { + set: function(data, parent, parent_type) { data.type = 'Cred'; + data.parent_type = parent_type; + data.parent = parent; if(data.metadata === undefined) data.metadata = ''; - if(data._id === undefined && parent) - data['_id'] = _generateID(parent, data.name, data.username, data.password); _checkFieldsOk(data); angular.extend(this, data); @@ -94,12 +93,12 @@ angular.module('faradayApp') var deferred = $q.defer(); - var result = this._id.split('.'); + var result = this.parent; var hostIdToSearch = undefined; var serviceIdToSearch = undefined; //Parent is Host - if (result.length === 2){ + if (result.parent_type === 'Host'){ hostIdToSearch = result[0]; ServerAPI.getObj(ws, hostIdToSearch).then(function(response){ @@ -108,9 +107,9 @@ angular.module('faradayApp') } //Parent is Service - else if (result.length === 4){ + else if (result.lparent_type == 'Service'){ hostIdToSearch = result[0]; - serviceIdToSearch = result.slice(0, result.length - 1).join('.'); + serviceIdToSearch = result.parent; ServerAPI.getObj(ws, hostIdToSearch).then(function(responseHost){ ServerAPI.getObj(ws, serviceIdToSearch).then(function(responseService){ @@ -123,11 +122,6 @@ angular.module('faradayApp') } }; - var _generateID = function(parent, name, username, password){ - var id = parent + '.' + CryptoJS.SHA1([name, username, password].join('._.')).toString(); - return id; - }; - // Check object to construct have all fields and also, type of they are OK. // All fields in _credentialFields should are in object. var _checkFieldsOk = function(credential){ From d4936b81160e16c945a39e30eb89160688670ad4 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 24 Oct 2017 19:35:03 -0300 Subject: [PATCH 0403/1506] Add PIL missing dep for thumbs --- requirements_server.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_server.txt b/requirements_server.txt index a747c3978d5..0ea4deb4317 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -6,6 +6,7 @@ flask-classful==0.14 Flask-Security==3.0.0 flask>=0.10.1 marshmallow +Pillow==4.2.1 pyasn1-modules pyopenssl>=16.0.0 python-slugify==1.2.4 From 5c8d5a78909b5390883a4bdd91a3e750399dd9b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 24 Oct 2017 19:39:04 -0300 Subject: [PATCH 0404/1506] Allow filtering on count endpoints --- server/api/base.py | 13 ++++++++++--- test_cases/test_api_vulnerability.py | 23 +++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index cf9b8e886c9..161e5b8b40f 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -80,6 +80,10 @@ def _validate_object_id(self, object_id): def _get_base_query(self): return self.model_class.query + def _filter_query(self, query): + """Return a new SQLAlchemy query with some filters applied""" + return query + def _get_object(self, object_id, **kwargs): self._validate_object_id(object_id) try: @@ -383,9 +387,12 @@ def count(self, **kwargs): table_name = inspect(self.model_class).tables[0].name group_by = '{0}.{1}'.format(table_name, group_by) - count = db.session.query(self.model_class).join(Workspace).group_by(group_by).filter( - Workspace.name == workspace_name).values(group_by, func.count(group_by)) - for key, count in count: + count = self._filter_query( + db.session.query(self.model_class) + .join(Workspace) + .group_by(group_by) + .filter(Workspace.name == workspace_name)) + for key, count in count.values(group_by, func.count(group_by)): res['groups'].append( {'count': count, 'name': key} ) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index a519e4db1a9..bc5224f1e4f 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -410,3 +410,26 @@ def test_create_vuln_with_invalid_type(self, host_with_hostnames, test_client, s assert res.status_code == 400 assert vuln_count_previous == session.query(Vulnerability).count() assert res.json['message'] == 'Invalid vulnerability type.' + + def test_count_confirmed(self, test_client, session): + for i, vuln in enumerate(self.objects[:3]): + vuln.confirmed = True + + # Set critical severity to first vuln, high to the others + if i == 0: + vuln.severity = 'critical' + else: + vuln.severity = 'high' + + session.add(vuln) + session.commit() + + print self.url() + res = test_client.get(self.url() + + 'count/?confirmed=1&group_by=severity') + assert res.status_code == 200 + assert res.json['total_count'] == 3 + assert sorted(res.json['groups']) == sorted([ + {"name": "high", "count": 2}, + {"name": "critical", "count": 1}, + ]) From d3174e31dbadf898df752c63ee1c579ddd3c4262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 24 Oct 2017 19:44:07 -0300 Subject: [PATCH 0405/1506] I forgot to add some code to the last commit (5c8d5a7) 5c8d5a78909b5390883a4bdd91a3e750399dd9b0 --- server/api/base.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 161e5b8b40f..f7afc79917f 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -191,10 +191,6 @@ def _envelope_list(self, objects, pagination_metadata=None): def _paginate(self, query): return query, None - def _filter_query(self, query): - """Return a new SQLAlchemy query with some filters applied""" - return query - def index(self, **kwargs): query = self._filter_query(self._get_base_query(**kwargs)) objects, pagination_metadata = self._paginate(query) From 7c22c70d938a3e7cf1d2ddc49100fe009aa6c52a Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 25 Oct 2017 13:45:20 -0300 Subject: [PATCH 0406/1506] Add credentials filtering --- server/api/modules/credentials.py | 26 ++++++++++++++++++++++++-- test_cases/test_api_credentials.py | 18 ++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/server/api/modules/credentials.py b/server/api/modules/credentials.py index 28f85de4074..2eeeee4953d 100644 --- a/server/api/modules/credentials.py +++ b/server/api/modules/credentials.py @@ -6,8 +6,13 @@ import flask from flask import Blueprint from marshmallow import fields, post_load +from filteralchemy import FilterSet, operators -from server.api.base import AutoSchema, ReadWriteWorkspacedView +from server.api.base import ( + AutoSchema, + ReadWriteWorkspacedView, + FilterSetMeta, + FilterAlchemyMixin) from server.models import Credential, Host, Service, db credentials_api = Blueprint('credentials_api', __name__) @@ -26,6 +31,10 @@ class CredentialSchema(AutoSchema): parent_type = fields.Method('get_parent_type', required=True) parent = fields.Method('get_parent', required=True) + # for filtering + host_id = fields.Integer(load_only=True) + service_id = fields.Integer(load_only=True) + def get_parent(self, obj): return obj.parent.id @@ -68,10 +77,23 @@ def set_parent(self, data): return data -class CredentialView(ReadWriteWorkspacedView): +class CredentialFilterSet(FilterSet): + class Meta(FilterSetMeta): + model = Credential + fields = ( + 'host_id', + 'service_id' + ) + + default_operator = operators.Equal + operators = (operators.Equal, ) + + +class CredentialView(FilterAlchemyMixin, ReadWriteWorkspacedView): route_base = 'credential' model_class = Credential schema_class = CredentialSchema + filterset_class = CredentialFilterSet def _envelope_list(self, objects, pagination_metadata=None): credentials = [] diff --git a/test_cases/test_api_credentials.py b/test_cases/test_api_credentials.py index 9c425903752..ce12be24cb5 100644 --- a/test_cases/test_api_credentials.py +++ b/test_cases/test_api_credentials.py @@ -4,6 +4,7 @@ ) from server.api.modules.credentials import CredentialView from server.models import Credential +from test_cases.factories import ServiceFactory class TestCredentialsAPIGeneric(ReadOnlyAPITests): @@ -57,3 +58,20 @@ def test_create_from_raw_data_host_as_parent(self, session, test_client, } res = test_client.post(self.url(), data=raw_data) assert res.status_code == 201 + + def test_get_credentials_for_a_host_backwards_compatibility(self, session, test_client): + credential = self.factory.create() + session.commit() + res = test_client.get(self.url(workspace=credential.workspace) + '?host_id={0}'.format(credential.host.id)) + assert res.status_code == 200 + assert map(lambda cred: cred['value']['parent'],res.json['rows']) == [credential.host.id] + assert map(lambda cred: cred['value']['parent_type'], res.json['rows']) == [u'Host'] + + def test_get_credentials_for_a_service_backwards_compatibility(self, session, test_client): + service = ServiceFactory.create() + credential = self.factory.create(service=service, host=None, workspace=service.workspace) + session.commit() + res = test_client.get(self.url(workspace=credential.workspace) + '?service={0}'.format(credential.service.id)) + assert res.status_code == 200 + assert map(lambda cred: cred['value']['parent'],res.json['rows']) == [credential.service.id] + assert map(lambda cred: cred['value']['parent_type'], res.json['rows']) == [u'Service'] From dbf5ca33fcf0128c7018c1969cdd616a601cf88a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 25 Oct 2017 14:09:51 -0300 Subject: [PATCH 0407/1506] Change typo in Jenkinsfile I used Flask-Classful egg name to install filteralchemy --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index da8316eaa5e..971abc6b13a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -25,7 +25,7 @@ node { pip install -r $WORKSPACE/requirements_extras.txt pip install -r $WORKSPACE/requirements_dev.txt pip uninstall -y filteralchemy - pip install -e git+https://github.com/sh4r3m4n/filteralchemy@dev#egg=Flask_Classful + pip install -e git+https://github.com/sh4r3m4n/filteralchemy@dev#egg=filteralchemy deactivate """ } From e794214d2fb48e404380674651065a0106fbb37c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 25 Oct 2017 14:32:22 -0300 Subject: [PATCH 0408/1506] SOme web ui fixes --- server/www/scripts/credentials/controllers/credentials.js | 2 +- server/www/scripts/credentials/providers/credential.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/credentials/controllers/credentials.js b/server/www/scripts/credentials/controllers/credentials.js index 4f9b2faef89..25585a0f5f7 100644 --- a/server/www/scripts/credentials/controllers/credentials.js +++ b/server/www/scripts/credentials/controllers/credentials.js @@ -65,7 +65,7 @@ angular.module('faradayApp') var loadCredentials = function (credentials){ credentials.forEach(function(cred){ - var object = new credential(cred.value); + var object = new credential(cred.value, cred.value.parent, cred.value.parent_type); object.getParentName($scope.workspace).then(function(response){ object.target = response; }); diff --git a/server/www/scripts/credentials/providers/credential.js b/server/www/scripts/credentials/providers/credential.js index f35a960f821..49e9ae4f353 100644 --- a/server/www/scripts/credentials/providers/credential.js +++ b/server/www/scripts/credentials/providers/credential.js @@ -54,7 +54,7 @@ angular.module('faradayApp') var deferred = $q.defer(); var self = this; - ServerAPI.getObj(ws, id).then(function(response){ + ServerAPI.getObj(ws, id, 'credential').then(function(response){ angular.extend(self, response.data); deferred.resolve(); }); From dd4eeac5d598fbcda70a8735ec15651c8cd5c6a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 25 Oct 2017 14:36:36 -0300 Subject: [PATCH 0409/1506] Add debugging reset_db command --- manage.py | 2 ++ server/commands/reset_db.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 server/commands/reset_db.py diff --git a/manage.py b/manage.py index 1eb259507bb..8f01b04d1bc 100755 --- a/manage.py +++ b/manage.py @@ -10,6 +10,7 @@ from server.commands.initdb import InitDB from server.commands.faraday_schema_display import DatabaseSchema from server.commands.app_urls import AppUrls +from server.commands.reset_db import ResetDB manager = Manager(app) @@ -20,4 +21,5 @@ manager.add_command('initdb', InitDB()) manager.add_command('faraday_schema_display', DatabaseSchema()) manager.add_command('show_urls', AppUrls()) + manager.add_command('reset_db', ResetDB()) manager.run() diff --git a/server/commands/reset_db.py b/server/commands/reset_db.py new file mode 100644 index 00000000000..96f4714a3bf --- /dev/null +++ b/server/commands/reset_db.py @@ -0,0 +1,14 @@ +from flask_script import Command +from server.models import db + +class ResetDB(Command): + def run(self): + # It might be required to do a cascade delete to correctly the + # vulnerability table + try: + db.engine.execute('DROP TABLE vulnerability CASCADE') + except: + pass + db.drop_all() + db.create_all() + From 0eb9192d4c77dafd673bc477ffdf6a6db3451e91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 25 Oct 2017 14:37:01 -0300 Subject: [PATCH 0410/1506] Fix bug when importing vulns with unknown severity It was failing with KeyError, now catch it by using "unclassified" severity and do a warning. Also improve the message when unknown service status found --- server/importer.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server/importer.py b/server/importer.py index 42f68a26c5a..d960edf29da 100644 --- a/server/importer.py +++ b/server/importer.py @@ -377,7 +377,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation } couchdb_status = document.get('status', 'open') if couchdb_status.lower() not in status_mapper: - logger.warn('Service with status {0} found! Status will default to open. Host is {1}'.format(couchdb_status, host.ip)) + logger.warn('Service with unknown status "{0}" found! Status will default to open. Host is {1}'.format(couchdb_status, host.ip)) service.status = status_mapper.get(couchdb_status, 'open') service.version = document.get('version') service.workspace = workspace @@ -394,7 +394,13 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation couch_parent_id = '.'.join(document['_id'].split('.')[:-1]) parent_ids = couchdb_relational_map[couch_parent_id] mapped_severity = MAPPED_VULN_SEVERITY - severity = mapped_severity[document.get('severity')] + try: + severity = mapped_severity[document.get('severity')] + except KeyError: + logger.warn("Unknown severity value '%s' of vuln with id %s. " + "Using 'unclassified'", + document.get('severity'), document['_id']) + severity = 'unclassified' for parent_id in parent_ids: if level == 2: parent = session.query(Host).filter_by(id=parent_id).first() From ff8dd7993653113363f3b33cbf934a6ac773bb20 Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 25 Oct 2017 14:53:25 -0300 Subject: [PATCH 0411/1506] Fix updateObject method in Server API factory Change URL generation replacing PostUrl with PutUrl. --- server/www/scripts/commons/providers/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index c92203f1e90..b9fd3dde087 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -192,7 +192,7 @@ angular.module("faradayApp") }; var updateObject = function(wsName, id, data, collectionName) { - var postUrl = createPostUrl(wsName, id, collectionName); + var postUrl = createPutUrl(wsName, id, collectionName); return send_data(postUrl, data, true, "PUT"); } From 7b299fe2d4c7bd5d07d6d70d675ab25ca4fc5150 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 25 Oct 2017 14:55:24 -0300 Subject: [PATCH 0412/1506] Update delete credential code --- server/www/scripts/commons/providers/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index aae51b9a62a..0308cdeb04e 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -466,7 +466,7 @@ angular.module("faradayApp") } ServerAPI.deleteCredential = function(wsName, credentialId, rev) { - var deleteUrl = createDeleteUrl(wsName, credentialId, rev); + var deleteUrl = createDeleteUrl(wsName, credentialId, 'credential'); if (typeof rev === "undefined") { return _delete(deleteUrl, false) } From c7bd8528f1e3a1627273db3c932aeec64c83963e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 25 Oct 2017 15:02:00 -0300 Subject: [PATCH 0413/1506] Update code to use new api --- server/www/scripts/commons/providers/server.js | 2 +- server/www/scripts/credentials/providers/credential.js | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index e3f2e6df691..977d839e373 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -173,7 +173,7 @@ angular.module("faradayApp") var modCredential = function(createOrUpdate, wsName, credential) { if (typeof credential.owner === "undefined") {credential.owner = ""}; if (typeof credential.description === "undefined") {credential.description = ""}; - return createOrUpdate(wsName, credential._id, credential, 'credential'); + return createOrUpdate(wsName, credential.id, credential, 'credential'); } var modCommand = function(createOrUpdate, wsName, command) { diff --git a/server/www/scripts/credentials/providers/credential.js b/server/www/scripts/credentials/providers/credential.js index 49e9ae4f353..7765cf99c35 100644 --- a/server/www/scripts/credentials/providers/credential.js +++ b/server/www/scripts/credentials/providers/credential.js @@ -18,6 +18,7 @@ angular.module('faradayApp') // Only this properties will be saved to server. var _credentialFieldsSaveToServer = { + 'id': 'integer', 'name': 'string', 'username': 'string', 'metadata': 'string', @@ -72,10 +73,7 @@ angular.module('faradayApp') var self = this; self.metadata = updateMetadata(self.metadata); - return ServerAPI.updateCredential(ws, buildObjectServer(self)) - .then(function(credentialData) { - self._rev = credentialData.rev; - }); + return ServerAPI.updateCredential(ws, buildObjectServer(self)); }, // Create object in server. From dffcf1f3761a2fd393a7bccc86d91bf1c8d87112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 25 Oct 2017 16:29:16 -0300 Subject: [PATCH 0414/1506] Improve workspaces API Make scope return a list of domains instead of IDs, and undeferer count fields to do one query instead of one per workspace. We still have to improve the scope loading to do it in one request --- server/api/modules/workspaces.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index e9a375519be..19cb9b7837b 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -6,6 +6,7 @@ import flask from flask import Blueprint from marshmallow import Schema, fields +from sqlalchemy.orm import undefer from server.models import db, Workspace from server.dao.host import HostDAO @@ -13,7 +14,11 @@ from server.dao.service import ServiceDAO from server.dao.workspace import WorkspaceDAO from server.utils.logger import get_logger -from server.schemas import SelfNestedField, JSTimestampField +from server.schemas import ( + JSTimestampField, + PrimaryKeyRelatedField, + SelfNestedField, +) from server.utils.web import ( build_bad_request_response, filter_request_args, @@ -55,6 +60,7 @@ class WorkspaceSchema(AutoSchema): stats = SelfNestedField(WorkspaceSummarySchema()) duration = SelfNestedField(WorkspaceDurationSchema()) _id = fields.Integer(dump_only=True, attribute='id') + scope = PrimaryKeyRelatedField('name', many=True, dump_only=True) class Meta: model = Workspace @@ -69,4 +75,16 @@ class WorkspaceView(ReadWriteView): model_class = Workspace schema_class = WorkspaceSchema + def _get_base_query(self): + original = super(WorkspaceView, self)._get_base_query() + return original.options( + undefer(Workspace.host_count), + undefer(Workspace.credential_count), + undefer(Workspace.service_count), + undefer(Workspace.vulnerability_web_count), + undefer(Workspace.vulnerability_code_count), + undefer(Workspace.vulnerability_standard_count), + undefer(Workspace.vulnerability_total_count), + ) + WorkspaceView.register(workspace_api) From 293ac5d740dec8a7de01e83e24da0141d7bb0a9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 25 Oct 2017 17:38:53 -0300 Subject: [PATCH 0415/1506] Fix dashboard vuln severity graphs It was broken due to a change of schema of the vuln count API --- server/api/base.py | 6 +++++- test_cases/test_api_vulnerability.py | 5 ++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index f7afc79917f..e4227a8a201 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -390,7 +390,11 @@ def count(self, **kwargs): .filter(Workspace.name == workspace_name)) for key, count in count.values(group_by, func.count(group_by)): res['groups'].append( - {'count': count, 'name': key} + {'count': count, + 'name': key, + # To add compatibility with the web ui + flask.request.args.get('group_by'): key, + } ) res['total_count'] += count return res diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 7bdc79487b0..6324288a2fa 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -424,12 +424,11 @@ def test_count_confirmed(self, test_client, session): session.add(vuln) session.commit() - print self.url() res = test_client.get(self.url() + 'count/?confirmed=1&group_by=severity') assert res.status_code == 200 assert res.json['total_count'] == 3 assert sorted(res.json['groups']) == sorted([ - {"name": "high", "count": 2}, - {"name": "critical", "count": 1}, + {"name": "high", "severity": "high", "count": 2}, + {"name": "critical", "severity": "critical", "count": 1}, ]) From a99d48521ebb5d680ba3c3b585f9e14a6222205d Mon Sep 17 00:00:00 2001 From: Mike Zhong Date: Wed, 25 Oct 2017 17:46:14 -0300 Subject: [PATCH 0416/1506] handles exception when user attempts to run faraday.py while another instance is already running --- AUTHORS | 11 ++++++----- model/api.py | 4 ++++ model/application.py | 14 ++++++++------ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/AUTHORS b/AUTHORS index 84d7816dfc5..206bbd1d1a7 100755 --- a/AUTHORS +++ b/AUTHORS @@ -1,13 +1,13 @@ The PRIMARY AUTHORS are: - * Daniel Foguelman - * Esteban Guillardoy + * Daniel Foguelman + * Esteban Guillardoy * Ezequiel Tavella - * Facundo de Guzmán - * Federico Kirschbaum + * Facundo de Guzmán + * Federico Kirschbaum * Francisco Amato * Franco Linares - * German Riera + * German Riera * Joaquín López Pereyra * Leonardo Lazzaro * Martín Rocha @@ -37,3 +37,4 @@ Project contributors * tsxltjecwb * Ulisses Albuquerque * xtr4nge + * Mike Zhong (go bears) diff --git a/model/api.py b/model/api.py index cb3059a2ceb..1caae280c72 100644 --- a/model/api.py +++ b/model/api.py @@ -121,6 +121,10 @@ def _setUpAPIServer(hostname=None, port=None): raise Exception("No ports available!") CONF.setApiConInfo(hostname, port) CONF.saveConfig() + elif exception.errno == 48: + # Address already open + # Another instance of faraday.py already running + raise Exception("Another instance of faraday.py already running!") else: raise exception except Exception as e: diff --git a/model/application.py b/model/application.py index f8a6c872cae..00d62062ac1 100644 --- a/model/application.py +++ b/model/application.py @@ -149,11 +149,11 @@ def start(self): exit_code = self.app.run(self.args) - except Exception: - print "There was an error while starting Faraday" - print "-" * 50 - traceback.print_exc() - print "-" * 50 + except Exception as exception: + print "There was an error while starting Faraday:" + print "*" * 3, + print exception, # instead of traceback.print_exc() + print "*" * 3 exit_code = -1 finally: @@ -169,7 +169,9 @@ def __exit(self, exit_code=0): model.api.stopAPIServer() restapi.stopServer() self._model_controller.stop() - self._model_controller.join() + if self._model_controller.isAlive(): + # runs only if thread has started, i.e. self._model_controller.start() is run first + self._model_controller.join() self.timer.stop() model.api.devlog("Waiting for controller threads to end...") return exit_code From 4b92775feed392c473b4b577f503b5eee911f392 Mon Sep 17 00:00:00 2001 From: Mike Zhong Date: Wed, 25 Oct 2017 17:47:07 -0300 Subject: [PATCH 0417/1506] added DNS records: AAAA, MX, DNS, SOA, and TXT to the dig plugin! --- plugins/repo/dig/plugin.py | 146 +++++++++++++++++++++++++++++++------ 1 file changed, 122 insertions(+), 24 deletions(-) diff --git a/plugins/repo/dig/plugin.py b/plugins/repo/dig/plugin.py index 0fd34b329be..7c6576ab4db 100644 --- a/plugins/repo/dig/plugin.py +++ b/plugins/repo/dig/plugin.py @@ -1,10 +1,13 @@ # -*- coding: utf-8 -*- """ +Updated by Mike Zhong, 25 Oct 2017. + Faraday Penetration Test IDE Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) See the file 'doc/LICENSE' for the license information """ import re +import socket from plugins import core @@ -41,33 +44,128 @@ def parseOutputString(self, output): # Parse results results = [] answer_section_columns = [u"domain", - u"ttl", u"class", u"type", u"address"] + u"ttl", u"class", u"type", u"data"] for line in parsed_output: - results.append(dict(zip(answer_section_columns, line.split()))) + line_split = line.split() # the first 4 elements are domain, ttl, class, type; everything else data + results.append(dict(zip(answer_section_columns, line_split[:4] + [line_split[4:]] ))) # Create hosts is results information is relevant - for result in results: - relevant_types = [u"A", u"AAAA"] - if result.get(u"type") in relevant_types: - ip_address = result.get(u"address") - domain = result.get(u"domain") - - # Create host - host = self.createAndAddHost(ip_address) - - # Create interface - if len(ip_address.split(".")) == 4: - iface = self.createAndAddInterface( - host, - ip_address, - ipv4_address=ip_address, - hostname_resolution=[domain]) - else: - iface = self.createAndAddInterface( - host, - ip_address, - ipv6_address=ip_address, - hostname_resolution=[domain]) + try: + for result in results: + relevant_types = [u"A", u"AAAA", u"MX", u"NS", u"SOA", u"TXT"] + # TODO implement more types from https://en.wikipedia.org/wiki/List_of_DNS_record_types + + if result.get(u"type") in relevant_types: + + # get domain + domain = result.get(u"domain") + + + # get IP address (special if type "A") + if result.get(u"type") == u"A": # A = IPv4 address from dig + ip_address = result.get(u"data")[0] + else: # if not, from socket + ip_address = socket.gethostbyname(domain) + + # Create host + host_id = self.createAndAddHost(ip_address) + + # create interface (special if type "AAAA") + if result.get(u"type") == u"AAAA": # AAAA = IPv6 address + # TODO is there a function to dynamically update the paramter ipv6_address of an already-created interface? + ipv6_address = result.get(u"data")[0] + interface_id = self.createAndAddInterface( + host_id, + ip_address, + ipv4_address=ip_address, + ipv6_address=ipv6_address, + hostname_resolution=[domain]) + else: + interface_id = self.createAndAddInterface( + host_id, + ip_address, + ipv4_address=ip_address, + hostname_resolution=[domain]) + + + # all other TYPES that aren't 'A' and 'AAAA' are dealt here: + if result.get(u"type") == u"MX": # Mail exchange record + mx_priority = result.get(u"data")[0] + mx_record = result.get(u"data")[1] + + service_id = self.createAndAddServiceToInterface( + host_id=host_id, + interface_id=interface_id, + name=mx_record, + protocol="SMTP", + ports=[25], + description="E-mail Server") + + text = "Priority: " + mx_priority + self. + + + + + + ( + host_id=host_id, + service_id=service_id, + name="priority", + text=text.encode('ascii', 'ignore')) + + elif result.get(u"type") == u"NS": # Name server record + ns_record = result.get(u"data")[0] + self.createAndAddServiceToInterface( + host_id=host_id, + interface_id=interface_id, + name=ns_record, + protocol="DNS", + ports=[53], + description="DNS Server") + + elif result.get(u"type") == u"SOA": # Start of Authority Record + ns_record = result.get(u"data")[0] # primary namer server + responsible_party = result.get(u"data")[1] # responsible party of domain + timestamp = result.get(u"data")[2] + refresh_zone_time = result.get(u"data")[3] + retry_refresh_time = result.get(u"data")[4] + upper_limit_time = result.get(u"data")[5] + negative_result_ttl = result.get(u"data")[6] + + service_id = self.createAndAddServiceToInterface( + host_id=host_id, + interface_id=interface_id, + name=ns_record, + protocol="DNS", + ports=[53], + description="Authority Record") + + text = ( + "Responsible Party: " + responsible_party + + "\nTimestep: " + timestamp + + "\nTime before zone refresh (sec): " + refresh_zone_time + + "\nTime before retry refresh (sec): " + retry_refresh_time + + "\nUpper Limit before Zone is no longer authoritive (sec): " + upper_limit_time + + "\nNegative Result TTL: " + negative_result_ttl) + + self.createAndAddNoteToService( + host_id=host_id, + service_id=service_id, + name="priority", + text=text.encode('ascii', 'ignore')) + + elif result.get(u"type") == u"TXT": # TXT record + text = " ".join(result.get(u"data")[:]) + self.createAndAddNoteToHost( + host_id=host_id, + name="TXT Information", + text=text.encode('ascii', 'ignore')) + + except: + print("some part of the dig plug-in caused an error! Please check repo/dig/plugin.py") + return False + return True From 2ef1399c8059d7f14df8cce9da39ac24f3c88b00 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 25 Oct 2017 17:01:43 -0300 Subject: [PATCH 0418/1506] Add test for credential. fix a bug when parent id is invalid --- server/api/modules/credentials.py | 4 +++- test_cases/test_api_credentials.py | 38 ++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/server/api/modules/credentials.py b/server/api/modules/credentials.py index 2eeeee4953d..e1e5ba31de4 100644 --- a/server/api/modules/credentials.py +++ b/server/api/modules/credentials.py @@ -12,7 +12,7 @@ AutoSchema, ReadWriteWorkspacedView, FilterSetMeta, - FilterAlchemyMixin) + FilterAlchemyMixin, InvalidUsage) from server.models import Credential, Host, Service, db credentials_api = Blueprint('credentials_api', __name__) @@ -73,6 +73,8 @@ def set_parent(self, data): parent_class = Service parent_field = 'service_id' parent = db.session.query(parent_class).filter_by(id=parent_id).first() + if not parent: + raise InvalidUsage('Parent not found.') data[parent_field] = parent.id return data diff --git a/test_cases/test_api_credentials.py b/test_cases/test_api_credentials.py index ce12be24cb5..f9b473a8dec 100644 --- a/test_cases/test_api_credentials.py +++ b/test_cases/test_api_credentials.py @@ -75,3 +75,41 @@ def test_get_credentials_for_a_service_backwards_compatibility(self, session, te assert res.status_code == 200 assert map(lambda cred: cred['value']['parent'],res.json['rows']) == [credential.service.id] assert map(lambda cred: cred['value']['parent_type'], res.json['rows']) == [u'Service'] + + def _generate_raw_update_data(self, name, username, password, parent_id): + return { + "id": 7, + "name": name, + "username": username, + "metadata": {"update_time": 1508960699994, "create_time": 1508965372, + "update_user": "", "update_action": 0, "creator": "Metasploit", "owner": "", + "update_controller_action": "No model controller call", + "command_id": "e1a042dd0e054c1495e1c01ced856438"}, + "password": password, + "type": "Cred", + "parent_type": "Host", + "parent": parent_id, + "owner": "", + "description": "", + "_rev": ""} + + def test_update_credentials_with_invalid_parent(self, test_client, session): + credential = self.factory.create() + session.commit() + + raw_data = self._generate_raw_update_data('Name1', 'Username2', 'Password3', parent_id=43) + + res = test_client.put(self.url(workspace=credential.workspace) + str(credential.id) + '/', data=raw_data) + assert res.status_code == 400 + + def test_update_credentials(self, test_client, session): + credential = self.factory.create() + session.commit() + + raw_data = self._generate_raw_update_data('Name1', 'Username2', 'Password3', parent_id=credential.host.id) + + res = test_client.put(self.url(workspace=credential.workspace) + str(credential.id) + '/', data=raw_data) + assert res.status_code == 200 + assert res.json['username'] == u'Username2' + assert res.json['password'] == u'Password3' + assert res.json['name'] == u'Name1' From 3fd37ff2f33cffff4bd8f4ac6ca0bfc02f8702ae Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 25 Oct 2017 18:43:40 -0300 Subject: [PATCH 0419/1506] Add logging when an update fails --- server/api/base.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/api/base.py b/server/api/base.py index e4227a8a201..dcd07b636a3 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -15,6 +15,9 @@ from webargs.flaskparser import FlaskParser, parser, abort from webargs.core import ValidationError from server.models import Workspace, db +import server.utils.logger + +logger = server.utils.logger.get_logger(__name__) def output_json(data, code, headers=None): @@ -326,7 +329,10 @@ def put(self, object_id, **kwargs): def _update_object(self, obj, data): for (key, value) in data.items(): - setattr(obj, key, value) + try: + setattr(obj, key, value) + except Exception as e: + logger.warn('There was a problem when trying to update ' + key + ' for ' + obj.type + ': ' + e.message) def _perform_update(self, object_id, obj): with db.session.no_autoflush: From e34666e59c0e4c06eeb5395b7b8ad95df6600ad2 Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 25 Oct 2017 18:45:46 -0300 Subject: [PATCH 0420/1506] Fix _rev property in Vuln API Modifies Vuln Schema to have _rev property only when dumping to JSON. --- server/api/modules/vulns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index eff22b51bbc..b85f929ee47 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -60,7 +60,7 @@ def get_data(self, file_obj): class VulnerabilitySchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') - _rev = fields.String(default='') + _rev = fields.String(dump_only=True, default='') _attachments = fields.Method('get_attachments', 'set_attachments', default=[]) owned = fields.Boolean(dump_only=True, default=False) owner = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator') From 17a93237b7c74bb9abd2d571f29dee2e6308d2ca Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 25 Oct 2017 18:48:48 -0300 Subject: [PATCH 0421/1506] Add joinedload for host views --- server/api/modules/hosts.py | 8 ++++++-- test_cases/test_api_commands.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 2c80a81a603..0075b57cc00 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -7,7 +7,7 @@ from flask_classful import route from marshmallow import fields from filteralchemy import FilterSet, operators -from sqlalchemy.orm import undefer +from sqlalchemy.orm import undefer, joinedload from server.utils.database import get_or_create from server.utils.logger import get_logger @@ -110,7 +110,11 @@ def _get_base_query(self, workspace_name): """Get services_count in one query and not deferred, that doe one query per host""" original = super(HostsView, self)._get_base_query(workspace_name) - return original.options(undefer(Host.service_count)) + return original.options( + undefer(Host.service_count), + undefer(Host.vulnerability_count), + joinedload(Host.hostnames) + ) def _envelope_list(self, objects, pagination_metadata=None): hosts = [] diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index 6e6faf2ed24..16eff9a6143 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -19,7 +19,7 @@ class TestListCommandView(ReadOnlyAPITests): api_endpoint = 'commands' view_class = CommandView - def test_(self, test_client, second_workspace, session): + def test_backwards_compatibility_list(self, test_client, second_workspace, session): self.factory.create(workspace=second_workspace) session.commit() res = test_client.get(self.url()) From 436e5c466652f9fbf7a66d5d87123028a28f6d0f Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 25 Oct 2017 18:49:59 -0300 Subject: [PATCH 0422/1506] Fix bug for vulns with status different from "opened" --- server/api/modules/vulns.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index b85f929ee47..57948c93f24 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -173,6 +173,7 @@ def load_impact(self, value): def load_status(self, value): if value == 'opened': return 'open' + return value def load_type(self, value): if value == 'Vulnerability': From ecc6e7b4b30d86688391232eacb4bfa848769252 Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 25 Oct 2017 18:50:33 -0300 Subject: [PATCH 0423/1506] Add translation from DB to the web The vuln status in the DB is "open" while the web still uses "opened" (which is incorrect, but should be fixed in the future, after finishing the CouchDB migration). --- server/api/modules/vulns.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 57948c93f24..18dfa9870c0 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -154,6 +154,8 @@ def load_parent(self, obj): return obj def get_status(self, obj): + if obj.status == 'open': + return 'opened' return obj.status def get_impact(self, obj): From 20daad3ded18980a6c411233fe05247406cb8b4c Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 25 Oct 2017 18:52:50 -0300 Subject: [PATCH 0424/1506] Fix delete programming bug The vuln id was lost when attempting to update from the Web. --- server/www/scripts/vulns/providers/vuln.js | 1 + 1 file changed, 1 insertion(+) diff --git a/server/www/scripts/vulns/providers/vuln.js b/server/www/scripts/vulns/providers/vuln.js index 4e6ae1bd96e..8a797019260 100644 --- a/server/www/scripts/vulns/providers/vuln.js +++ b/server/www/scripts/vulns/providers/vuln.js @@ -71,6 +71,7 @@ angular.module('faradayApp') set: function(ws, data) { var self = this; + if(data._id !== undefined) self._id = data._id; if(data.metadata !== undefined) self.metadata = data.metadata; if(data.target !== undefined) self.target = data.target; if(data.hostnames !== undefined) self.hostnames = data.hostnames; From 2685271c450ede63391a7cc5c1bd5012c42ff346 Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 25 Oct 2017 18:57:50 -0300 Subject: [PATCH 0425/1506] Fix bug on vulnerability schema when updating The hostnames prop shouldn't be updated with the vuln. --- server/api/modules/vulns.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 18dfa9870c0..7a5671667d2 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -222,6 +222,18 @@ def set_parent(self, data): data[parent_field] = parent.id return data + @post_load + def post_load_hostnames(self, data): + """ + Since we don't allow a parent change on update, it makes no sense to have this data + Removing this method will result in error, since the relation in the database would have to be updated and that is not implemented + :param data: + :return: + """ + data.pop('hostnames', None) + return data + + class VulnerabilityWebSchema(VulnerabilitySchema): From 55a63892754bedcd43dd8520bcaa2ff7bd66bc9e Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 25 Oct 2017 19:00:13 -0300 Subject: [PATCH 0426/1506] Fix naming convention --- server/api/modules/vulns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 7a5671667d2..0002afb8528 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -188,7 +188,7 @@ def load_parent(self, value): return value @post_load - def set_impact(self, data): + def post_load_impact(self, data): impact = data.pop('impact', None) if impact: data['impact_accountability'] = impact['accountability'] @@ -198,7 +198,7 @@ def set_impact(self, data): return data @post_load - def set_parent(self, data): + def post_load_parent(self, data): # schema guarantees that parent_type exists. parent_class = None parent_type = data.pop('parent_type', None) From 693356e2dbf2a04bf66bf684746788778998fff4 Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 25 Oct 2017 19:06:40 -0300 Subject: [PATCH 0427/1506] Fix naming convention and explicit param names --- server/api/modules/vulns.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 0002afb8528..e2adc1a37c0 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -61,27 +61,27 @@ class VulnerabilitySchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') _rev = fields.String(dump_only=True, default='') - _attachments = fields.Method('get_attachments', 'set_attachments', default=[]) + _attachments = fields.Method(serialize='get_attachments', deserialize='load_attachments', default=[]) owned = fields.Boolean(dump_only=True, default=False) owner = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator') - impact = fields.Method('get_impact', deserialize='load_impact') + impact = fields.Method(serialize='get_impact', deserialize='load_impact') desc = fields.String(dump_only=True, attribute='description') policyviolations = fields.List(fields.String, attribute='policy_violations') refs = fields.List(fields.String(), attribute='references') - issuetracker = fields.Method('get_issuetracker') - parent = fields.Method('get_parent', deserialize='load_parent', required=True) - parent_type = fields.Method('get_parent_type', required=True) - tags = fields.Method('get_tags') + issuetracker = fields.Method(serialize='get_issuetracker') + parent = fields.Method(serialize='get_parent', deserialize='load_parent', required=True) + parent_type = fields.Method(serialize='get_parent_type', required=True) + tags = fields.Method(serialize='get_tags') easeofresolution = fields.String(dump_only=True, attribute='ease_of_resolution') hostnames = PrimaryKeyRelatedField('name', many=True) - metadata = fields.Method('get_metadata') + metadata = fields.Method(serialize='get_metadata') service = fields.Nested(ServiceSchema(only=[ '_id', 'ports', 'status', 'protocol', 'name', 'version', 'summary' ]), dump_only=True) host = fields.Integer(dump_only=True, attribute='host_id') - status = fields.Method('get_status', deserialize='load_status') # TODO: this breaks enum validation. - type = fields.Method('get_type', deserialize='load_type') + status = fields.Method(serialize='get_status', deserialize='load_status') # TODO: this breaks enum validation. + type = fields.Method(serialize='get_type', deserialize='load_type') obj_id = fields.String(dump_only=True, attribute='id') target = fields.String(default='') # TODO: review this attribute @@ -124,8 +124,8 @@ def get_attachments(self, obj): return res - def set_attachments(self, obj): - return obj + def load_attachments(self, value): + return value def get_hostnames(self, obj): # TODO: improve performance here From c1b56fff0aa1164f3ba430d2a78054431619307c Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 25 Oct 2017 19:11:59 -0300 Subject: [PATCH 0428/1506] Fix missing description field on vulns When creating or updating vulns the description was lost. --- server/api/modules/vulns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index e2adc1a37c0..dafdca93e22 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -65,7 +65,7 @@ class VulnerabilitySchema(AutoSchema): owned = fields.Boolean(dump_only=True, default=False) owner = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator') impact = fields.Method(serialize='get_impact', deserialize='load_impact') - desc = fields.String(dump_only=True, attribute='description') + desc = fields.String(attribute='description') policyviolations = fields.List(fields.String, attribute='policy_violations') refs = fields.List(fields.String(), attribute='references') @@ -89,7 +89,7 @@ class Meta: model = Vulnerability fields = ( '_id', 'status', - 'issuetracker', 'description', 'parent', 'parent_type', + 'issuetracker', 'parent', 'parent_type', 'tags', 'severity', '_rev', 'easeofresolution', 'owned', 'hostnames', 'owner', 'data', 'refs', From c5462bcd13ef2fa4a23ca7cf6ccfe5dd8681776e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 25 Oct 2017 19:30:41 -0300 Subject: [PATCH 0429/1506] Change host's service_count so it only counts open services To maintain compatibility with the web ui --- server/api/modules/hosts.py | 4 ++-- server/models.py | 8 ++++++-- test_cases/test_api_hosts.py | 23 ++++++++++++++++++++--- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 2c80a81a603..2914a75de45 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -39,7 +39,7 @@ class HostSchema(AutoSchema): os = fields.String(default='') owned = fields.Boolean(default=False) owner = PrimaryKeyRelatedField('username', attribute='creator', dump_only=True) - services = fields.Integer(attribute='service_count', dump_only=True) + services = fields.Integer(attribute='open_service_count', dump_only=True) vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) credentials = fields.Integer(attribute='vulnerability_count', dump_only=True) hostnames = PrimaryKeyRelatedField('name', many=True, @@ -110,7 +110,7 @@ def _get_base_query(self, workspace_name): """Get services_count in one query and not deferred, that doe one query per host""" original = super(HostsView, self)._get_base_query(workspace_name) - return original.options(undefer(Host.service_count)) + return original.options(undefer(Host.open_service_count)) def _envelope_list(self, objects, pagination_metadata=None): hosts = [] diff --git a/server/models.py b/server/models.py index 624f12fe22c..ee77ddb1245 100644 --- a/server/models.py +++ b/server/models.py @@ -90,7 +90,7 @@ def do_begin(conn): SCHEMA_VERSION = 'W.3.0.0' -def _make_generic_count_property(parent_table, children_table): +def _make_generic_count_property(parent_table, children_table, where=None): """Make a deferred by default column property that counts the amount of childrens of some parent object""" children_id_field = '{}.id'.format(children_table) @@ -100,6 +100,8 @@ def _make_generic_count_property(parent_table, children_table): select_from(table(children_table)). where(text('{} = {}'.format( children_rel_field, parent_id_field)))) + if where is not None: + query = query.where(where) return column_property(query, deferred=True) @@ -197,7 +199,9 @@ class Host(Metadata): foreign_keys=[workspace_id] ) - service_count = _make_generic_count_property('host', 'service') + open_service_count = _make_generic_count_property( + 'host', 'service', where=text("service.status = 'open'")) + total_service_count = _make_generic_count_property('host', 'service') vulnerability_count = _make_generic_count_property('host', 'vulnerability') credentials_count = _make_generic_count_property('host', 'credential') diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 0a57eea8f3f..74ead2004fa 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -37,7 +37,7 @@ def host_services(self, session, service_factory): ret = {} for (count, host) in zip(SERVICE_COUNT, self.hosts): ret[host] = service_factory.create_batch( - count, host=host, workspace=host.workspace) + count, host=host, workspace=host.workspace, status='open') session.commit() return ret @@ -199,14 +199,31 @@ def test_get_host_services(self, test_client, session, assert other_host.id not in ids_returned assert ids_expected == ids_returned - def test_retrieve_shows_service_count(self, test_client, host_services): + def test_retrieve_shows_service_count(self, test_client, host_services, + service_factory): for (host, services) in host_services.items(): + # Adding closed and filtered services shouldn't impact on the + # service count since it should only count opened services + service_factory.create_batch(3, status='closed', host=host, + workspace=host.workspace) + service_factory.create_batch(2, status='filtered', host=host, + workspace=host.workspace) res = test_client.get(self.url(host)) assert res.json['services'] == len(services) - def test_index_shows_service_count(self, test_client, host_services): + def test_index_shows_service_count(self, test_client, host_services, + service_factory): ids_map = {host.id: services for (host, services) in host_services.items()} + + # Adding closed and filtered services shouldn't impact on the + # service count since it should only count opened services + for host in host_services.keys(): + service_factory.create_batch(3, status='closed', host=host, + workspace=host.workspace) + service_factory.create_batch(2, status='filtered', host=host, + workspace=host.workspace) + res = test_client.get(self.url()) assert len(res.json['rows']) >= len(ids_map) # Some hosts can have no services for host in res.json['rows']: From 11d79bb20448ab0fd43c349727aed599f6c7403e Mon Sep 17 00:00:00 2001 From: Mike Zhong Date: Wed, 25 Oct 2017 19:37:43 -0300 Subject: [PATCH 0430/1506] mysterious typo appeared where text was replaced with lots of space ... --- plugins/repo/dig/plugin.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/plugins/repo/dig/plugin.py b/plugins/repo/dig/plugin.py index 7c6576ab4db..d2ac1ecace1 100644 --- a/plugins/repo/dig/plugin.py +++ b/plugins/repo/dig/plugin.py @@ -102,13 +102,7 @@ def parseOutputString(self, output): description="E-mail Server") text = "Priority: " + mx_priority - self. - - - - - - ( + self.createAndAddNoteToService( host_id=host_id, service_id=service_id, name="priority", From 1edc4997d21ee8a60c3c32dba472b44e6d64f88f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 26 Oct 2017 10:32:11 -0300 Subject: [PATCH 0431/1506] Add extra service statuses to the importer --- server/importer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index 42f68a26c5a..fe9e41da05c 100644 --- a/server/importer.py +++ b/server/importer.py @@ -368,12 +368,14 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation document['status'] = 'open' status_mapper = { 'open': 'open', + 'opened': 'open', 'up': 'open', 'closed': 'closed', 'down': 'closed', 'filtered': 'filtered', 'open|filtered': 'filtered', - 'unknown': 'closed' + 'unknown': 'closed', + '-': 'closed', } couchdb_status = document.get('status', 'open') if couchdb_status.lower() not in status_mapper: From 48f133ff58b0af66aa9a759951cac30800a314ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 26 Oct 2017 12:21:18 -0300 Subject: [PATCH 0432/1506] Improve MutableField and make vulns API compatible with marshmallow 3 --- server/api/modules/vulns.py | 9 ++++----- server/schemas.py | 15 +++++++++++++++ test_cases/test_marshmallow_fields.py | 22 ++++++++++++++++++++++ 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index dafdca93e22..5c2c56c1965 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -31,7 +31,7 @@ from server.utils.database import get_or_create from server.api.modules.services import ServiceSchema -from server.schemas import PrimaryKeyRelatedField +from server.schemas import MutableField, PrimaryKeyRelatedField vulns_api = Blueprint('vulns_api', __name__) logger = logging.getLogger(__name__) @@ -71,7 +71,9 @@ class VulnerabilitySchema(AutoSchema): refs = fields.List(fields.String(), attribute='references') issuetracker = fields.Method(serialize='get_issuetracker') parent = fields.Method(serialize='get_parent', deserialize='load_parent', required=True) - parent_type = fields.Method(serialize='get_parent_type', required=True) + parent_type = MutableField(fields.Method('get_parent_type'), + fields.String(), + required=True) tags = fields.Method(serialize='get_tags') easeofresolution = fields.String(dump_only=True, attribute='ease_of_resolution') hostnames = PrimaryKeyRelatedField('name', many=True) @@ -150,9 +152,6 @@ def get_parent(self, obj): def get_parent_type(self, obj): return obj.parent.__class__.__name__ - def load_parent(self, obj): - return obj - def get_status(self, obj): if obj.status == 'open': return 'opened' diff --git a/server/schemas.py b/server/schemas.py index a5c39a15be4..7ea21f74eb0 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -66,6 +66,15 @@ class MutableField(fields.Field): def __init__(self, read_field, write_field, **kwargs): self.read_field = read_field self.write_field = write_field + + # Set _CHECK_ATTRIBUTE based on the read field because it is used + # during serialization + self._CHECK_ATTRIBUTE = self.read_field._CHECK_ATTRIBUTE + + # Propagate required=True to the children fields + if kwargs.get('required'): + self.read_field.required = self.write_field.required = True + super(MutableField, self).__init__(**kwargs) def _serialize(self, value, attr, obj): @@ -73,3 +82,9 @@ def _serialize(self, value, attr, obj): def _deserialize(self, value, attr, data): return self.write_field._deserialize(value, attr, data) + + def _add_to_schema(self, field_name, schema): + # Propagate to child fields + super(MutableField, self)._add_to_schema(field_name, schema) + self.read_field._add_to_schema(field_name, schema) + self.write_field._add_to_schema(field_name, schema) diff --git a/test_cases/test_marshmallow_fields.py b/test_cases/test_marshmallow_fields.py index b14fb9da47f..269b89e375c 100644 --- a/test_cases/test_marshmallow_fields.py +++ b/test_cases/test_marshmallow_fields.py @@ -104,6 +104,7 @@ class Blogpost2Schema(Schema): user = MutableField(fields.Nested(UserSchema, only=('username',)), fields.String()) + class TestMutableField: serialized_data = {"id": 1, "title": "test", "user": {"username": "john"}} @@ -129,3 +130,24 @@ def test_deserialize(self): def test_deserialize_fails(self): with pytest.raises(ValidationError): self.load(self.serialized_data) + + def test_required_propagation(self): + read_field = fields.String() + write_field = fields.Float() + mutable = MutableField(read_field, write_field, required=True) + assert mutable.required + assert read_field.required + assert write_field.required + + def test_load_method_field(self): + class PlaceSchema(Schema): + name = fields.String() + x = MutableField(fields.Method('get_x'), + fields.String) + + def get_x(self, obj): + return 5 + assert self.serialize(Place('test', 1, 1), PlaceSchema) == { + "name": "test", + "x": 5, + } From 292a8ad7f80e124b2a64006f19d42802eaf016ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 26 Oct 2017 12:29:29 -0300 Subject: [PATCH 0433/1506] Make credentials API compatible with marshmallow 3 --- server/api/modules/credentials.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/server/api/modules/credentials.py b/server/api/modules/credentials.py index e1e5ba31de4..1f051586858 100644 --- a/server/api/modules/credentials.py +++ b/server/api/modules/credentials.py @@ -5,7 +5,7 @@ import flask from flask import Blueprint -from marshmallow import fields, post_load +from marshmallow import fields, post_load, ValidationError from filteralchemy import FilterSet, operators from server.api.base import ( @@ -14,6 +14,7 @@ FilterSetMeta, FilterAlchemyMixin, InvalidUsage) from server.models import Credential, Host, Service, db +from server.schemas import MutableField credentials_api = Blueprint('credentials_api', __name__) @@ -28,8 +29,12 @@ class CredentialSchema(AutoSchema): password = fields.String(default='') description = fields.String(default='') couchdbid = fields.String(default='') # backwards compatibility - parent_type = fields.Method('get_parent_type', required=True) - parent = fields.Method('get_parent', required=True) + parent_type = MutableField(fields.Method('get_parent_type'), + fields.String(), + required=True) + parent = MutableField(fields.Method('get_parent'), + fields.Integer(), + required=True) # for filtering host_id = fields.Integer(load_only=True) @@ -69,9 +74,12 @@ def set_parent(self, data): if parent_type == 'Host': parent_class = Host parent_field = 'host_id' - if parent_type == 'Service': + elif parent_type == 'Service': parent_class = Service parent_field = 'service_id' + else: + raise ValidationError( + 'Unknown parent type: {}'.format(parent_type)) parent = db.session.query(parent_class).filter_by(id=parent_id).first() if not parent: raise InvalidUsage('Parent not found.') From 9917c5e786d35cc9fe2f11e556217ab446617276 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 26 Oct 2017 12:46:09 -0300 Subject: [PATCH 0434/1506] Performance improvent on some views --- server/api/modules/services.py | 7 +++++++ server/api/modules/vulns.py | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/server/api/modules/services.py b/server/api/modules/services.py index 26add482150..00bae5031a3 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -6,6 +6,7 @@ import flask from flask import Blueprint from marshmallow import fields +from sqlalchemy.orm import joinedload from server.api.base import AutoSchema, ReadWriteWorkspacedView from server.models import Service @@ -72,4 +73,10 @@ def _envelope_list(self, objects, pagination_metadata=None): 'services': services, } + def _get_base_query(self, workspace_name): + original = super(ServiceView, self)._get_base_query(workspace_name) + return original.options( + joinedload(Service.credentials) + ) + ServiceView.register(services_api) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 5c2c56c1965..75b9f365b4f 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -10,6 +10,7 @@ from flask import request from flask import Blueprint from marshmallow import fields, post_load, ValidationError +from sqlalchemy.orm import joinedload from depot.manager import DepotManager from server.api.base import ( @@ -357,4 +358,11 @@ def _envelope_list(self, objects, pagination_metadata=None): 'vulnerabilities': vulns, } + def _get_base_query(self, workspace_name): + original = super(VulnerabilityView, self)._get_base_query(workspace_name) + return original.options( + joinedload(VulnerabilityGeneric.policy_violations), + joinedload(VulnerabilityGeneric.references) + ) + VulnerabilityView.register(vulns_api) From 416ad0e6723f53351fe2154c5ca6524f7858a8d9 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 26 Oct 2017 12:49:51 -0300 Subject: [PATCH 0435/1506] Add references to vuln templates --- server/api/modules/vulnerability_template.py | 8 +- server/models.py | 132 ++++++++++++------ test_cases/factories.py | 10 +- test_cases/test_api_vulnerability_template.py | 11 +- test_cases/test_model_events.py | 3 + 5 files changed, 120 insertions(+), 44 deletions(-) diff --git a/server/api/modules/vulnerability_template.py b/server/api/modules/vulnerability_template.py index 52852e90137..2555ff46dd7 100644 --- a/server/api/modules/vulnerability_template.py +++ b/server/api/modules/vulnerability_template.py @@ -34,12 +34,18 @@ class VulnerabilityTemplateSchema(AutoSchema): _rev = fields.String(default='', dump_only=True) cwe = fields.String(dump_only=True, default='') # deprecated field, the legacy data is added to refs on import exploitation = fields.String(attribute='severity', validate=OneOf(VulnerabilityTemplate.SEVERITIES), required=True) - references = PrimaryKeyRelatedField('name', many=True, attribute='references') + references = fields.Method('get_references', deserialize='load_references', required=True) class Meta: model = VulnerabilityTemplate fields = ('id', '_id', '_rev', 'cwe', 'description', 'exploitation', 'name', 'references', 'resolution') + def get_references(self, obj): + return ', '.join(map(lambda ref_tmpl: ref_tmpl.name, obj.reference_template_instances)) + + def load_references(self, value): + return value.split(',') + class VulnerabilityTemplateFilterSet(FilterSet): class Meta(FilterSetMeta): diff --git a/server/models.py b/server/models.py index ee77ddb1245..052cddc8357 100644 --- a/server/models.py +++ b/server/models.py @@ -333,14 +333,6 @@ def parent(self): raise NotImplementedError('ABC property called') -class VulnerabilityTemplate(VulnerabilityABC): - __tablename__ = 'vulnerability_template' - - __table_args__ = ( - UniqueConstraint('name', name='uix_vulnerability_template_name'), - ) - - class CustomAssociationSet(_AssociationSet): """ A custom associacion set that passes the creator method the both @@ -395,6 +387,59 @@ def creator(name, vulnerability): return creator +def _build_associationproxy_creator_non_workspaced(model_class_name): + def creator(name, vulnerability): + """Get or create a reference/policyviolation with the + corresponding name. This must be worspace aware""" + + # Ugly hack to avoid the fact that Reference is defined after + # Vulnerability + model_class = globals()[model_class_name] + child = model_class.query.filter( + getattr(model_class, 'name') == name, + ).first() + if child is None: + # Doesn't exist + child = model_class(name) + return child + + return creator + + +class VulnerabilityTemplate(VulnerabilityABC): + __tablename__ = 'vulnerability_template' + + __table_args__ = ( + UniqueConstraint('name', name='uix_vulnerability_template_name'), + ) + + # We use ReferenceTemplate and not Reference since Templates does not have workspace. + + reference_template_instances = relationship( + "ReferenceTemplate", + secondary="reference_template_vulnerability_association", + collection_class=set + ) + + references = association_proxy( + 'reference_template_instances', 'name', + proxy_factory=CustomAssociationSet, + creator=_build_associationproxy_creator_non_workspaced('ReferenceTemplate') + ) + + policy_violation_template_instances = relationship( + "PolicyViolationTemplate", + secondary="policy_violation_template_vulnerability_association", + collection_class=set + ) + + policy_violations = association_proxy( + 'policy_violation_template_instances', 'name', + proxy_factory=CustomAssociationSet, + creator=_build_associationproxy_creator_non_workspaced('PolicyViolationTemplate') + ) + + class VulnerabilityGeneric(VulnerabilityABC): STATUSES = [ 'open', @@ -547,21 +592,14 @@ class ReferenceTemplate(Metadata): id = Column(Integer, primary_key=True) name = Column(Text, nullable=False) - vulnerability_id = Column( - Integer, - ForeignKey(VulnerabilityTemplate.id), - index=True - ) - vulnerability = relationship( - 'VulnerabilityTemplate', - backref='references', - foreign_keys=[vulnerability_id], - ) - __table_args__ = ( - UniqueConstraint('name', 'vulnerability_id', name='uix_reference_template_name_vulnerability'), + UniqueConstraint('name', name='uix_reference_template_name'), ) + def __init__(self, name=None, **kwargs): + super(ReferenceTemplate, self).__init__(name=name, + **kwargs) + class Reference(Metadata): __tablename__ = 'reference' @@ -607,14 +645,38 @@ class ReferenceVulnerabilityAssociation(db.Model): class PolicyViolationVulnerabilityAssociation(db.Model): - __tablename__ = 'policy_violation_vulnerability_association' - vulnerability_id = Column(Integer, ForeignKey('vulnerability.id'), primary_key=True) - policy_violation_id = Column(Integer, ForeignKey('policy_violation.id'), primary_key=True) + __tablename__ = 'policy_violation_vulnerability_association' + + vulnerability_id = Column(Integer, ForeignKey('vulnerability.id'), primary_key=True) + policy_violation_id = Column(Integer, ForeignKey('policy_violation.id'), primary_key=True) + + policy_violation = relationship("PolicyViolation", backref="policy_violation_associations", foreign_keys=[policy_violation_id]) + vulnerability = relationship("Vulnerability", backref="policy_violationvulnerability_associations", + foreign_keys=[vulnerability_id]) + + +class ReferenceVulnerabilityAssociation(db.Model): + + __tablename__ = 'reference_template_vulnerability_association' + + vulnerability_id = Column(Integer, ForeignKey('vulnerability_template.id'), primary_key=True) + reference_id = Column(Integer, ForeignKey('reference_template.id'), primary_key=True) + + reference = relationship("ReferenceTemplate", backref="reference_template_associations", foreign_keys=[reference_id]) + vulnerability = relationship("VulnerabilityTemplate", backref="reference_template_vulnerability_associations", foreign_keys=[vulnerability_id]) + + +class PolicyViolationVulnerabilityAssociation(db.Model): + + __tablename__ = 'policy_violation_template_vulnerability_association' + + vulnerability_id = Column(Integer, ForeignKey('vulnerability_template.id'), primary_key=True) + policy_violation_id = Column(Integer, ForeignKey('policy_violation_template.id'), primary_key=True) - policy_violation = relationship("PolicyViolation", backref="policy_violation_associations", foreign_keys=[policy_violation_id]) - vulnerability = relationship("Vulnerability", backref="policy_violationvulnerability_associations", - foreign_keys=[vulnerability_id]) + policy_violation = relationship("PolicyViolationTemplate", backref="policy_violation_template_associations", foreign_keys=[policy_violation_id]) + vulnerability = relationship("VulnerabilityTemplate", backref="policy_violation_template_vulnerability_associations", + foreign_keys=[vulnerability_id]) class PolicyViolationTemplate(Metadata): @@ -622,24 +684,16 @@ class PolicyViolationTemplate(Metadata): id = Column(Integer, primary_key=True) name = Column(Text, nullable=False) - vulnerability_id = Column( - Integer, - ForeignKey(VulnerabilityTemplate.id), - index=True - ) - vulnerability = relationship( - 'VulnerabilityTemplate', - backref='policy_violations', - foreign_keys=[vulnerability_id] - ) - __table_args__ = ( UniqueConstraint( 'name', - 'vulnerability_id', - name='uix_policy_violation_template_name_vulnerability'), + name='uix_policy_violation_template_name'), ) + def __init__(self, name=None, **kwargs): + super(PolicyViolationTemplate, self).__init__(name=name, + **kwargs) + class PolicyViolation(Metadata): __tablename__ = 'policy_violation' diff --git a/test_cases/factories.py b/test_cases/factories.py index 4c29d639829..c6ab093b025 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -29,7 +29,7 @@ VulnerabilityTemplate, VulnerabilityWeb, Workspace, -) + ReferenceTemplate) # Make partials for start and end date. End date must be after start date FuzzyStartTime = lambda: ( @@ -139,6 +139,14 @@ class Meta: sqlalchemy_session = db.session +class ReferenceTemplateFactory(FaradayFactory): + name = FuzzyText() + + class Meta: + model = ReferenceTemplate + sqlalchemy_session = db.session + + class ServiceFactory(WorkspaceObjectFactory): name = FuzzyText() description = FuzzyText() diff --git a/test_cases/test_api_vulnerability_template.py b/test_cases/test_api_vulnerability_template.py index f2ad689907b..c7d24ee66f6 100644 --- a/test_cases/test_api_vulnerability_template.py +++ b/test_cases/test_api_vulnerability_template.py @@ -11,7 +11,7 @@ from server.models import ( VulnerabilityTemplate, ) -from test_cases.factories import VulnerabilityTemplateFactory +from test_cases.factories import VulnerabilityTemplateFactory, ReferenceTemplateFactory CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) @@ -77,10 +77,13 @@ def test_update_vulnerability_template(self, session, test_client): assert updated_template.severity == raw_data['exploitation'] assert updated_template.resolution == raw_data['resolution'] assert updated_template.description == raw_data['description'] - assert updated_template.references == [] + assert updated_template.references == set([]) def test_update_vulnerabiliy_template_change_refs(self, session, test_client): template = self.factory.create() + for ref_name in set(['old1', 'old2']): + ref = ReferenceTemplateFactory.create(name=ref_name) + self.first_object.reference_template_instances.add(ref) session.commit() raw_data = self._create_post_data_vulnerability_template(references=['new_ref', 'another_ref']) res = test_client.put('/v2/vulnerability_template/{0}/'.format(template.id), data=raw_data) @@ -90,7 +93,7 @@ def test_update_vulnerabiliy_template_change_refs(self, session, test_client): assert updated_template.severity == raw_data['exploitation'] assert updated_template.resolution == raw_data['resolution'] assert updated_template.description == raw_data['description'] - assert updated_template.references == [] + assert updated_template.references == set([u'another_ref', u'new_ref']) def test_create_new_vulnerability_template_with_references(self, session, test_client): vuln_count_previous = session.query(VulnerabilityTemplate).count() @@ -99,6 +102,8 @@ def test_create_new_vulnerability_template_with_references(self, session, test_c assert res.status_code == 201 assert res.json['_id'] == 6 assert vuln_count_previous + 1 == session.query(VulnerabilityTemplate).count() + new_template = session.query(VulnerabilityTemplate).filter_by(id=6).first() + assert new_template.references == set([u'ref1', u'ref2']) def test_delete_vuln_template(self, session, test_client): template = self.factory.create() diff --git a/test_cases/test_model_events.py b/test_cases/test_model_events.py index 9f1842378f9..baebce85a26 100644 --- a/test_cases/test_model_events.py +++ b/test_cases/test_model_events.py @@ -12,6 +12,9 @@ def test_child_parent_verification_event(session, workspace, second_workspace): return False def test_child_parent_verification_event(session, workspace): + """ + Asserts that no exception will be raised when workspace are the same. + """ host = HostFactory.build(workspace=workspace) ServiceFactory.build(host=host, workspace=workspace) session.commit() From bedf02258a864bd82578b24ab6ae34359e64b112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 26 Oct 2017 15:42:45 -0300 Subject: [PATCH 0436/1506] Don't use writable helper fields without that funcionality Raise an error when using a writable JSTimestampField, PrimaryKeyRelatedField or SelfNestedField. Fix the views to make them readonly --- server/api/modules/hosts.py | 7 +++++-- server/api/modules/vulnerability_template.py | 5 +++-- server/api/modules/vulns.py | 1 - server/schemas.py | 6 ++++++ test_cases/test_marshmallow_fields.py | 14 ++++++++++++++ 5 files changed, 28 insertions(+), 5 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index cd639dab46d..977cae59a52 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -43,7 +43,10 @@ class HostSchema(AutoSchema): vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) credentials = fields.Integer(attribute='vulnerability_count', dump_only=True) hostnames = PrimaryKeyRelatedField('name', many=True, - attribute="hostnames", default=[]) + attribute="hostnames", + # TODO migration: make it writable + dump_only=True, # Only for now + default=[]) def get_metadata(self, obj): return { @@ -53,7 +56,7 @@ def get_metadata(self, obj): "owner": None, "update_action": None, "update_controller_action": None, - "update_time":1504796508.21, + "update_time": 1504796508.21, "update_user": None } diff --git a/server/api/modules/vulnerability_template.py b/server/api/modules/vulnerability_template.py index 52852e90137..88f3287a638 100644 --- a/server/api/modules/vulnerability_template.py +++ b/server/api/modules/vulnerability_template.py @@ -34,7 +34,8 @@ class VulnerabilityTemplateSchema(AutoSchema): _rev = fields.String(default='', dump_only=True) cwe = fields.String(dump_only=True, default='') # deprecated field, the legacy data is added to refs on import exploitation = fields.String(attribute='severity', validate=OneOf(VulnerabilityTemplate.SEVERITIES), required=True) - references = PrimaryKeyRelatedField('name', many=True, attribute='references') + references = PrimaryKeyRelatedField('name', many=True, attribute='references', + dump_only=True) class Meta: model = VulnerabilityTemplate @@ -71,4 +72,4 @@ def _envelope_list(self, objects, pagination_metadata=None): } -VulnerabilityTemplateView.register(vulnerability_template_api) \ No newline at end of file +VulnerabilityTemplateView.register(vulnerability_template_api) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 5c2c56c1965..12f4c2afdbc 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -183,7 +183,6 @@ def load_type(self, value): return 'vulnerability_web' def load_parent(self, value): - self.parent_id = value return value @post_load diff --git a/server/schemas.py b/server/schemas.py index 7ea21f74eb0..0d64d3d8ca2 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -11,6 +11,9 @@ def _serialize(self, value, attr, obj): if value is not None: return int(time.mktime(value.timetuple()) * 1000) + def _deserialize(self, value, attr, data): + raise NotImplementedError("Only dump is implemented for now") + class PrimaryKeyRelatedField(fields.Field): def __init__(self, field_name='id', *args, **kwargs): @@ -29,6 +32,9 @@ def _serialize(self, value, attr, obj): return None return getattr(value, self.field_name) + def _deserialize(self, value, attr, data): + raise NotImplementedError("Only dump is implemented for now") + class SelfNestedField(fields.Field): """A field to make namespaced schemas. It allows to have diff --git a/test_cases/test_marshmallow_fields.py b/test_cases/test_marshmallow_fields.py index 269b89e375c..fbec4788a74 100644 --- a/test_cases/test_marshmallow_fields.py +++ b/test_cases/test_marshmallow_fields.py @@ -30,6 +30,11 @@ def test_field_serialization(self): dumped = schema.dump(point).data assert dumped == {"name": "home", "coords": {"x": 123.0, "y": 456.1}} + def test_deserialization_fails(self): + with pytest.raises(NotImplementedError): + PlaceSchema().load({"name": "home", + "coords": {"x": 123.0, "y": 456.1}}) + class TestJSTimestampField: def test_parses_current_datetime(self): @@ -42,6 +47,10 @@ def test_parses_current_datetime(self): def test_parses_null_datetime(self): assert JSTimestampField()._serialize(None, None, None) is None + def test_deserialization_fails(self): + with pytest.raises(NotImplementedError): + JSTimestampField()._deserialize(time.time() * 1000, None, None) + User = namedtuple('User', ['username', 'blogposts']) Blogpost = namedtuple('Blogpost', ['id', 'title']) @@ -94,6 +103,11 @@ def test_single_with_none_value(self): "first_name": "other" } + def test_deserialization_fails(self): + with pytest.raises(NotImplementedError): + UserSchema().load({"username": "test", + "blogposts": [1, 2, 3]}) + Blogpost2 = namedtuple('Blogpost', ['id', 'title', 'user']) From dc6562510080e86dda4c234e07edf464d870bf90 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 26 Oct 2017 15:50:38 -0300 Subject: [PATCH 0437/1506] Remove joinedload on vulns for now --- server/api/modules/vulns.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 75b9f365b4f..4ec2ab2ab47 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -10,7 +10,6 @@ from flask import request from flask import Blueprint from marshmallow import fields, post_load, ValidationError -from sqlalchemy.orm import joinedload from depot.manager import DepotManager from server.api.base import ( @@ -358,11 +357,5 @@ def _envelope_list(self, objects, pagination_metadata=None): 'vulnerabilities': vulns, } - def _get_base_query(self, workspace_name): - original = super(VulnerabilityView, self)._get_base_query(workspace_name) - return original.options( - joinedload(VulnerabilityGeneric.policy_violations), - joinedload(VulnerabilityGeneric.references) - ) VulnerabilityView.register(vulns_api) From cf12ac60e170dd48c740d4809bfaba83ae876e0e Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 26 Oct 2017 16:33:42 -0300 Subject: [PATCH 0438/1506] Fix hostnames issue in vukln API --- server/api/base.py | 5 +---- server/api/modules/vulns.py | 14 +------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index dcd07b636a3..e75b066fd48 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -329,10 +329,7 @@ def put(self, object_id, **kwargs): def _update_object(self, obj, data): for (key, value) in data.items(): - try: - setattr(obj, key, value) - except Exception as e: - logger.warn('There was a problem when trying to update ' + key + ' for ' + obj.type + ': ' + e.message) + setattr(obj, key, value) def _perform_update(self, object_id, obj): with db.session.no_autoflush: diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index dfc9e392e0b..84b6bee5110 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -76,7 +76,7 @@ class VulnerabilitySchema(AutoSchema): required=True) tags = fields.Method(serialize='get_tags') easeofresolution = fields.String(dump_only=True, attribute='ease_of_resolution') - hostnames = PrimaryKeyRelatedField('name', many=True) + hostnames = PrimaryKeyRelatedField('name', many=True, dump_only=True) metadata = fields.Method(serialize='get_metadata') service = fields.Nested(ServiceSchema(only=[ '_id', 'ports', 'status', 'protocol', 'name', 'version', 'summary' @@ -220,18 +220,6 @@ def post_load_parent(self, data): data[parent_field] = parent.id return data - @post_load - def post_load_hostnames(self, data): - """ - Since we don't allow a parent change on update, it makes no sense to have this data - Removing this method will result in error, since the relation in the database would have to be updated and that is not implemented - :param data: - :return: - """ - data.pop('hostnames', None) - return data - - class VulnerabilityWebSchema(VulnerabilitySchema): From 03e84fc5525d084da2cb860f9b0c1f67b4139eb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 26 Oct 2017 16:47:08 -0300 Subject: [PATCH 0439/1506] Implement writable SelfNestedField --- server/schemas.py | 5 ++++- test_cases/test_marshmallow_fields.py | 24 ++++++++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/server/schemas.py b/server/schemas.py index 0d64d3d8ca2..940756e0e4a 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -55,7 +55,10 @@ def _serialize(self, value, attr, obj): return ret def _deserialize(self, value, attr, data): - raise NotImplementedError("Only dump is implemented for now") + load = self.target_schema.load(value) + if load.errors: + raise ValidationError(load.errors) + return load.data class MutableField(fields.Field): diff --git a/test_cases/test_marshmallow_fields.py b/test_cases/test_marshmallow_fields.py index fbec4788a74..eb3aaf3fb9c 100644 --- a/test_cases/test_marshmallow_fields.py +++ b/test_cases/test_marshmallow_fields.py @@ -14,8 +14,8 @@ class PointSchema(Schema): - x = fields.Float() - y = fields.Float() + x = fields.Float(required=True) + y = fields.Float(required=True) class PlaceSchema(Schema): @@ -24,16 +24,28 @@ class PlaceSchema(Schema): class TestSelfNestedField: + + def load(self, data, schema=PlaceSchema): + return schema(strict=True).load(data).data + def test_field_serialization(self): point = Place('home', 123, 456.1) schema = PlaceSchema() dumped = schema.dump(point).data assert dumped == {"name": "home", "coords": {"x": 123.0, "y": 456.1}} - def test_deserialization_fails(self): - with pytest.raises(NotImplementedError): - PlaceSchema().load({"name": "home", - "coords": {"x": 123.0, "y": 456.1}}) + def test_deserialization_success(self): + load = PlaceSchema().load({"coords": {"x": 123.0, "y": 456.1}}).data + assert load == {"coords": {"x": 123.0, "y": 456.1}} + + @pytest.mark.parametrize('data', [ + {"coords": {"x": 1}}, + {"coords": {"x": None, "y": 2}}, + {"coords": {"x": "xxx", "y": 2}}, + ]) + def test_deserialization_fails(self, data): + with pytest.raises(ValidationError): + self.load(data) class TestJSTimestampField: From bc4df4258d0ad3bd7523507d95607e80d91cb60b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 26 Oct 2017 16:48:12 -0300 Subject: [PATCH 0440/1506] Improve loading impact in Vulnerability schema Use implementation of SelfNestedFields, handle missing keys in nested "impact" key --- server/api/modules/vulns.py | 34 +++++++++++++--------------- test_cases/test_api_vulnerability.py | 21 +++++++++++++++++ 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 12f4c2afdbc..35882bca91e 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -9,7 +9,7 @@ from filteralchemy import FilterSet, operators from flask import request from flask import Blueprint -from marshmallow import fields, post_load, ValidationError +from marshmallow import Schema, fields, post_load, ValidationError from depot.manager import DepotManager from server.api.base import ( @@ -31,7 +31,11 @@ from server.utils.database import get_or_create from server.api.modules.services import ServiceSchema -from server.schemas import MutableField, PrimaryKeyRelatedField +from server.schemas import ( + MutableField, + PrimaryKeyRelatedField, + SelfNestedField, +) vulns_api = Blueprint('vulns_api', __name__) logger = logging.getLogger(__name__) @@ -57,6 +61,13 @@ def get_data(self, file_obj): return b64encode(depot.get(file_obj.content.get('file_id')).read()) +class ImpactSchema(Schema): + accountability = fields.Boolean(attribute='impact_accountability') + availability = fields.Boolean(attribute='impact_availability') + confidentiality = fields.Boolean(attribute='impact_confidentiality') + integrity = fields.Boolean(attribute='impact_integrity') + + class VulnerabilitySchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') @@ -64,7 +75,7 @@ class VulnerabilitySchema(AutoSchema): _attachments = fields.Method(serialize='get_attachments', deserialize='load_attachments', default=[]) owned = fields.Boolean(dump_only=True, default=False) owner = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator') - impact = fields.Method(serialize='get_impact', deserialize='load_impact') + impact = SelfNestedField(ImpactSchema()) desc = fields.String(attribute='description') policyviolations = fields.List(fields.String, attribute='policy_violations') @@ -157,20 +168,9 @@ def get_status(self, obj): return 'opened' return obj.status - def get_impact(self, obj): - return { - 'accountability': obj.impact_accountability, - 'availability': obj.impact_availability, - 'confidentiality': obj.impact_confidentiality, - 'integrity': obj.impact_integrity, - } - def get_issuetracker(self, obj): return {} - def load_impact(self, value): - return value - def load_status(self, value): if value == 'opened': return 'open' @@ -187,12 +187,10 @@ def load_parent(self, value): @post_load def post_load_impact(self, data): + # Unflatten impact (move data[impact][*] to data[*]) impact = data.pop('impact', None) if impact: - data['impact_accountability'] = impact['accountability'] - data['impact_availability'] = impact['availability'] - data['impact_confidentiality'] = impact['confidentiality'] - data['impact_integrity'] = impact['integrity'] + data.update(impact) return data @post_load diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 6324288a2fa..a420a4c89c2 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -394,6 +394,27 @@ def test_create_vuln_imapct_verification(self, host_with_hostnames, test_client, u'confidentiality': True, u'integrity': True} + def test_handles_invalid_impact(self, host_with_hostnames, test_client, + session): + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='Vulnerability', + parent_id=host_with_hostnames.id, + parent_type='Host', + refs=[], + policyviolations=[], + impact={ + 'accountability': True, + 'integrity': 'aaaa', + 'invalid': None, + } + ) + ws_name = host_with_hostnames.workspace.name + vuln_count_previous = session.query(Vulnerability).count() + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) + assert res.status_code == 400 + def test_create_vuln_with_invalid_type(self, host_with_hostnames, test_client, session): session.commit() # flush host_with_hostnames raw_data = self._create_post_data_vulnerability( From f79e075e1375e5e896d352f5fca8ecf0afad0eac Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 26 Oct 2017 17:16:33 -0300 Subject: [PATCH 0441/1506] Fix description in vuln API Backward compat --- server/api/modules/vulns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 43988f2553e..f4a4b06e4ae 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -102,7 +102,7 @@ class Meta: model = Vulnerability fields = ( '_id', 'status', - 'issuetracker', 'parent', 'parent_type', + 'issuetracker', 'description', 'parent', 'parent_type', 'tags', 'severity', '_rev', 'easeofresolution', 'owned', 'hostnames', 'owner', 'data', 'refs', From 219e2077249d0458d546e1b459ae6cbe01bca977 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 26 Oct 2017 17:36:47 -0300 Subject: [PATCH 0442/1506] Fix importer after adding assoc proxy --- server/importer.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/importer.py b/server/importer.py index 2c93401507f..3ae434d6377 100644 --- a/server/importer.py +++ b/server/importer.py @@ -894,10 +894,9 @@ def run(self): if cwe_field not in references: references.append(cwe_field) for ref_doc in references: - get_or_create(session, - ReferenceTemplate, - vulnerability=vuln_template, - name=ref_doc) + vuln_template.references.add(ref_doc) + + def get_name(self, document): doc_name = document.get('name') From d8a50adbbea8db89b86c7b04f2862d647863961d Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 26 Oct 2017 17:37:02 -0300 Subject: [PATCH 0443/1506] Add vuln update test --- test_cases/test_api_vulnerability.py | 92 +++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 6324288a2fa..1724a147d30 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -123,6 +123,7 @@ def test_hostnames(self, host_with_hostnames, test_client, session): def _create_post_data_vulnerability(self, name, vuln_type, parent_id, parent_type, refs, policyviolations, + status='opened', attachments=None, impact=None): if not impact: impact = {'accountability': False, 'availability': False, 'confidentiality': False, @@ -146,7 +147,7 @@ def _create_post_data_vulnerability(self, name, vuln_type, parent_id, 'refs': refs, 'resolution': '', 'severity': 'high', - 'status': 'opened', + 'status': status, '_attachments': {}, 'description': '', 'protocol': '', @@ -193,6 +194,95 @@ def test_create_vuln(self, host_with_hostnames, test_client, session): assert res.json['parent'] == host_with_hostnames.id assert res.json['parent_type'] == 'Host' + def test_create_vuln_with_closed_status(self, host_with_hostnames, test_client, session): + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='Vulnerability', + parent_id=host_with_hostnames.id, + parent_type='Host', + status='closed', + refs=[], + policyviolations=[] + ) + ws_name = host_with_hostnames.workspace.name + vuln_count_previous = session.query(Vulnerability).count() + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) + assert res.status_code == 201 + assert vuln_count_previous + 1 == session.query(Vulnerability).count() + assert res.json['status'] == 'closed' + assert res.json['name'] == 'New vulns' + assert res.json['type'] == 'Vulnerability' + assert res.json['parent'] == host_with_hostnames.id + assert res.json['parent_type'] == 'Host' + + def _create_put_data(self, + name, desc, status, parent, parent_type, + impact=None, refs=None, policy_violations=None): + if not refs: + refs = [] + if not policy_violations: + policy_violations = [] + + if not impact: + impact = {"accountability": False, "availability": False, "confidentiality": False, "integrity": False} + + raw_data = { + "_id":"e1b45f5375facfb1435d37e182ebc22de5f77bb3.e05df1c85617fffb575d2ced2679e9a0ebda7c3e", + "metadata":{ + "update_time":1509045001.279, + "update_user":"", + "update_action":0, + "creator":"UI Web", + "create_time":1509045001.279, + "update_controller_action": + "UI Web New", + "owner":""}, + "obj_id":"e05df1c85617fffb575d2ced2679e9a0ebda7c3e", + "owner":"", + "parent": parent, + "type":"Vulnerability", + "ws":"cloud", + "confirmed": True, + "data":"", + "desc": desc, + "easeofresolution":"", + "impact": impact, + "name": name, + "owned": False, + "policyviolations":policy_violations, + "refs": refs, + "resolution":"", + "severity": "critical", + "status": status, + "_attachments":{}, + "description":"", + "parent_type": parent_type, + "protocol":"", + "version":""} + + return raw_data + + def test_update_vuln_from_open_to_close(self, test_client, session, host_with_hostnames): + vuln = self.factory.create(status='open', host=host_with_hostnames, service=None) + session.commit() + raw_data = self._create_put_data( + name='New name', + desc='New desc', + status='closed', + parent=vuln.host.id, + parent_type='Host', + refs=['ref1'], + policy_violations=['pv0'] + ) + vuln_count_previous = session.query(Vulnerability).count() + res = test_client.put(self.url(vuln), data=raw_data) + assert res.status_code == 200 + assert vuln_count_previous == session.query(Vulnerability).count() + assert res.json['status'] == 'closed' + assert res.json['name'] == 'New name' + assert res.json['desc'] == 'New desc' + def test_create_vuln_web(self, host_with_hostnames, test_client, session): service = ServiceFactory.create(host=host_with_hostnames, workspace=host_with_hostnames.workspace) session.commit() # flush host_with_hostnames From 790bcce4db7d5b6c5fb3705b776ddfa8b6951aba Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 26 Oct 2017 17:45:02 -0300 Subject: [PATCH 0444/1506] Add nplusone when testing is True --- requirements_server.txt | 2 ++ server/app.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/requirements_server.txt b/requirements_server.txt index 0ea4deb4317..9d9e7b5b8ce 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -20,3 +20,5 @@ webargs marshmallow-sqlalchemy filteralchemy filedepot==0.5.0 +nplusone==0.8.1 + diff --git a/server/app.py b/server/app.py index aa5fea334f4..e0320b14c92 100644 --- a/server/app.py +++ b/server/app.py @@ -19,6 +19,7 @@ Security, SQLAlchemyUserDatastore, ) +from nplusone.ext.flask_sqlalchemy import NPlusOne from depot.manager import DepotManager import server.config @@ -79,6 +80,10 @@ def create_app(db_connection_string=None, testing=None): }) if testing: app.config['TESTING'] = testing + app.config['NPLUSONE_LOGGER'] = logging.getLogger('faraday.nplusone') + app.config['NPLUSONE_LOG_LEVEL'] = logging.ERROR + app.config['NPLUSONE_RAISE'] = True + NPlusOne(app) try: app.config['SQLALCHEMY_DATABASE_URI'] = db_connection_string or server.config.database.connection_string.strip("'") except AttributeError: From dd66c41cacee551bbe599baa7bcafb2c7fde8684 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 26 Oct 2017 17:36:47 -0300 Subject: [PATCH 0445/1506] Fix importer after adding assoc proxy --- server/importer.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/importer.py b/server/importer.py index 2c93401507f..3ae434d6377 100644 --- a/server/importer.py +++ b/server/importer.py @@ -894,10 +894,9 @@ def run(self): if cwe_field not in references: references.append(cwe_field) for ref_doc in references: - get_or_create(session, - ReferenceTemplate, - vulnerability=vuln_template, - name=ref_doc) + vuln_template.references.add(ref_doc) + + def get_name(self, document): doc_name = document.get('name') From e3ebf72889800214d92b95cbda5522d27ae6d95a Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 26 Oct 2017 17:37:02 -0300 Subject: [PATCH 0446/1506] Add vuln update test --- test_cases/test_api_vulnerability.py | 92 +++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index a420a4c89c2..f0d7020ef86 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -123,6 +123,7 @@ def test_hostnames(self, host_with_hostnames, test_client, session): def _create_post_data_vulnerability(self, name, vuln_type, parent_id, parent_type, refs, policyviolations, + status='opened', attachments=None, impact=None): if not impact: impact = {'accountability': False, 'availability': False, 'confidentiality': False, @@ -146,7 +147,7 @@ def _create_post_data_vulnerability(self, name, vuln_type, parent_id, 'refs': refs, 'resolution': '', 'severity': 'high', - 'status': 'opened', + 'status': status, '_attachments': {}, 'description': '', 'protocol': '', @@ -193,6 +194,95 @@ def test_create_vuln(self, host_with_hostnames, test_client, session): assert res.json['parent'] == host_with_hostnames.id assert res.json['parent_type'] == 'Host' + def test_create_vuln_with_closed_status(self, host_with_hostnames, test_client, session): + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='Vulnerability', + parent_id=host_with_hostnames.id, + parent_type='Host', + status='closed', + refs=[], + policyviolations=[] + ) + ws_name = host_with_hostnames.workspace.name + vuln_count_previous = session.query(Vulnerability).count() + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) + assert res.status_code == 201 + assert vuln_count_previous + 1 == session.query(Vulnerability).count() + assert res.json['status'] == 'closed' + assert res.json['name'] == 'New vulns' + assert res.json['type'] == 'Vulnerability' + assert res.json['parent'] == host_with_hostnames.id + assert res.json['parent_type'] == 'Host' + + def _create_put_data(self, + name, desc, status, parent, parent_type, + impact=None, refs=None, policy_violations=None): + if not refs: + refs = [] + if not policy_violations: + policy_violations = [] + + if not impact: + impact = {"accountability": False, "availability": False, "confidentiality": False, "integrity": False} + + raw_data = { + "_id":"e1b45f5375facfb1435d37e182ebc22de5f77bb3.e05df1c85617fffb575d2ced2679e9a0ebda7c3e", + "metadata":{ + "update_time":1509045001.279, + "update_user":"", + "update_action":0, + "creator":"UI Web", + "create_time":1509045001.279, + "update_controller_action": + "UI Web New", + "owner":""}, + "obj_id":"e05df1c85617fffb575d2ced2679e9a0ebda7c3e", + "owner":"", + "parent": parent, + "type":"Vulnerability", + "ws":"cloud", + "confirmed": True, + "data":"", + "desc": desc, + "easeofresolution":"", + "impact": impact, + "name": name, + "owned": False, + "policyviolations":policy_violations, + "refs": refs, + "resolution":"", + "severity": "critical", + "status": status, + "_attachments":{}, + "description":"", + "parent_type": parent_type, + "protocol":"", + "version":""} + + return raw_data + + def test_update_vuln_from_open_to_close(self, test_client, session, host_with_hostnames): + vuln = self.factory.create(status='open', host=host_with_hostnames, service=None) + session.commit() + raw_data = self._create_put_data( + name='New name', + desc='New desc', + status='closed', + parent=vuln.host.id, + parent_type='Host', + refs=['ref1'], + policy_violations=['pv0'] + ) + vuln_count_previous = session.query(Vulnerability).count() + res = test_client.put(self.url(vuln), data=raw_data) + assert res.status_code == 200 + assert vuln_count_previous == session.query(Vulnerability).count() + assert res.json['status'] == 'closed' + assert res.json['name'] == 'New name' + assert res.json['desc'] == 'New desc' + def test_create_vuln_web(self, host_with_hostnames, test_client, session): service = ServiceFactory.create(host=host_with_hostnames, workspace=host_with_hostnames.workspace) session.commit() # flush host_with_hostnames From 46fccf34104586c962260bb6325534efc1324530 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 26 Oct 2017 18:02:00 -0300 Subject: [PATCH 0447/1506] Add random password generator for postgresql password --- server/commands/initdb.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index 639d1c0580e..cd437d1f9e2 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -1,6 +1,9 @@ +import string + import os import sys import getpass +from random import SystemRandom from tempfile import TemporaryFile from subprocess import Popen, PIPE @@ -88,6 +91,10 @@ def _check_psql_output(self, psql_log_output, process_status): if process_status is not 0: sys.exit(process_status) + def generate_random_pw(self, pwlen): + rng = SystemRandom() + return "".join([rng.choice(string.ascii_letters + string.digits + string.punctuation) for _ in xrange(pwlen)]) + def _configure_postgres(self, psql_log_file): """ This step will create the role on the database. @@ -95,13 +102,9 @@ def _configure_postgres(self, psql_log_file): """ username_default = 'faraday_db_admin' print('This script will {blue} create a new postgres user {white} and {blue} save faraday-server settings {white}(server.ini). '.format(blue=Fore.BLUE, white=Fore.WHITE)) - username = raw_input('Please enter the {blue} database user {white} (press enter to use "{0}"): '.format(username_default, blue=Fore.BLUE, white=Fore.WHITE)) or username_default + username = raw_input('Please enter the new {blue} database username {white} (press enter to use "{0}"): '.format(username_default, blue=Fore.BLUE, white=Fore.WHITE)) or username_default postgres_command = ['sudo', '-u', 'postgres'] - password = None - while not password: - password = getpass.getpass(prompt='Please enter the {blue} password for the postgreSQL username {white}: '.format(blue=Fore.BLUE, white=Fore.WHITE)) - if not password: - print('Please type a valid password') + password = self.generate_random_pw(25) command = postgres_command + ['psql', '-c', 'CREATE ROLE {0} WITH LOGIN PASSWORD \'{1}\';'.format(username, password)] p = Popen(command, stderr=psql_log_file, stdout=psql_log_file) p.wait() From fd1adf2de510447a3c0c6c6aee1ff615fafcb000 Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 26 Oct 2017 18:12:29 -0300 Subject: [PATCH 0448/1506] Update test case to verify description is saved --- test_cases/test_api_vulnerability.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index a420a4c89c2..e856a885239 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -123,7 +123,7 @@ def test_hostnames(self, host_with_hostnames, test_client, session): def _create_post_data_vulnerability(self, name, vuln_type, parent_id, parent_type, refs, policyviolations, - attachments=None, impact=None): + attachments=None, impact=None, description=None): if not impact: impact = {'accountability': False, 'availability': False, 'confidentiality': False, 'integrity': False} @@ -137,7 +137,7 @@ def _create_post_data_vulnerability(self, name, vuln_type, parent_id, 'ws': 'airbnb', 'confirmed': True, 'data': '', - 'desc': 'fsdfsd', + 'desc': description or 'fsdfsd', 'easeofresolution': '', 'impact': impact, 'name': name, @@ -181,7 +181,8 @@ def test_create_vuln(self, host_with_hostnames, test_client, session): parent_id=host_with_hostnames.id, parent_type='Host', refs=[], - policyviolations=[] + policyviolations=[], + description='helloworld', ) ws_name = host_with_hostnames.workspace.name vuln_count_previous = session.query(Vulnerability).count() @@ -192,6 +193,8 @@ def test_create_vuln(self, host_with_hostnames, test_client, session): assert res.json['type'] == 'Vulnerability' assert res.json['parent'] == host_with_hostnames.id assert res.json['parent_type'] == 'Host' + assert res.json['desc'] == 'helloworld' + assert res.json['description'] == 'helloworld' def test_create_vuln_web(self, host_with_hostnames, test_client, session): service = ServiceFactory.create(host=host_with_hostnames, workspace=host_with_hostnames.workspace) From 23d1a656fe5dc0c598149f6e7d84a466f84306fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 27 Oct 2017 16:03:49 -0300 Subject: [PATCH 0449/1506] Fix bug in references field of vuln templates API It didn't correctly handle values other than a comma-separated string nor empty strings A combination of bugs in the tests, the load_references method and marshmallow 2.13.6 made the test altough the implementation was incorrect. See https://github.com/marshmallow-code/marshmallow/issues/395 for more info on the marshmallow bug --- server/api/modules/vulnerability_template.py | 12 +++++-- test_cases/test_api_vulnerability_template.py | 31 ++++++++++++++++--- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/server/api/modules/vulnerability_template.py b/server/api/modules/vulnerability_template.py index a44875f5c08..4c066d82271 100644 --- a/server/api/modules/vulnerability_template.py +++ b/server/api/modules/vulnerability_template.py @@ -8,7 +8,7 @@ FilterSet, operators, ) -from marshmallow import fields +from marshmallow import fields, ValidationError from marshmallow.validate import ( OneOf, ) @@ -44,7 +44,15 @@ def get_references(self, obj): return ', '.join(map(lambda ref_tmpl: ref_tmpl.name, obj.reference_template_instances)) def load_references(self, value): - return value.split(',') + if not isinstance(value, (unicode, str)): + raise ValidationError('references must be a string') + if len(value) == 0: + # Required because "".split(",") == [""] + return [] + references = value.split(',') + if any(len(ref) == 0 for ref in references): + raise ValidationError('Empty name detected in reference') + return references class VulnerabilityTemplateFilterSet(FilterSet): diff --git a/test_cases/test_api_vulnerability_template.py b/test_cases/test_api_vulnerability_template.py index c7d24ee66f6..a8c6f6d76dc 100644 --- a/test_cases/test_api_vulnerability_template.py +++ b/test_cases/test_api_vulnerability_template.py @@ -60,16 +60,18 @@ def _create_post_data_vulnerability_template(self, references): def test_create_new_vulnerability_template(self, session, test_client): vuln_count_previous = session.query(VulnerabilityTemplate).count() - raw_data = self._create_post_data_vulnerability_template(references=[]) + raw_data = self._create_post_data_vulnerability_template(references='') res = test_client.post('/v2/vulnerability_template/', data=raw_data) assert res.status_code == 201 assert res.json['_id'] == 6 assert vuln_count_previous + 1 == session.query(VulnerabilityTemplate).count() + vuln_template = VulnerabilityTemplate.query.get(6) + assert vuln_template.references == set() def test_update_vulnerability_template(self, session, test_client): template = self.factory.create() session.commit() - raw_data = self._create_post_data_vulnerability_template(references=[]) + raw_data = self._create_post_data_vulnerability_template(references='') res = test_client.put('/v2/vulnerability_template/{0}/'.format(template.id), data=raw_data) assert res.status_code == 200 updated_template = session.query(VulnerabilityTemplate).filter_by(id=template.id).first() @@ -79,13 +81,32 @@ def test_update_vulnerability_template(self, session, test_client): assert updated_template.description == raw_data['description'] assert updated_template.references == set([]) + @pytest.mark.parametrize('references', [ + ',', + ',,', + 'a,', + ['a', 'b', 'c'], + ['a', 'b', ''], + [], + {"a": 1}, + {} + ]) + def test_400_on_invalid_reference(self, session, test_client, references): + template = self.factory.create() + session.commit() + raw_data = self._create_post_data_vulnerability_template( + references=references) + res = test_client.put('/v2/vulnerability_template/{0}/'.format( + template.id), data=raw_data) + assert res.status_code == 400 + def test_update_vulnerabiliy_template_change_refs(self, session, test_client): template = self.factory.create() for ref_name in set(['old1', 'old2']): ref = ReferenceTemplateFactory.create(name=ref_name) self.first_object.reference_template_instances.add(ref) session.commit() - raw_data = self._create_post_data_vulnerability_template(references=['new_ref', 'another_ref']) + raw_data = self._create_post_data_vulnerability_template(references='new_ref,another_ref') res = test_client.put('/v2/vulnerability_template/{0}/'.format(template.id), data=raw_data) assert res.status_code == 200 updated_template = session.query(VulnerabilityTemplate).filter_by(id=template.id).first() @@ -97,7 +118,7 @@ def test_update_vulnerabiliy_template_change_refs(self, session, test_client): def test_create_new_vulnerability_template_with_references(self, session, test_client): vuln_count_previous = session.query(VulnerabilityTemplate).count() - raw_data = self._create_post_data_vulnerability_template(references=['ref1', 'ref2']) + raw_data = self._create_post_data_vulnerability_template(references='ref1,ref2') res = test_client.post('/v2/vulnerability_template/', data=raw_data) assert res.status_code == 201 assert res.json['_id'] == 6 @@ -111,4 +132,4 @@ def test_delete_vuln_template(self, session, test_client): res = test_client.delete('/v2/vulnerability_template/{0}/'.format(template.id)) assert res.status_code == 204 - assert vuln_count_previous - 1 == session.query(VulnerabilityTemplate).count() \ No newline at end of file + assert vuln_count_previous - 1 == session.query(VulnerabilityTemplate).count() From 4ad964c071f1aa2a9589de43b7b646923b6858eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 27 Oct 2017 16:16:45 -0300 Subject: [PATCH 0450/1506] Fix duplicate class name bug This was causing SQLAlchemy warnings but didn't break any code --- server/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/models.py b/server/models.py index 052cddc8357..d98dcc0995f 100644 --- a/server/models.py +++ b/server/models.py @@ -656,7 +656,7 @@ class PolicyViolationVulnerabilityAssociation(db.Model): foreign_keys=[vulnerability_id]) -class ReferenceVulnerabilityAssociation(db.Model): +class ReferenceTemplateVulnerabilityAssociation(db.Model): __tablename__ = 'reference_template_vulnerability_association' @@ -667,7 +667,7 @@ class ReferenceVulnerabilityAssociation(db.Model): vulnerability = relationship("VulnerabilityTemplate", backref="reference_template_vulnerability_associations", foreign_keys=[vulnerability_id]) -class PolicyViolationVulnerabilityAssociation(db.Model): +class PolicyViolationTemplateVulnerabilityAssociation(db.Model): __tablename__ = 'policy_violation_template_vulnerability_association' From 3114479efae655a8ef88e8380812dbc715e46cda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 27 Oct 2017 16:48:04 -0300 Subject: [PATCH 0451/1506] Delete programming Delete everything related to the EntityMetadata model (now we are inheriting from Metadata instead) and remove dao imports and old views --- server/api/modules/commandsrun.py | 1 - server/api/modules/hosts.py | 1 - server/api/modules/services.py | 1 - server/api/modules/vuln_csv.py | 29 ------------ server/api/modules/workspaces.py | 4 -- server/app.py | 2 - server/importer.py | 5 +- server/models.py | 77 ------------------------------- test_cases/factories.py | 9 ---- 9 files changed, 3 insertions(+), 126 deletions(-) delete mode 100644 server/api/modules/vuln_csv.py diff --git a/server/api/modules/commandsrun.py b/server/api/modules/commandsrun.py index ac66836d8da..f8962d9ecde 100644 --- a/server/api/modules/commandsrun.py +++ b/server/api/modules/commandsrun.py @@ -14,7 +14,6 @@ validate_workspace, filter_request_args, get_integer_parameter ) -from server.dao.command import CommandDAO from server.models import Command commandsrun_api = Blueprint('commandsrun_api', __name__) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 977cae59a52..89a3c271b6d 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -13,7 +13,6 @@ from server.utils.logger import get_logger from server.utils.web import gzipped, validate_workspace,\ get_integer_parameter, filter_request_args -from server.dao.host import HostDAO from server.api.base import ( ReadWriteWorkspacedView, PaginatedMixin, diff --git a/server/api/modules/services.py b/server/api/modules/services.py index 00bae5031a3..2a88ddf5419 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -11,7 +11,6 @@ from server.api.base import AutoSchema, ReadWriteWorkspacedView from server.models import Service from server.utils.logger import get_logger -from server.dao.service import ServiceDAO from server.utils.web import gzipped, validate_workspace, get_integer_parameter diff --git a/server/api/modules/vuln_csv.py b/server/api/modules/vuln_csv.py deleted file mode 100644 index 7ca0b3e0a77..00000000000 --- a/server/api/modules/vuln_csv.py +++ /dev/null @@ -1,29 +0,0 @@ -from flask import jsonify -from flask import Blueprint - -from server.utils.logger import get_logger -from server.utils.web import ( - gzipped, - validate_workspace, - filter_request_args, -) -from server.dao.credential import CredentialDAO - -vuln_csv_api = Blueprint('vuln_csv_api', __name__) - - -@gzipped -@vuln_csv_api.route('/ws/vulns/create_csv', methods=['GET']) -def create_csv_from_vulns(workspace=None): - - validate_workspace(workspace) - - get_logger(__name__).debug("Request parameters: {!r}"\ - .format(flask.request.args)) - - cred_filter = filter_request_args() - - dao = CredentialDAO(workspace) - result = dao.list(cred_filter=cred_filter) - - return jsonify(result) diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index 19cb9b7837b..3144dbbe34b 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -9,10 +9,6 @@ from sqlalchemy.orm import undefer from server.models import db, Workspace -from server.dao.host import HostDAO -from server.dao.vuln import VulnerabilityDAO -from server.dao.service import ServiceDAO -from server.dao.workspace import WorkspaceDAO from server.utils.logger import get_logger from server.schemas import ( JSTimestampField, diff --git a/server/app.py b/server/app.py index aa5fea334f4..c20d8020444 100644 --- a/server/app.py +++ b/server/app.py @@ -114,7 +114,6 @@ def create_app(db_connection_string=None, testing=None): from server.api.modules.services import services_api from server.api.modules.session import session_api from server.api.modules.vulns import vulns_api - from server.api.modules.vuln_csv import vuln_csv_api from server.api.modules.vulnerability_template import vulnerability_template_api from server.api.modules.workspaces import workspace_api app.register_blueprint(commandsrun_api) @@ -126,7 +125,6 @@ def create_app(db_connection_string=None, testing=None): app.register_blueprint(services_api) app.register_blueprint(session_api) app.register_blueprint(vulns_api) - app.register_blueprint(vuln_csv_api) app.register_blueprint(vulnerability_template_api) app.register_blueprint(workspace_api) diff --git a/server/importer.py b/server/importer.py index 2c93401507f..7cccc44c442 100644 --- a/server/importer.py +++ b/server/importer.py @@ -35,7 +35,6 @@ Comment, CommentObject, Credential, - EntityMetadata, ExecutiveReport, Host, Hostname, @@ -189,7 +188,9 @@ def __init__(self, entity_id): class EntityMetadataImporter(object): def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): - entity, created = get_or_create(session, EntityMetadata, couchdb_id=document.get('_id')) + # entity, created = get_or_create(session, EntityMetadata, couchdb_id=document.get('_id')) + # TODO: use inline metadata, not additional class + return metadata = document.get('metadata', dict()) entity.update_time = metadata.get('update_time', None) entity.update_user = metadata.get('update_user', None) diff --git a/server/models.py b/server/models.py index d98dcc0995f..11784568e67 100644 --- a/server/models.py +++ b/server/models.py @@ -128,27 +128,6 @@ def creator(cls): update_date = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) -class EntityMetadata(db.Model): - __tablename__ = 'metadata' - __table_args__ = ( - UniqueConstraint('couchdb_id'), - ) - - id = Column(Integer, primary_key=True) - update_time = Column(Float, nullable=True) - update_user = Column(String(250), nullable=True) - update_action = Column(Integer, nullable=True) - create_time = Column(Float, nullable=True) - update_controller_action = Column(String(250), nullable=True) - creator = Column(String(250), nullable=True) - owner = Column(String(250), nullable=True) - command_id = Column(String(250), nullable=True) - - couchdb_id = Column(String(250)) - revision = Column(String(250)) - document_type = Column(String(250)) - - class SourceCode(Metadata): __tablename__ = 'source_code' id = Column(Integer, primary_key=True) @@ -183,15 +162,6 @@ class Host(Metadata): mac = Column(Text, nullable=True) net_segment = Column(Text, nullable=True) - entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) - entity_metadata = relationship( - EntityMetadata, - uselist=False, - cascade="all, delete-orphan", - single_parent=True, - foreign_keys=[entity_metadata_id] - ) - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship( 'Workspace', @@ -258,15 +228,6 @@ class Service(Metadata): banner = Column(Text, nullable=True) - entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) - entity_metadata = relationship( - EntityMetadata, - uselist=False, - cascade="all, delete-orphan", - single_parent=True, - foreign_keys=[entity_metadata_id] - ) - host_id = Column(Integer, ForeignKey('host.id'), index=True, nullable=False) host = relationship('Host', backref='services', foreign_keys=[host_id]) @@ -738,15 +699,6 @@ class Credential(Metadata): description = Column(Text(), nullable=True) name = Column(String(250), nullable=True) - entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) - entity_metadata = relationship( - EntityMetadata, - uselist=False, - cascade="all, delete-orphan", - single_parent=True, - foreign_keys=[entity_metadata_id], - ) - host_id = Column(Integer, ForeignKey(Host.id), index=True, nullable=True) host = relationship('Host', backref='credentials', foreign_keys=[host_id]) @@ -797,19 +749,6 @@ class Command(Metadata): workspace = relationship('Workspace', foreign_keys=[workspace_id]) # TODO: add Tool relationship and report_attachment - entity_metadata_id = Column( - Integer, - ForeignKey(EntityMetadata.id), - index=True - ) - entity_metadata = relationship( - EntityMetadata, - uselist=False, - cascade="all, delete-orphan", - single_parent=True, - foreign_keys=[entity_metadata_id] - ) - @property def parent(self): return @@ -970,19 +909,6 @@ class Methodology(Metadata): id = Column(Integer, primary_key=True) name = Column(Text, nullable=False) - entity_metadata_id = Column( - Integer, - ForeignKey(EntityMetadata.id), - index=True - ) - entity_metadata = relationship( - EntityMetadata, - uselist=False, - cascade="all, delete-orphan", - single_parent=True, - foreign_keys=[entity_metadata_id] - ) - template = relationship('MethodologyTemplate', backref='methodologies') template_id = Column( Integer, @@ -1042,9 +968,6 @@ class Task(TaskABC): 'concrete': True } - entity_metadata = relationship(EntityMetadata, uselist=False, cascade="all, delete-orphan", single_parent=True) - entity_metadata_id = Column(Integer, ForeignKey(EntityMetadata.id), index=True) - assigned_to_id = Column(Integer, ForeignKey('user.id'), nullable=True) assigned_to = relationship('User', backref='assigned_tasks', foreign_keys=[assigned_to_id]) diff --git a/test_cases/factories.py b/test_cases/factories.py index c6ab093b025..b4c4d98fe53 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -15,7 +15,6 @@ db, Command, Credential, - EntityMetadata, Host, Hostname, License, @@ -115,14 +114,6 @@ class Meta: sqlalchemy_session = db.session -class EntityMetadataFactory(WorkspaceObjectFactory): - couchdb_id = factory.Sequence(lambda n: '{0}.1.2'.format(n)) - - class Meta: - model = EntityMetadata - sqlalchemy_session = db.session - - class PolicyViolationFactory(WorkspaceObjectFactory): name = FuzzyText() From e843b90452c68586e77e6a1eb75e44f8bc0021ac Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 27 Oct 2017 18:55:04 -0300 Subject: [PATCH 0452/1506] Add creator from g.user --- server/api/base.py | 8 +++----- server/app.py | 9 ++++++++- test_cases/test_api_vulnerability.py | 2 ++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index e75b066fd48..53e45c5bf7e 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -1,13 +1,11 @@ import json import flask -from flask import abort -from sqlalchemy import inspect +from flask import abort, g from flask_classful import FlaskView from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.inspection import inspect from sqlalchemy import func -from werkzeug.routing import parse_rule from marshmallow import Schema from marshmallow.compat import with_metaclass from marshmallow_sqlalchemy import ModelConverter @@ -281,8 +279,9 @@ class CreateMixin(object): def post(self, **kwargs): data = self._parse_data(self._get_schema_class()(strict=True), flask.request) - created = self._perform_create(data, **kwargs) + created.creator = g.user + db.session.commit() return self._dump(created), 201 def _perform_create(self, data, **kwargs): @@ -293,7 +292,6 @@ def _perform_create(self, data, **kwargs): # outside a no_autoflush block would result in a premature create. self._validate_uniqueness(obj) db.session.add(obj) - db.session.commit() return obj diff --git a/server/app.py b/server/app.py index c20d8020444..0f8a4f6e888 100644 --- a/server/app.py +++ b/server/app.py @@ -5,6 +5,8 @@ import os from os.path import join, expanduser +from server.models import User + try: # py2.7 from configparser import ConfigParser, NoSectionError, NoOptionError @@ -13,7 +15,7 @@ from ConfigParser import ConfigParser, NoSectionError, NoOptionError import flask -from flask import Flask +from flask import Flask, session, g from flask.json import JSONEncoder from flask_security import ( Security, @@ -141,6 +143,11 @@ def default_login_required(): if (not logged_in and not getattr(view, 'is_public', False)): flask.abort(403) + g.user = None + if logged_in: + user = User.query.filter_by(id=session["user_id"]).first() + g.user = user + return app diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 7dc81923c09..d9d37ddf6c3 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -266,6 +266,7 @@ def _create_put_data(self, return raw_data + @pytest.mark.skip(reason="Update not yet implemented") def test_update_vuln_from_open_to_close(self, test_client, session, host_with_hostnames): vuln = self.factory.create(status='open', host=host_with_hostnames, service=None, workspace=host_with_hostnames.workspace) session.commit() @@ -305,6 +306,7 @@ def test_create_vuln_web(self, host_with_hostnames, test_client, session): assert vuln_web_count_previous + 1 == session.query(VulnerabilityWeb).count() assert vuln_count_previous == session.query(Vulnerability).count() assert res.json['name'] == 'New vulns' + assert res.json['owner'] == 'test' assert res.json['type'] == 'VulnerabilityWeb' assert res.json['parent'] == service.id assert res.json['parent_type'] == 'Service' From 293b1b2fafda03ef57d04cdd507dc505ad696dc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 27 Oct 2017 19:06:35 -0300 Subject: [PATCH 0453/1506] Allow to filter confirmed vulns in workspace summary API Maybe this implementation is a bit overthought --- server/api/modules/workspaces.py | 24 ++--- server/importer.py | 2 +- server/models.py | 62 +++++++++++-- test_cases/models/test_workspace.py | 91 ++++++++++++++++--- test_cases/test_api_non_workspaced_various.py | 31 +++++++ 5 files changed, 172 insertions(+), 38 deletions(-) diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index 3144dbbe34b..ac4fcb57845 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -37,13 +37,13 @@ class WorkspaceSummarySchema(Schema): credentials = fields.Integer(dump_only=True, attribute='credential_count') hosts = fields.Integer(dump_only=True, attribute='host_count') services = fields.Integer(dump_only=True, attribute='service_count') - web_vulns = fields.Integer(dump_only=True, + web_vulns = fields.Integer(dump_only=True, allow_none=False, attribute='vulnerability_web_count') - code_vulns = fields.Integer(dump_only=True, + code_vulns = fields.Integer(dump_only=True, allow_none=False, attribute='vulnerability_code_count') - std_vulns = fields.Integer(dump_only=True, + std_vulns = fields.Integer(dump_only=True, allow_none=False, attribute='vulnerability_standard_count') - total_vulns = fields.Integer(dump_only=True, + total_vulns = fields.Integer(dump_only=True, allow_none=False, attribute='vulnerability_total_count') @@ -72,15 +72,11 @@ class WorkspaceView(ReadWriteView): schema_class = WorkspaceSchema def _get_base_query(self): - original = super(WorkspaceView, self)._get_base_query() - return original.options( - undefer(Workspace.host_count), - undefer(Workspace.credential_count), - undefer(Workspace.service_count), - undefer(Workspace.vulnerability_web_count), - undefer(Workspace.vulnerability_code_count), - undefer(Workspace.vulnerability_standard_count), - undefer(Workspace.vulnerability_total_count), - ) + try: + only_confirmed = bool(json.loads(flask.request.args['confirmed'])) + except (KeyError, ValueError): + only_confirmed = False + return Workspace.query_with_count(only_confirmed) + WorkspaceView.register(workspace_api) diff --git a/server/importer.py b/server/importer.py index 7cccc44c442..4e1511911f5 100644 --- a/server/importer.py +++ b/server/importer.py @@ -189,7 +189,7 @@ class EntityMetadataImporter(object): def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): # entity, created = get_or_create(session, EntityMetadata, couchdb_id=document.get('_id')) - # TODO: use inline metadata, not additional class + # TODO migration: use inline metadata, not additional class return metadata = document.get('metadata', dict()) entity.update_time = metadata.get('update_time', None) diff --git a/server/models.py b/server/models.py index 11784568e67..ab8b0ac731f 100644 --- a/server/models.py +++ b/server/models.py @@ -18,10 +18,10 @@ UniqueConstraint, event ) -from sqlalchemy.orm import relationship, backref +from sqlalchemy.orm import backref, relationship, undefer from sqlalchemy.sql import select, text, table from sqlalchemy import func -from sqlalchemy.orm import column_property +from sqlalchemy.orm import column_property, query_expression, with_expression from sqlalchemy.schema import DDL from sqlalchemy.ext.associationproxy import association_proxy, _AssociationSet from sqlalchemy.ext.declarative import declared_attr @@ -754,7 +754,8 @@ def parent(self): return -def _make_vuln_count_property(type_=None): +def _make_vuln_count_property(type_=None, only_confirmed=False, + use_column_property=True): query = (select([func.count(text('vulnerability.id'))]). select_from(table('vulnerability')). where(text('vulnerability.workspace_id = workspace.id')) @@ -764,7 +765,12 @@ def _make_vuln_count_property(type_=None): # This can cause SQL injection vulnerabilities # In this case type_ is supplied from a whitelist so this is safe query = query.where(text("vulnerability.type = '%s'" % type_)) - return column_property(query, deferred=True) + if only_confirmed: + query = query.where(text("vulnerability.confirmed = 1")) + if use_column_property: + return column_property(query, deferred=True) + else: + return query class Workspace(Metadata): @@ -781,10 +787,50 @@ class Workspace(Metadata): credential_count = _make_generic_count_property('workspace', 'credential') host_count = _make_generic_count_property('workspace', 'host') service_count = _make_generic_count_property('workspace', 'service') - vulnerability_web_count = _make_vuln_count_property('vulnerability_web') - vulnerability_code_count = _make_vuln_count_property('vulnerability_code') - vulnerability_standard_count = _make_vuln_count_property('vulnerability') - vulnerability_total_count = _make_vuln_count_property() + + vulnerability_web_count = query_expression() + vulnerability_code_count = query_expression() + vulnerability_standard_count = query_expression() + vulnerability_total_count = query_expression() + + @classmethod + def query_with_count(cls, only_confirmed): + """ + Add count fields to the query. + + If only_confirmed is True, it will only show the count for confirmed + vulnerabilities. Otherwise, it will show the count of all of them + """ + from sqlalchemy.sql.expression import literal_column + return cls.query.options( + undefer(cls.host_count), + undefer(cls.credential_count), + undefer(cls.service_count), + with_expression( + cls.vulnerability_web_count, + _make_vuln_count_property('vulnerability_web', + only_confirmed=only_confirmed, + use_column_property=False) + ), + with_expression( + cls.vulnerability_code_count, + _make_vuln_count_property('vulnerability_code', + only_confirmed=only_confirmed, + use_column_property=False) + ), + with_expression( + cls.vulnerability_standard_count, + _make_vuln_count_property('vulnerability', + only_confirmed=only_confirmed, + use_column_property=False) + ), + with_expression( + cls.vulnerability_total_count, + _make_vuln_count_property(type_=None, + only_confirmed=only_confirmed, + use_column_property=False) + ), + ) class Scope(Metadata): diff --git a/test_cases/models/test_workspace.py b/test_cases/models/test_workspace.py index c4acbd62683..390bfa506f3 100644 --- a/test_cases/models/test_workspace.py +++ b/test_cases/models/test_workspace.py @@ -1,5 +1,4 @@ -import pytest -from server.models import db +from server.models import db, Workspace from test_cases.factories import ( HostFactory, ServiceFactory, @@ -9,9 +8,17 @@ VulnerabilityWebFactory, ) -SOURCE_CODE_VULN_COUNT = 3 -STANDARD_VULN_COUNT = [6, 2] # With host parent and with service parent -WEB_VULN_COUNT = 5 +C_SOURCE_CODE_VULN_COUNT = 3 +C_STANDARD_VULN_COUNT = [6, 2] # With host parent and with service parent +C_WEB_VULN_COUNT = 5 + +NC_SOURCE_CODE_VULN_COUNT = 1 +NC_STANDARD_VULN_COUNT = [1, 2] # With host parent and with service parent +NC_WEB_VULN_COUNT = 2 + +SOURCE_CODE_VULN_COUNT = C_SOURCE_CODE_VULN_COUNT + NC_SOURCE_CODE_VULN_COUNT +STANDARD_VULN_COUNT = C_STANDARD_VULN_COUNT + NC_STANDARD_VULN_COUNT +WEB_VULN_COUNT = C_WEB_VULN_COUNT + NC_WEB_VULN_COUNT def populate_workspace(workspace): @@ -19,19 +26,44 @@ def populate_workspace(workspace): service = ServiceFactory.create(workspace=workspace, host=host) code = SourceCodeFactory.create(workspace=workspace) + # Create non confirmed vulnerabilities + + # Create standard vulns + VulnerabilityFactory.create_batch( + NC_STANDARD_VULN_COUNT[0], workspace=workspace, host=host, + service=None, confirmed=False) + VulnerabilityFactory.create_batch( + NC_STANDARD_VULN_COUNT[1], workspace=workspace, service=service, + host=None, confirmed=False) + + # Create web vulns + VulnerabilityWebFactory.create_batch( + NC_WEB_VULN_COUNT, workspace=workspace, service=service, + confirmed=False) + + # Create source code vulns + VulnerabilityCodeFactory.create_batch( + NC_SOURCE_CODE_VULN_COUNT, workspace=workspace, source_code=code, + confirmed=False) + + # Create confirmed vulnerabilities + # Create standard vulns - host_vulns = VulnerabilityFactory.create_batch( - STANDARD_VULN_COUNT[0], workspace=workspace, host=host, service=None) - service_vulns = VulnerabilityFactory.create_batch( - STANDARD_VULN_COUNT[1], workspace=workspace, service=service, host=None) + VulnerabilityFactory.create_batch( + C_STANDARD_VULN_COUNT[0], workspace=workspace, host=host, service=None, + confirmed=True) + VulnerabilityFactory.create_batch( + C_STANDARD_VULN_COUNT[1], workspace=workspace, service=service, + host=None, confirmed=True) # Create web vulns - web_vulns = VulnerabilityWebFactory.create_batch( - WEB_VULN_COUNT, workspace=workspace, service=service) + VulnerabilityWebFactory.create_batch( + C_WEB_VULN_COUNT, workspace=workspace, service=service, confirmed=True) # Create source code vulns - code_vulns = VulnerabilityCodeFactory.create_batch( - SOURCE_CODE_VULN_COUNT, workspace=workspace, source_code=code) + VulnerabilityCodeFactory.create_batch( + C_SOURCE_CODE_VULN_COUNT, workspace=workspace, source_code=code, + confirmed=True) db.session.commit() @@ -39,9 +71,38 @@ def populate_workspace(workspace): def test_vuln_count(workspace, second_workspace): populate_workspace(workspace) populate_workspace(second_workspace) + workspace = Workspace.query_with_count(False).filter( + Workspace.id == workspace.id).first() assert workspace.vulnerability_web_count == WEB_VULN_COUNT assert workspace.vulnerability_code_count == SOURCE_CODE_VULN_COUNT - assert workspace.vulnerability_standard_count == sum(STANDARD_VULN_COUNT) + assert workspace.vulnerability_standard_count == sum( + STANDARD_VULN_COUNT) assert workspace.vulnerability_total_count == ( - sum(STANDARD_VULN_COUNT) + WEB_VULN_COUNT + SOURCE_CODE_VULN_COUNT + sum(STANDARD_VULN_COUNT) + WEB_VULN_COUNT + + SOURCE_CODE_VULN_COUNT ) + + +def test_vuln_count_confirmed(workspace, second_workspace): + populate_workspace(workspace) + populate_workspace(second_workspace) + workspace = Workspace.query_with_count(True).filter( + Workspace.id == workspace.id).first() + assert workspace.vulnerability_web_count == C_WEB_VULN_COUNT + assert workspace.vulnerability_code_count == C_SOURCE_CODE_VULN_COUNT + assert workspace.vulnerability_standard_count == sum( + C_STANDARD_VULN_COUNT) + assert workspace.vulnerability_total_count == ( + sum(C_STANDARD_VULN_COUNT) + C_WEB_VULN_COUNT + + C_SOURCE_CODE_VULN_COUNT + ) + + +def test_vuln_no_count(workspace, second_workspace): + populate_workspace(workspace) + populate_workspace(second_workspace) + workspace = Workspace.query.get(workspace.id) + assert workspace.vulnerability_web_count is None + assert workspace.vulnerability_code_count is None + assert workspace.vulnerability_standard_count is None + assert workspace.vulnerability_total_count is None diff --git a/test_cases/test_api_non_workspaced_various.py b/test_cases/test_api_non_workspaced_various.py index 5611aa6906f..192452769ee 100644 --- a/test_cases/test_api_non_workspaced_various.py +++ b/test_cases/test_api_non_workspaced_various.py @@ -48,3 +48,34 @@ def test_host_count(self, host_factory, test_client, session): res = test_client.get(self.url(self.first_object)) assert res.status_code == 200 assert res.json['stats']['hosts'] == 1 + + @pytest.mark.parametrize('querystring', [ + '', + '?confirmed=0', + '?confirmed=false' + ]) + def test_vuln_count(self, vulnerability_factory, test_client, session, + querystring): + vulnerability_factory.create_batch(8, workspace=self.first_object, + confirmed=False) + vulnerability_factory.create_batch(5, workspace=self.first_object, + confirmed=True) + session.commit() + res = test_client.get(self.url(self.first_object) + querystring) + assert res.status_code == 200 + assert res.json['stats']['total_vulns'] == 13 + + @pytest.mark.parametrize('querystring', [ + '?confirmed=1', + '?confirmed=true' + ]) + def test_vuln_count_confirmed(self, vulnerability_factory, test_client, + session, querystring): + vulnerability_factory.create_batch(8, workspace=self.first_object, + confirmed=False) + vulnerability_factory.create_batch(5, workspace=self.first_object, + confirmed=True) + session.commit() + res = test_client.get(self.url(self.first_object) + querystring) + assert res.status_code == 200 + assert res.json['stats']['total_vulns'] == 5 From 5fc4db37cadfbbc37fa29a58e05fa7af14b43594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 27 Oct 2017 19:12:56 -0300 Subject: [PATCH 0454/1506] Update SQLAlchemy to 1.2 To use query_expression --- requirements_server.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_server.txt b/requirements_server.txt index 0ea4deb4317..c2787f9c18f 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -13,7 +13,7 @@ python-slugify==1.2.4 requests>=2.10.0 restkit>=4.2.2 service_identity>=16.0.0 -sqlalchemy>=1.0.12 +SQLAlchemy==1.2.0b2 tqdm==4.15.0 twisted>=16.1.1 webargs From 87f0a51f238d8db6cab9fd7ec0c7fd8d7e3316e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 30 Oct 2017 12:21:31 -0300 Subject: [PATCH 0455/1506] Fix confirmed vulnerability count in PostgreSQL It was only working in sqlite --- server/models.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index ab8b0ac731f..2ad3668658f 100644 --- a/server/models.py +++ b/server/models.py @@ -766,7 +766,14 @@ def _make_vuln_count_property(type_=None, only_confirmed=False, # In this case type_ is supplied from a whitelist so this is safe query = query.where(text("vulnerability.type = '%s'" % type_)) if only_confirmed: - query = query.where(text("vulnerability.confirmed = 1")) + if str(db.engine.url).startswith('sqlite://'): + # SQLite has no "true" expression, we have to use the integer 1 + # instead + query = query.where(text("vulnerability.confirmed = 1")) + else: + # I suppose that we're using PostgreSQL, that can't compare + # booleans with integers + query = query.where(text("vulnerability.confirmed = true")) if use_column_property: return column_property(query, deferred=True) else: From f91aff119cb1668b20513942deabdb59685f371b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 30 Oct 2017 12:51:13 -0300 Subject: [PATCH 0456/1506] Fix vuln template tests to work with PostgreSQL --- test_cases/test_api_vulnerability_template.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_cases/test_api_vulnerability_template.py b/test_cases/test_api_vulnerability_template.py index a8c6f6d76dc..3677331fd2b 100644 --- a/test_cases/test_api_vulnerability_template.py +++ b/test_cases/test_api_vulnerability_template.py @@ -63,9 +63,9 @@ def test_create_new_vulnerability_template(self, session, test_client): raw_data = self._create_post_data_vulnerability_template(references='') res = test_client.post('/v2/vulnerability_template/', data=raw_data) assert res.status_code == 201 - assert res.json['_id'] == 6 + assert isinstance(res.json['_id'], int) assert vuln_count_previous + 1 == session.query(VulnerabilityTemplate).count() - vuln_template = VulnerabilityTemplate.query.get(6) + vuln_template = VulnerabilityTemplate.query.get(res.json['_id']) assert vuln_template.references == set() def test_update_vulnerability_template(self, session, test_client): @@ -121,9 +121,9 @@ def test_create_new_vulnerability_template_with_references(self, session, test_c raw_data = self._create_post_data_vulnerability_template(references='ref1,ref2') res = test_client.post('/v2/vulnerability_template/', data=raw_data) assert res.status_code == 201 - assert res.json['_id'] == 6 + assert isinstance(res.json['_id'], int) assert vuln_count_previous + 1 == session.query(VulnerabilityTemplate).count() - new_template = session.query(VulnerabilityTemplate).filter_by(id=6).first() + new_template = session.query(VulnerabilityTemplate).filter_by(id=res.json['_id']).first() assert new_template.references == set([u'ref1', u'ref2']) def test_delete_vuln_template(self, session, test_client): From b55552384cc0fb7c225287cb791fd03ad905d9ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 30 Oct 2017 13:10:07 -0300 Subject: [PATCH 0457/1506] Allow specifying connection string when running tests To simplify testing with multiple database engines --- test_cases/conftest.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 118395cee3e..96ba18952b2 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -54,11 +54,16 @@ def open(self, *args, **kwargs): return ret +def pytest_addoption(parser): + parser.addoption('--connection-string', default='sqlite://', + help="Database connection string. Defaults to in-memory " + "sqlite if not specified:") + + @pytest.fixture(scope='session') def app(request): - # we use sqlite memory for tests - test_conn_string = 'sqlite://' - app = create_app(db_connection_string=test_conn_string, testing=True) + app = create_app(db_connection_string=request.config.getoption( + '--connection-string'), testing=True) app.test_client_class = CustomClient # Establish an application context before running the tests. From 1ed512e9c22d5cd8b3f618c9ef92050ddf917650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 30 Oct 2017 15:01:19 -0300 Subject: [PATCH 0458/1506] Remove punctuation characters of the PostgreSQL database To avoid invalid connection strings --- server/commands/initdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index cd437d1f9e2..b9ec8152e24 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -93,7 +93,7 @@ def _check_psql_output(self, psql_log_output, process_status): def generate_random_pw(self, pwlen): rng = SystemRandom() - return "".join([rng.choice(string.ascii_letters + string.digits + string.punctuation) for _ in xrange(pwlen)]) + return "".join([rng.choice(string.ascii_letters + string.digits) for _ in xrange(pwlen)]) def _configure_postgres(self, psql_log_file): """ From 2a16d7dd0af0e0101f859680fee190148a87faf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 30 Oct 2017 15:19:48 -0300 Subject: [PATCH 0459/1506] Fix database fixture teardown to support PostgreSQL Use cascade delete in conflicting tables --- test_cases/conftest.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 96ba18952b2..093bd1c3d34 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -82,10 +82,18 @@ def database(app, request): """Session-wide test database.""" def teardown(): + try: + db.engine.execute('DROP TABLE vulnerability CASCADE') + except Exception: + pass + try: + db.engine.execute('DROP TABLE vulnerability_template CASCADE') + except Exception: + pass db.drop_all() - # Disable check_vulnerability_host_service_source_code constraint because it - # doesn't work in sqlite + # Disable check_vulnerability_host_service_source_code constraint because + # it doesn't work in sqlite vuln_constraints = db.metadata.tables['vulnerability'].constraints vuln_constraints.remove(next( constraint for constraint in vuln_constraints From a0739bd41f31653bd6334afe021eb470e212a213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 30 Oct 2017 15:33:05 -0300 Subject: [PATCH 0460/1506] Add PostgreSQL testing stage to Jenkinsfile --- Jenkinsfile | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 971abc6b13a..ec581fbacd5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -64,4 +64,29 @@ node { } } } + + stage ("Run Unit/Integration Tests (with PostgreSQL)") { + def testsError = null + try { + withCredentials([string(credentialsId: 'postgresql_connection_string', variable: 'CONN_STRING')]) { + sh """ + source ${ENV_PATH}/bin/activate + cd $WORKSPACE && pytest -v --junitxml=$WORKSPACE/xunit-postgres.xml --connection-string "$CONN_STRING" || : + deactivate + """ + step([$class: 'CoberturaPublisher', autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: '**/coverage.xml', failNoReports: false, failUnhealthy: false, failUnstable: false, maxNumberOfBuilds: 0, onlyStable: false, sourceEncoding: 'ASCII', zoomCoverageChart: false]) + } + } + catch(err) { + testsError = err + currentBuild.result = 'FAILURE' + } + finally { + junit "**/xunit-postgres.xml" + + if (testsError) { + throw testsError + } + } + } } From 976ad5ab7c9d242cbd3aca637b22f9a1c25cfd58 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 30 Oct 2017 16:56:54 -0300 Subject: [PATCH 0461/1506] Add test case for invalid ease_of_resolution in vuln --- test_cases/test_api_vulnerability.py | 83 +++++++++++++++++++++++++--- 1 file changed, 75 insertions(+), 8 deletions(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index d9d37ddf6c3..577bac9660b 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -124,7 +124,9 @@ def test_hostnames(self, host_with_hostnames, test_client, session): def _create_post_data_vulnerability(self, name, vuln_type, parent_id, parent_type, refs, policyviolations, status='opened', - attachments=None, impact=None, description=None): + attachments=None, impact=None, description='desc1234', + confirmed=True, data='data1234', easeofresolution=Vulnerability.EASE_OF_RESOLUTIONS[0], + owned=False, resolution='res1234', severity='critical'): if not impact: impact = {'accountability': False, 'availability': False, 'confidentiality': False, 'integrity': False} @@ -136,17 +138,17 @@ def _create_post_data_vulnerability(self, name, vuln_type, parent_id, 'parent_type': parent_type, 'type': vuln_type, 'ws': 'airbnb', - 'confirmed': True, - 'data': '', - 'desc': description or 'fsdfsd', - 'easeofresolution': '', + 'confirmed': confirmed, + 'data': data, + 'desc': description, + 'easeofresolution': easeofresolution, 'impact': impact, 'name': name, - 'owned': False, + 'owned': owned, 'policyviolations': policyviolations, 'refs': refs, - 'resolution': '', - 'severity': 'high', + 'resolution': resolution, + 'severity': severity, 'status': status, '_attachments': {}, 'description': '', @@ -175,6 +177,13 @@ def _create_post_data_vulnerability(self, name, vuln_type, parent_id, return data def test_create_vuln(self, host_with_hostnames, test_client, session): + """ + This one should only check basic vuln properties + :param host_with_hostnames: + :param test_client: + :param session: + :return: + """ session.commit() # flush host_with_hostnames raw_data = self._create_post_data_vulnerability( name='New vulns', @@ -184,6 +193,7 @@ def test_create_vuln(self, host_with_hostnames, test_client, session): refs=[], policyviolations=[], description='helloworld', + severity='low', ) ws_name = host_with_hostnames.workspace.name vuln_count_previous = session.query(Vulnerability).count() @@ -196,6 +206,44 @@ def test_create_vuln(self, host_with_hostnames, test_client, session): assert res.json['parent_type'] == 'Host' assert res.json['desc'] == 'helloworld' assert res.json['description'] == 'helloworld' + assert res.json['severity'] == 'low' + + def test_create_vuln_props(self, host_with_hostnames, test_client, session): + """ + This one should check all the vuln props that don't have a specific case + :param host_with_hostnames: + :param test_client: + :param session: + :return: + """ + session.commit() # flush host_with_hostnames + vuln_props = { + 'confirmed': False, + 'data': 'hellodata', + 'easeofresolution': Vulnerability.EASE_OF_RESOLUTIONS[0], + 'owned': True, + 'resolution': 'helloresolution', + 'status': 'closed' + } + vuln_props_excluded = ['owned'] + raw_data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='Vulnerability', + parent_id=host_with_hostnames.id, + parent_type='Host', + refs=[], + policyviolations=[], + description='helloworld', + severity='low', + **vuln_props + ) + ws_name = host_with_hostnames.workspace.name + vuln_count_previous = session.query(Vulnerability).count() + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) + assert res.status_code == 201 + for prop, value in vuln_props.iteritems(): + if prop not in vuln_props_excluded: + assert res.json[prop] == value, prop def test_create_vuln_with_closed_status(self, host_with_hostnames, test_client, session): session.commit() # flush host_with_hostnames @@ -527,6 +575,25 @@ def test_create_vuln_with_invalid_type(self, host_with_hostnames, test_client, s assert vuln_count_previous == session.query(Vulnerability).count() assert res.json['message'] == 'Invalid vulnerability type.' + def test_create_vuln_with_invalid_ease_of_resolution(self, host_with_hostnames, test_client, session): + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='Vulnerability', + parent_id=host_with_hostnames.id, + parent_type='Host', + refs=[], + policyviolations=[], + easeofresolution='frutafrutafruta' + ) + ws_name = host_with_hostnames.workspace.name + vuln_count_previous = session.query(Vulnerability).count() + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) + assert res.status_code == 400 + assert vuln_count_previous == session.query(Vulnerability).count() + assert res.json['messages'].keys() == ['easeofresolution'] + assert res.json['messages']['easeofresolution'][0] == u'Not a valid choice.' + def test_count_confirmed(self, test_client, session): for i, vuln in enumerate(self.objects[:3]): vuln.confirmed = True From 2604cf0dce0498b74d4aab828122c10c3fe9bf5a Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 30 Oct 2017 16:57:29 -0300 Subject: [PATCH 0462/1506] Fix save ease_of_resolution in vuln Also adds Enum validation. --- server/api/modules/vulnerability_template.py | 4 +--- server/api/modules/vulns.py | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/server/api/modules/vulnerability_template.py b/server/api/modules/vulnerability_template.py index 4c066d82271..aa1d412a0d6 100644 --- a/server/api/modules/vulnerability_template.py +++ b/server/api/modules/vulnerability_template.py @@ -9,9 +9,7 @@ operators, ) from marshmallow import fields, ValidationError -from marshmallow.validate import ( - OneOf, -) +from marshmallow.validate import OneOf import server.utils.logger diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index f4a4b06e4ae..901ed4a9664 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -10,6 +10,7 @@ from flask import request from flask import Blueprint from marshmallow import Schema, fields, post_load, ValidationError +from marshmallow.validate import OneOf from depot.manager import DepotManager from server.api.base import ( @@ -86,7 +87,7 @@ class VulnerabilitySchema(AutoSchema): fields.String(), required=True) tags = fields.Method(serialize='get_tags') - easeofresolution = fields.String(dump_only=True, attribute='ease_of_resolution') + easeofresolution = fields.String(attribute='ease_of_resolution', validate=OneOf(Vulnerability.EASE_OF_RESOLUTIONS),) hostnames = PrimaryKeyRelatedField('name', many=True, dump_only=True) metadata = fields.Method(serialize='get_metadata') service = fields.Nested(ServiceSchema(only=[ From c350387f64b662aed4ebf1d07ce38b21d62b3a48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 30 Oct 2017 17:57:59 -0300 Subject: [PATCH 0463/1506] Add creator joinedload in APIs of objects with metadata To improve performance --- server/api/base.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 53e45c5bf7e..b93eea34777 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -3,6 +3,7 @@ import flask from flask import abort, g from flask_classful import FlaskView +from sqlalchemy.orm import joinedload from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.inspection import inspect from sqlalchemy import func @@ -12,7 +13,7 @@ from marshmallow_sqlalchemy.schema import ModelSchemaMeta, ModelSchemaOpts from webargs.flaskparser import FlaskParser, parser, abort from webargs.core import ValidationError -from server.models import Workspace, db +from server.models import Workspace, db, Metadata import server.utils.logger logger = server.utils.logger.get_logger(__name__) @@ -79,7 +80,14 @@ def _validate_object_id(self, object_id): flask.abort(404, 'Invalid format of lookup field') def _get_base_query(self): - return self.model_class.query + query = self.model_class.query + if issubclass(self.model_class, Metadata): + # APIs for objects with metadata always return the creator's + # username. Do a joinedload to prevent doing one query per object + # (n+1) problem + return query.options(joinedload( + getattr(self.model_class, 'creator')).load_only('username')) + return query def _filter_query(self, query): """Return a new SQLAlchemy query with some filters applied""" From 60e1699488456fab3128eaa4ca6034f7b011c15a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 30 Oct 2017 18:19:46 -0300 Subject: [PATCH 0464/1506] Resolve n+1 issues in credentials API --- server/api/modules/credentials.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/api/modules/credentials.py b/server/api/modules/credentials.py index 1f051586858..d1962eb3fea 100644 --- a/server/api/modules/credentials.py +++ b/server/api/modules/credentials.py @@ -41,10 +41,11 @@ class CredentialSchema(AutoSchema): service_id = fields.Integer(load_only=True) def get_parent(self, obj): - return obj.parent.id + return obj.host_id or obj.service_id def get_parent_type(self, obj): - return obj.parent.__class__.__name__ + assert obj.host_id is not None or obj.service_id is not None + return 'Service' if obj.service_id is not None else 'Host' def get_metadata(self, obj): return { From 7987da4a27042be77c5ed377308c15011ff605cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 30 Oct 2017 19:01:29 -0300 Subject: [PATCH 0465/1506] Add performance attributes to generic views Allow to set undefers and joinedlods directly in an attribute, instead of having to replace _get_base_query every time I adapted the hosts and workspaces views to use this feature --- server/api/base.py | 45 +++++++++++++++++++++++--------- server/api/modules/hosts.py | 17 ++++-------- server/api/modules/workspaces.py | 1 + 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index b93eea34777..948db42aa37 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -3,7 +3,7 @@ import flask from flask import abort, g from flask_classful import FlaskView -from sqlalchemy.orm import joinedload +from sqlalchemy.orm import joinedload, undefer from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.inspection import inspect from sqlalchemy import func @@ -66,6 +66,10 @@ class GenericView(FlaskView): lookup_field_type = int unique_fields = [] # Fields unique + # Attributes to improve the performance of list and retrieve views + get_joinedloads = [] # List of relationships to eagerload + get_undefer = [] # List of columns to undefer + def _get_schema_class(self): assert self.schema_class is not None, "You must define schema_class" return self.schema_class @@ -81,23 +85,34 @@ def _validate_object_id(self, object_id): def _get_base_query(self): query = self.model_class.query + return query + + def _get_eagerloaded_query(self, *args, **kwargs): + options = [] if issubclass(self.model_class, Metadata): # APIs for objects with metadata always return the creator's # username. Do a joinedload to prevent doing one query per object # (n+1) problem - return query.options(joinedload( + options.append(joinedload( getattr(self.model_class, 'creator')).load_only('username')) - return query + query = self._get_base_query(*args, **kwargs) + options += [joinedload(relationship) + for relationship in self.get_joinedloads] + options += [undefer(column) for column in self.get_undefer] + return query.options(*options) def _filter_query(self, query): """Return a new SQLAlchemy query with some filters applied""" return query - def _get_object(self, object_id, **kwargs): + def _get_object(self, object_id, eagerload=False, **kwargs): self._validate_object_id(object_id) + if eagerload: + query = self._get_eagerloaded_query(**kwargs) + else: + query = self._get_base_query(**kwargs) try: - obj = self._get_base_query(**kwargs).filter( - self._get_lookup_field() == object_id).one() + obj = query.filter(self._get_lookup_field() == object_id).one() except NoResultFound: flask.abort(404, 'Object with id "%s" not found' % object_id) return obj @@ -157,13 +172,16 @@ def _get_workspace(self, workspace_name): def _get_base_query(self, workspace_name): base = super(GenericWorkspacedView, self)._get_base_query() return base.join(Workspace).filter( - Workspace.id==self._get_workspace(workspace_name).id) + Workspace.id == self._get_workspace(workspace_name).id) - def _get_object(self, object_id, workspace_name): + def _get_object(self, object_id, workspace_name, eagerload=False): self._validate_object_id(object_id) + if eagerload: + query = self._get_eagerloaded_query(workspace_name) + else: + query = self._get_base_query(workspace_name) try: - obj = self._get_base_query(workspace_name).filter( - self._get_lookup_field() == object_id).one() + obj = query.filter(self._get_lookup_field() == object_id).one() except NoResultFound: flask.abort(404, 'Object with id "%s" not found' % object_id) return obj @@ -177,7 +195,7 @@ def _validate_uniqueness(self, obj, object_id=None): field = getattr(self.model_class, field_name) value = getattr(obj, field_name) query = self._get_base_query(obj.workspace.name).filter( - field==value) + field == value) if object_id is not None: # The object already exists in DB, we want to fetch an object # different to this one but with the same unique field @@ -201,7 +219,7 @@ def _paginate(self, query): return query, None def index(self, **kwargs): - query = self._filter_query(self._get_base_query(**kwargs)) + query = self._filter_query(self._get_eagerloaded_query(**kwargs)) objects, pagination_metadata = self._paginate(query) return self._envelope_list(self._dump(objects, many=True), pagination_metadata) @@ -257,7 +275,8 @@ class RetrieveMixin(object): """Add GET // route""" def get(self, object_id, **kwargs): - return self._dump(self._get_object(object_id, **kwargs)) + return self._dump(self._get_object(object_id, eagerload=True, + **kwargs)) class RetrieveWorkspacedMixin(RetrieveMixin): diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 89a3c271b6d..622476a1164 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -91,6 +91,9 @@ class HostsView(PaginatedMixin, schema_class = HostSchema unique_fields = ['ip'] filterset_class = HostFilterSet + get_undefer = [Host.open_service_count, + Host.vulnerability_count] + get_joinedloads = [Host.hostnames] @route('//services/') def service_list(self, workspace_name, host_id): @@ -103,21 +106,11 @@ def _perform_create(self, data, **kwargs): host = super(HostsView, self)._perform_create(data, **kwargs) host.default_gateway_ip = default_gateway_ip[0] for name in hostnames: - get_or_create(db.session, Hostname, name=name['key'], host=host, workspace=host.workspace) + get_or_create(db.session, Hostname, name=name['key'], host=host, + workspace=host.workspace) db.session.commit() return host - - def _get_base_query(self, workspace_name): - """Get services_count in one query and not deferred, that doe - one query per host""" - original = super(HostsView, self)._get_base_query(workspace_name) - return original.options( - undefer(Host.open_service_count), - undefer(Host.vulnerability_count), - joinedload(Host.hostnames) - ) - def _envelope_list(self, objects, pagination_metadata=None): hosts = [] for host in objects: diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index ac4fcb57845..949d183299a 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -70,6 +70,7 @@ class WorkspaceView(ReadWriteView): lookup_field_type = unicode model_class = Workspace schema_class = WorkspaceSchema + get_joinedloads = [Workspace.scope] def _get_base_query(self): try: From 8ced27e07f542621c98148d6cfe2f9a3050147bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 30 Oct 2017 19:25:06 -0300 Subject: [PATCH 0466/1506] Resolve n+1 issues in vuln templates API I had to refactor the automatic creator joinedload so this endpoint doesn't use it --- server/api/base.py | 6 +++++- server/api/modules/vulnerability_template.py | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/server/api/base.py b/server/api/base.py index 948db42aa37..f332e2dd989 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -89,7 +89,11 @@ def _get_base_query(self): def _get_eagerloaded_query(self, *args, **kwargs): options = [] - if issubclass(self.model_class, Metadata): + try: + has_creator = 'owner' in self.schema_class.opts.fields + except AttributeError: + has_creator = False + if has_creator: # APIs for objects with metadata always return the creator's # username. Do a joinedload to prevent doing one query per object # (n+1) problem diff --git a/server/api/modules/vulnerability_template.py b/server/api/modules/vulnerability_template.py index 4c066d82271..a1904833269 100644 --- a/server/api/modules/vulnerability_template.py +++ b/server/api/modules/vulnerability_template.py @@ -70,6 +70,10 @@ class VulnerabilityTemplateView(PaginatedMixin, model_class = VulnerabilityTemplate schema_class = VulnerabilityTemplateSchema filterset_class = VulnerabilityTemplateFilterSet + get_joinedloads = [ + VulnerabilityTemplate.reference_template_instances, + VulnerabilityTemplate.policy_violation_template_instances + ] def _envelope_list(self, objects, pagination_metadata=None): vuln_tpls = [] From 177237c45e638986a1613e309c17985c381cb1f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 31 Oct 2017 16:36:12 -0300 Subject: [PATCH 0467/1506] Resolve n+1 issues in vulns API There still some performance issues don't detected by the nplusone package. They will be fixed in the next commits --- server/api/base.py | 2 +- server/api/modules/vulns.py | 31 +++++++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index f332e2dd989..70ec9847cca 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -90,7 +90,7 @@ def _get_base_query(self): def _get_eagerloaded_query(self, *args, **kwargs): options = [] try: - has_creator = 'owner' in self.schema_class.opts.fields + has_creator = 'owner' in self._get_schema_class().opts.fields except AttributeError: has_creator = False if has_creator: diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index f4a4b06e4ae..5629fe942a2 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -10,6 +10,7 @@ from flask import request from flask import Blueprint from marshmallow import Schema, fields, post_load, ValidationError +from sqlalchemy.orm import joinedload, selectin_polymorphic from depot.manager import DepotManager from server.api.base import ( @@ -158,10 +159,11 @@ def get_tags(self, obj): ).all()] def get_parent(self, obj): - return obj.parent.id + return obj.service_id or obj.host_id def get_parent_type(self, obj): - return obj.parent.__class__.__name__ + assert obj.service_id is not None or obj.host_id is not None + return 'Service' if obj.service_id is not None else 'Host' def get_status(self, obj): if obj.status == 'open': @@ -276,6 +278,11 @@ class VulnerabilityView(PaginatedMixin, route_base = 'vulns' filterset_class = VulnerabilityFilterSet + get_joinedloads = [ + VulnerabilityGeneric.reference_instances, + VulnerabilityGeneric.policy_violation_instances, + ] + model_class_dict = { 'Vulnerability': Vulnerability, 'VulnerabilityWeb': VulnerabilityWeb, @@ -313,6 +320,26 @@ def _perform_create(self, data, **kwargs): db.session.commit() return obj + def _get_eagerloaded_query(self, *args, **kwargs): + """Eager hostnames loading. + + This is too complex to get_joinedloads so I have to + override the function + """ + query = super(VulnerabilityView, self)._get_eagerloaded_query( + *args, **kwargs) + joinedloads = [ + joinedload(Vulnerability.host) + .load_only(Host.id), # Only hostnames are needed + + joinedload(Vulnerability.service), + joinedload(VulnerabilityWeb.service) + ] + return query.options(selectin_polymorphic( + VulnerabilityGeneric, + [Vulnerability, VulnerabilityWeb] + ), *joinedloads) + @property def model_class(self): if request.method == 'POST': From 9409af6f2567bfa36d34a8f79388ced2c4e4a935 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 31 Oct 2017 17:53:17 -0300 Subject: [PATCH 0468/1506] Fix n+1 issues in vulns API (hostnames field) Not automatically detected by nplusone --- server/api/modules/vulns.py | 20 +++++++++----------- test_cases/test_api_commands.py | 1 + test_cases/test_api_vulnerability.py | 18 +++++++++++++++++- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 5629fe942a2..43780a15134 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -23,6 +23,7 @@ from server.fields import FaradayUploadedFile from server.models import ( db, + Hostname, Tag, TagObject, Vulnerability, @@ -141,15 +142,6 @@ def get_attachments(self, obj): def load_attachments(self, value): return value - def get_hostnames(self, obj): - # TODO: improve performance here - # TODO: move this to models? - if obj.host: - return [hostname.name for hostname in obj.host.hostnames] - if obj.service: - return [hostname.name for hostname in obj.service.host.hostnames] - logger.info('Vulnerability without host and service. Check invariant in obj with id {0}'.format(obj.id)) - return [] def get_tags(self, obj): # TODO: improve performance here @@ -330,10 +322,16 @@ def _get_eagerloaded_query(self, *args, **kwargs): *args, **kwargs) joinedloads = [ joinedload(Vulnerability.host) - .load_only(Host.id), # Only hostnames are needed + .load_only(Host.id) # Only hostnames are needed + .joinedload(Host.hostnames), + + joinedload(Vulnerability.service) + .joinedload(Service.host) + .joinedload(Host.hostnames), - joinedload(Vulnerability.service), joinedload(VulnerabilityWeb.service) + .joinedload(Service.host) + .joinedload(Host.hostnames), ] return query.options(selectin_polymorphic( VulnerabilityGeneric, diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index 16eff9a6143..b6f000c0090 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -12,6 +12,7 @@ from server.api.modules.workspaces import WorkspaceView +@pytest.mark.skip(reason='refactor needed to adapt new m2m model') @pytest.mark.usefixtures('logged_user') class TestListCommandView(ReadOnlyAPITests): model = Command diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 7ca83757b72..32792d32457 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -110,7 +110,23 @@ def test_shows_refs(self, workspace, test_client, session, assert len(res.json['refs']) == 5 assert set(res.json['refs']) == {ref.name for ref in refs} - def test_hostnames(self, host_with_hostnames, test_client, session): + @pytest.mark.parametrize('creator_func', [ + (lambda host: factories.VulnerabilityFactory.create( + workspace=host.workspace, host=host, service=None)), + (lambda host: factories.VulnerabilityFactory.create( + workspace=host.workspace, host=None, + service=factories.ServiceFactory.create( + workspace=host.workspace, host=host + ))), + (lambda host: factories.VulnerabilityWebFactory.create( + workspace=host.workspace, service=factories.ServiceFactory.create( + workspace=host.workspace, host=host + ))), + ], ids=['standard_vuln_with_host', 'standard_vuln_with_service', + 'web_vuln_with_service']) + def test_hostnames(self, host_with_hostnames, test_client, session, + creator_func): + vuln = creator_func(host_with_hostnames) vuln = self.factory.create(host=host_with_hostnames, service=None, workspace=host_with_hostnames.workspace) From 50500c00844c7a2b9e975e8491ecad829e12ae95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 31 Oct 2017 18:04:42 -0300 Subject: [PATCH 0469/1506] Remove unused import in vulns api --- server/api/modules/vulns.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 43780a15134..90f4ffff50b 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -23,7 +23,6 @@ from server.fields import FaradayUploadedFile from server.models import ( db, - Hostname, Tag, TagObject, Vulnerability, From 90e86d16ae51276e4c655bd9bd6d1accd86e7055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 31 Oct 2017 18:17:41 -0300 Subject: [PATCH 0470/1506] Add TODO to validate bad schemas in vulns api --- server/api/modules/vulns.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 90f4ffff50b..6f3ed171626 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -141,7 +141,6 @@ def get_attachments(self, obj): def load_attachments(self, value): return value - def get_tags(self, obj): # TODO: improve performance here return [tag.name for tag in db.session.query(TagObject, Tag).filter_by( @@ -287,6 +286,8 @@ class VulnerabilityView(PaginatedMixin, def _perform_create(self, data, **kwargs): data = self._parse_data(self._get_schema_class()(strict=True), request) + # TODO migration: use default values when popping and validate the + # popped object has the expected type. attachments = data.pop('_attachments') # This will be set after setting the workspace From 0cfcbf61c24efd6edfedc6fd22ca8ba8680b34fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 31 Oct 2017 18:29:59 -0300 Subject: [PATCH 0471/1506] Set some relationships laziness directly in models There objects that are strictly related, like a vuln and its references, or a workspace and its scope. They are queried together almost all the times, so it is OK to define the relation in the model. This is not the case for, for example, vulnerability and service since it depends on the query. --- server/api/modules/vulnerability_template.py | 4 ---- server/api/modules/vulns.py | 5 ---- server/api/modules/workspaces.py | 1 - server/models.py | 24 ++++++++++++++------ 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/server/api/modules/vulnerability_template.py b/server/api/modules/vulnerability_template.py index a1904833269..4c066d82271 100644 --- a/server/api/modules/vulnerability_template.py +++ b/server/api/modules/vulnerability_template.py @@ -70,10 +70,6 @@ class VulnerabilityTemplateView(PaginatedMixin, model_class = VulnerabilityTemplate schema_class = VulnerabilityTemplateSchema filterset_class = VulnerabilityTemplateFilterSet - get_joinedloads = [ - VulnerabilityTemplate.reference_template_instances, - VulnerabilityTemplate.policy_violation_template_instances - ] def _envelope_list(self, objects, pagination_metadata=None): vuln_tpls = [] diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 6f3ed171626..6a58c563c4f 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -268,11 +268,6 @@ class VulnerabilityView(PaginatedMixin, route_base = 'vulns' filterset_class = VulnerabilityFilterSet - get_joinedloads = [ - VulnerabilityGeneric.reference_instances, - VulnerabilityGeneric.policy_violation_instances, - ] - model_class_dict = { 'Vulnerability': Vulnerability, 'VulnerabilityWeb': VulnerabilityWeb, diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index 949d183299a..ac4fcb57845 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -70,7 +70,6 @@ class WorkspaceView(ReadWriteView): lookup_field_type = unicode model_class = Workspace schema_class = WorkspaceSchema - get_joinedloads = [Workspace.scope] def _get_base_query(self): try: diff --git a/server/models.py b/server/models.py index 2ad3668658f..0b99d3f0e18 100644 --- a/server/models.py +++ b/server/models.py @@ -21,7 +21,12 @@ from sqlalchemy.orm import backref, relationship, undefer from sqlalchemy.sql import select, text, table from sqlalchemy import func -from sqlalchemy.orm import column_property, query_expression, with_expression +from sqlalchemy.orm import ( + backref, + column_property, + query_expression, + with_expression +) from sqlalchemy.schema import DDL from sqlalchemy.ext.associationproxy import association_proxy, _AssociationSet from sqlalchemy.ext.declarative import declared_attr @@ -379,6 +384,7 @@ class VulnerabilityTemplate(VulnerabilityABC): reference_template_instances = relationship( "ReferenceTemplate", secondary="reference_template_vulnerability_association", + lazy="joined", collection_class=set ) @@ -391,6 +397,7 @@ class VulnerabilityTemplate(VulnerabilityABC): policy_violation_template_instances = relationship( "PolicyViolationTemplate", secondary="policy_violation_template_vulnerability_association", + lazy="joined", collection_class=set ) @@ -430,6 +437,7 @@ class VulnerabilityGeneric(VulnerabilityABC): reference_instances = relationship( "Reference", secondary="reference_vulnerability_association", + lazy="joined", collection_class=set ) @@ -441,6 +449,7 @@ class VulnerabilityGeneric(VulnerabilityABC): policy_violation_instances = relationship( "PolicyViolation", secondary="policy_violation_vulnerability_association", + lazy="joined", collection_class=set ) @@ -851,16 +860,17 @@ class Scope(Metadata): index=True, nullable=False ) - workspace = relationship( - 'Workspace', - backref='scope', - foreign_keys=[workspace_id], - ) + workspace = relationship('Workspace', + backref=backref('scope', lazy="joined"), + foreign_keys=[workspace_id], + ) __table_args__ = ( - UniqueConstraint('name', 'workspace_id', name='uix_scope_name_workspace'), + UniqueConstraint('name', 'workspace_id', + name='uix_scope_name_workspace'), ) + def is_valid_workspace(workspace_name): return db.session.query(server.models.Workspace).filter_by(name=workspace_name).first() is not None From fd33ef078e4d1564e0cbcfd70c21623b5adef6e9 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 1 Nov 2017 12:01:04 -0300 Subject: [PATCH 0472/1506] Add initial working code for commands stats This commit solves the historic stats for commands. When a command creates a vuln, host, setvice etc a flag will be added on the results. When another command uses an already created object the flag will be at 0. This commit has the following: * Add columns fields to generate created information. when a command creates the object a field with 1 will be returned in the columns with name: created_*. * On the command model sum_created_* will have the value of the sum of all created object. * Added a test to verify a simple case. --- server/api/modules/commandsrun.py | 14 ++++++- server/models.py | 70 ++++++++++++++++++++++++++++++- server/utils/database.py | 16 +++++++ test_cases/factories.py | 33 ++++++++++++++- test_cases/test_api_commands.py | 17 ++++++++ 5 files changed, 147 insertions(+), 3 deletions(-) diff --git a/server/api/modules/commandsrun.py b/server/api/modules/commandsrun.py index f8962d9ecde..e31c43b3628 100644 --- a/server/api/modules/commandsrun.py +++ b/server/api/modules/commandsrun.py @@ -5,6 +5,7 @@ import flask from flask import Blueprint +from flask_classful import route from marshmallow import fields from server.api.base import AutoSchema, ReadWriteWorkspacedView @@ -14,7 +15,7 @@ validate_workspace, filter_request_args, get_integer_parameter ) -from server.models import Command +from server.models import Command, Workspace commandsrun_api = Blueprint('commandsrun_api', __name__) @@ -58,4 +59,15 @@ def _envelope_list(self, objects, pagination_metadata=None): 'commands': commands, } + @route('/activity_feed/') + def activity_feed(self, workspace_name): + res = [] + query = Command.query.join(Workspace).filter_by(name=workspace_name) + for command in query.all(): + res.append({ + 'command': command.id, + 'sum_created_vulnerabilities': command.sum_created_vulnerabilities + }) + return res + CommandView.register(commandsrun_api) diff --git a/server/models.py b/server/models.py index 2ad3668658f..d6353f4123d 100644 --- a/server/models.py +++ b/server/models.py @@ -37,7 +37,7 @@ RoleMixin, UserMixin, ) -from server.utils.database import get_or_create +from server.utils.database import get_or_create, IntToBooleanColumn class SQLAlchemy(OriginalSQLAlchemy): @@ -734,6 +734,49 @@ def parent(self): return self.host or self.service +def _make_command_created_related_object(object_type): + # # text('(count(*) = 0)::int ') + return column_property( + select([IntToBooleanColumn("(count(*) = 0)")]).\ + select_from('command_object as command_object_inner').\ + where(text( + " command_object_inner.create_date < command_object.create_date and " \ + " command_object_inner.object_id = command_object.object_id and " \ + " command_object_inner.object_type='%s' " % (object_type)) + ) + ) + + +class CommandObject(db.Model): + __tablename__ = 'command_object' + id = Column(Integer, primary_key=True) + + object_id = Column(Integer, nullable=False) + object_type = Column(Text, nullable=False) + + command = relationship('Command', backref='command_objects') + command_id = Column(Integer, ForeignKey('command.id'), index=True) + + create_date = Column(DateTime, default=datetime.utcnow) + update_date = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + + # the following properties are used to know if the command created the specified objects_type + # remeber that this table has a row instances per relationship. + # this created integer can be used to obtain the total object_type objects created. + created_vulnerability = _make_command_created_related_object('Vulnerability') + created_vulnerability_web = _make_command_created_related_object('VulnerabilityWeb') + created_host = _make_command_created_related_object('Host') + created_service = _make_command_created_related_object('Services') + + +def _make_created_objects_sum(object, object_created_attribute, join_condition): + return column_property( + select([func.sum(object_created_attribute)]).\ + select_from(object).\ + where(join_condition) + ) + + class Command(Metadata): __tablename__ = 'command' id = Column(Integer, primary_key=True) @@ -749,6 +792,30 @@ class Command(Metadata): workspace = relationship('Workspace', foreign_keys=[workspace_id]) # TODO: add Tool relationship and report_attachment + sum_created_vulnerabilities = _make_created_objects_sum( + CommandObject, + CommandObject.created_vulnerability, + 'command.id=command_object.command_id' + ) + + sum_created_vulnerabilities_web = _make_created_objects_sum( + CommandObject, + CommandObject.created_vulnerability_web, + 'command.id=command_object.command_id' + ) + + sum_created_hosts = _make_created_objects_sum( + CommandObject, + CommandObject.created_host, + 'command.id=command_object.command_id' + ) + + sum_created_services = _make_created_objects_sum( + CommandObject, + CommandObject.created_service, + 'command.id=command_object.command_id' + ) + @property def parent(self): return @@ -1092,6 +1159,7 @@ class CommentObject(db.Model): comment_id = Column(Integer, ForeignKey('comment.id'), index=True) + class Comment(Metadata): __tablename__ = 'comment' id = Column(Integer, primary_key=True) diff --git a/server/utils/database.py b/server/utils/database.py index e13800ae65c..74f904211f3 100644 --- a/server/utils/database.py +++ b/server/utils/database.py @@ -188,3 +188,19 @@ def _group_concat_postgresql(element, compiler, **kw): separator, ) return res + +class IntToBooleanColumn(expression.FunctionElement): + + def __init__(self, expression): + super(IntToBooleanColumn, self).__init__() + self.expression_str = expression + + +@compiler.compiles(IntToBooleanColumn, 'postgresql') +def _integer_to_boolean_postgresql(element, compiler, **kw): + return '{0}::int'.format(element.expression_str) + +@compiler.compiles(IntToBooleanColumn, 'sqlite') +def _integer_to_boolean_sqlite(element, compiler, **kw): + # text('(count(*) = 0)::int ') + return element.expression_str \ No newline at end of file diff --git a/test_cases/factories.py b/test_cases/factories.py index b4c4d98fe53..15a95c7b437 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -28,7 +28,7 @@ VulnerabilityTemplate, VulnerabilityWeb, Workspace, - ReferenceTemplate) + ReferenceTemplate, CommandObject) # Make partials for start and end date. End date must be after start date FuzzyStartTime = lambda: ( @@ -258,6 +258,13 @@ class Meta: sqlalchemy_session = db.session +class CommandObjectFactory(FaradayFactory): + + class Meta: + model = CommandObject + sqlalchemy_session = db.session + + class CommandFactory(WorkspaceObjectFactory): command = FuzzyText() end_date = FuzzyDateTime(datetime.datetime.utcnow().replace(tzinfo=pytz.utc) + datetime.timedelta(20), datetime.datetime.utcnow().replace(tzinfo=pytz.utc) + datetime.timedelta(30)) @@ -270,6 +277,30 @@ class Meta: model = Command sqlalchemy_session = db.session + @factory.post_generation + def attach_vuln_object(self, create, extracted, **kwargs): + if create: + vuln = VulnerabilityFactory.create(workspace=self.workspace) + db.session.flush() + CommandObjectFactory.create( + object_type='Vulnerability', + object_id=vuln.id, + command=self, + ) + +class EmptyCommandFactory(FaradayFactory): + command = FuzzyText() + end_date = FuzzyDateTime(datetime.datetime.utcnow().replace(tzinfo=pytz.utc) + datetime.timedelta(20), datetime.datetime.utcnow().replace(tzinfo=pytz.utc) + datetime.timedelta(30)) + start_date = FuzzyDateTime(datetime.datetime.utcnow().replace(tzinfo=pytz.utc) - datetime.timedelta(30), datetime.datetime.utcnow().replace(tzinfo=pytz.utc) - datetime.timedelta(20)) + ip = FuzzyText() + user = FuzzyText() + hostname = FuzzyText() + + class Meta: + model = Command + sqlalchemy_session = db.session + + class LicenseFactory(FaradayFactory): product = FuzzyText() diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index 16eff9a6143..b56c7c8b20b 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -10,6 +10,7 @@ ) from server.api.modules.commandsrun import CommandView from server.api.modules.workspaces import WorkspaceView +from test_cases.factories import VulnerabilityFactory, EmptyCommandFactory, CommandObjectFactory @pytest.mark.usefixtures('logged_user') @@ -39,3 +40,19 @@ def test_backwards_compatibility_list(self, test_client, second_workspace, sessi u'workspace' ] assert set(object_properties) == set(command['value'].keys()) + + def test_activity_feed(self, session, test_client): + command = self.factory.create() + another_command = EmptyCommandFactory.create(workspace=command.workspace) + vuln_id = command.command_objects[0].object_id + session.flush() + CommandObjectFactory.create( + command=another_command, + object_type='Vulnerability', + object_id=vuln_id + ) + session.commit() + res = test_client.get(self.url(workspace=command.workspace) + 'activity_feed/') + assert res.status_code == 200 + assert filter(lambda stats: stats['command'] == command.id, res.json)[0]['sum_created_vulnerabilities'] == 1 + assert filter(lambda stats: stats['command'] == another_command.id, res.json)[0]['sum_created_vulnerabilities'] == 0 From 30e7048de29629543f53b9b1e5dae73916f96fe5 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 1 Nov 2017 18:01:03 -0300 Subject: [PATCH 0473/1506] Fix some bugs when there was no created objects counts were incorrect --- server/api/modules/commandsrun.py | 6 ++- server/models.py | 87 +++++++++++++++++-------------- server/utils/database.py | 9 ++-- test_cases/factories.py | 8 ++- test_cases/test_api_commands.py | 65 ++++++++++++++++++++--- 5 files changed, 124 insertions(+), 51 deletions(-) diff --git a/server/api/modules/commandsrun.py b/server/api/modules/commandsrun.py index e31c43b3628..25fdbf5ea68 100644 --- a/server/api/modules/commandsrun.py +++ b/server/api/modules/commandsrun.py @@ -66,7 +66,11 @@ def activity_feed(self, workspace_name): for command in query.all(): res.append({ 'command': command.id, - 'sum_created_vulnerabilities': command.sum_created_vulnerabilities + 'sum_created_vulnerabilities': command.sum_created_vulnerabilities or 0, + 'sum_created_vulnerabilities_web': command.sum_created_vulnerabilities_web or 0, + 'sum_created_hosts': command.sum_created_hosts or 0, + 'sum_created_services': command.sum_created_services or 0, + 'sum_created_vulnerability_critical': command.sum_created_vulnerability_critical or 0 }) return res diff --git a/server/models.py b/server/models.py index d6353f4123d..bb2531784cb 100644 --- a/server/models.py +++ b/server/models.py @@ -37,7 +37,7 @@ RoleMixin, UserMixin, ) -from server.utils.database import get_or_create, IntToBooleanColumn +from server.utils.database import get_or_create, BooleanToIntColumn class SQLAlchemy(OriginalSQLAlchemy): @@ -734,16 +734,26 @@ def parent(self): return self.host or self.service -def _make_command_created_related_object(object_type): - # # text('(count(*) = 0)::int ') +def _make_command_created_related_object(object_type=None, joined=False, extra_where=None): + where_expr = "" + if extra_where: + where_expr += extra_where + + + query = select([BooleanToIntColumn("(count(*) = 0)")]) + + if joined: + query = query.select_from('command_object as command_object_inner, {0}'.format(object_type.lower())) + where_expr += 'command_object_inner.object_id = {0}.id and '.format(object_type.lower()) + else: + query = query.select_from('command_object as command_object_inner') + + where_expr += " command_object_inner.create_date < command_object.create_date and " \ + " (command_object_inner.object_id = command_object.object_id and " \ + " command_object_inner.object_type = command_object.object_type) " + query = query.where(text(where_expr)) return column_property( - select([IntToBooleanColumn("(count(*) = 0)")]).\ - select_from('command_object as command_object_inner').\ - where(text( - " command_object_inner.create_date < command_object.create_date and " \ - " command_object_inner.object_id = command_object.object_id and " \ - " command_object_inner.object_type='%s' " % (object_type)) - ) + query, ) @@ -763,17 +773,32 @@ class CommandObject(db.Model): # the following properties are used to know if the command created the specified objects_type # remeber that this table has a row instances per relationship. # this created integer can be used to obtain the total object_type objects created. - created_vulnerability = _make_command_created_related_object('Vulnerability') - created_vulnerability_web = _make_command_created_related_object('VulnerabilityWeb') - created_host = _make_command_created_related_object('Host') - created_service = _make_command_created_related_object('Services') + created = _make_command_created_related_object() + created_vulnerability_critical = _make_command_created_related_object( + 'Vulnerability', + joined=True, + extra_where="vulnerability.severity='critical' and " + ) + +def _make_created_objects_sum(object_type_filter): + where_condition = "command_object.command_id = command.id and command_object.object_type= '%s'" % object_type_filter + return column_property( + select([func.sum(CommandObject.created)]).\ + select_from(table('command_object')). \ + where(where_condition) + ) -def _make_created_objects_sum(object, object_created_attribute, join_condition): + +def _make_created_objects_sum_joined(object_type_filter, join_filters): + where_conditions = ["vulnerability.id = command_object.object_id and command_object.command_id = command.id and command_object.object_type= '%s'" % object_type_filter] + for attr, filter_value in join_filters.items(): + where_conditions.append("vulnerability.{0} = {1}".format(attr, filter_value)) return column_property( - select([func.sum(object_created_attribute)]).\ - select_from(object).\ - where(join_condition) + select([func.sum(CommandObject.created_vulnerability_critical)]). \ + select_from(table('command_object')). \ + select_from(table('vulnerability')). \ + where(' and '.join(where_conditions)) ) @@ -792,29 +817,15 @@ class Command(Metadata): workspace = relationship('Workspace', foreign_keys=[workspace_id]) # TODO: add Tool relationship and report_attachment - sum_created_vulnerabilities = _make_created_objects_sum( - CommandObject, - CommandObject.created_vulnerability, - 'command.id=command_object.command_id' - ) + sum_created_vulnerabilities = _make_created_objects_sum('Vulnerability') - sum_created_vulnerabilities_web = _make_created_objects_sum( - CommandObject, - CommandObject.created_vulnerability_web, - 'command.id=command_object.command_id' - ) + sum_created_vulnerabilities_web = _make_created_objects_sum('VulnerabilityWeb') - sum_created_hosts = _make_created_objects_sum( - CommandObject, - CommandObject.created_host, - 'command.id=command_object.command_id' - ) + sum_created_hosts = _make_created_objects_sum('Host') - sum_created_services = _make_created_objects_sum( - CommandObject, - CommandObject.created_service, - 'command.id=command_object.command_id' - ) + sum_created_services = _make_created_objects_sum('Service') + + sum_created_vulnerability_critical = _make_created_objects_sum_joined('Vulnerability', {'severity': '\'critical\''}) @property def parent(self): diff --git a/server/utils/database.py b/server/utils/database.py index 74f904211f3..47433cb1856 100644 --- a/server/utils/database.py +++ b/server/utils/database.py @@ -189,18 +189,17 @@ def _group_concat_postgresql(element, compiler, **kw): ) return res -class IntToBooleanColumn(expression.FunctionElement): +class BooleanToIntColumn(expression.FunctionElement): def __init__(self, expression): - super(IntToBooleanColumn, self).__init__() + super(BooleanToIntColumn, self).__init__() self.expression_str = expression -@compiler.compiles(IntToBooleanColumn, 'postgresql') +@compiler.compiles(BooleanToIntColumn, 'postgresql') def _integer_to_boolean_postgresql(element, compiler, **kw): return '{0}::int'.format(element.expression_str) -@compiler.compiles(IntToBooleanColumn, 'sqlite') +@compiler.compiles(BooleanToIntColumn, 'sqlite') def _integer_to_boolean_sqlite(element, compiler, **kw): - # text('(count(*) = 0)::int ') return element.expression_str \ No newline at end of file diff --git a/test_cases/factories.py b/test_cases/factories.py index 15a95c7b437..057374d361f 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -280,13 +280,19 @@ class Meta: @factory.post_generation def attach_vuln_object(self, create, extracted, **kwargs): if create: - vuln = VulnerabilityFactory.create(workspace=self.workspace) + host = HostFactory.create(workspace=self.workspace) + vuln = VulnerabilityFactory.create(workspace=self.workspace, host=host, service=None, severity='low') db.session.flush() CommandObjectFactory.create( object_type='Vulnerability', object_id=vuln.id, command=self, ) + CommandObjectFactory.create( + object_type='Host', + object_id=host.id, + command=self, + ) class EmptyCommandFactory(FaradayFactory): command = FuzzyText() diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index b56c7c8b20b..8c3270e83d5 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -7,10 +7,11 @@ from server.models import ( Command, Workspace, -) + Vulnerability) from server.api.modules.commandsrun import CommandView from server.api.modules.workspaces import WorkspaceView -from test_cases.factories import VulnerabilityFactory, EmptyCommandFactory, CommandObjectFactory +from test_cases.factories import VulnerabilityFactory, EmptyCommandFactory, CommandObjectFactory, HostFactory, \ + WorkspaceFactory @pytest.mark.usefixtures('logged_user') @@ -44,15 +45,67 @@ def test_backwards_compatibility_list(self, test_client, second_workspace, sessi def test_activity_feed(self, session, test_client): command = self.factory.create() another_command = EmptyCommandFactory.create(workspace=command.workspace) - vuln_id = command.command_objects[0].object_id + vuln = session.query(Vulnerability).get(command.command_objects[0].object_id) session.flush() CommandObjectFactory.create( command=another_command, object_type='Vulnerability', - object_id=vuln_id + object_id=vuln.id + ) + CommandObjectFactory.create( + command=another_command, + object_type='Host', + object_id=vuln.host.id + ) + session.commit() + res = test_client.get(self.url(workspace=command.workspace) + 'activity_feed/') + assert res.status_code == 200 + + assert filter(lambda stats: stats['command'] == command.id, res.json) == [{u'command': command.id, + u'sum_created_hosts': 1, + u'sum_created_services': 0, + u'sum_created_vulnerabilities': 1, + u'sum_created_vulnerabilities_web': 0, + u'sum_created_vulnerability_critical': 0}] + + assert filter(lambda stats: stats['command'] == another_command.id, res.json) == [{u'command': another_command.id, + u'sum_created_hosts': 0, + u'sum_created_services': 0, + u'sum_created_vulnerabilities': 0, + u'sum_created_vulnerabilities_web': 0, + u'sum_created_vulnerability_critical': 0}] + + + + def test_verify_created_critical_vulns_is_correctly_showing_sum_values(self, session, test_client): + workspace = WorkspaceFactory.create() + command = EmptyCommandFactory.create(workspace=workspace) + host = HostFactory.create(workspace=workspace) + vuln = VulnerabilityFactory.create(severity='critical', workspace=workspace, host=host, service=None) + vuln_med = VulnerabilityFactory.create(severity='medium', workspace=workspace, host=host, service=None) + session.flush() + CommandObjectFactory.create( + command=command, + object_type='Host', + object_id=host.id + ) + CommandObjectFactory.create( + command=command, + object_type='Vulnerability', + object_id=vuln.id + ) + CommandObjectFactory.create( + command=command, + object_type='Vulnerability', + object_id=vuln_med.id ) session.commit() res = test_client.get(self.url(workspace=command.workspace) + 'activity_feed/') assert res.status_code == 200 - assert filter(lambda stats: stats['command'] == command.id, res.json)[0]['sum_created_vulnerabilities'] == 1 - assert filter(lambda stats: stats['command'] == another_command.id, res.json)[0]['sum_created_vulnerabilities'] == 0 + assert res.json == [{u'command': command.id, + u'sum_created_hosts': 1, + u'sum_created_services': 0, + u'sum_created_vulnerabilities': 2, + u'sum_created_vulnerabilities_web': 0, + u'sum_created_vulnerability_critical': 1 + }] From e78564238eb397e723b02012c2a949045e079889 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 1 Nov 2017 18:03:11 -0300 Subject: [PATCH 0474/1506] Code cleanup --- server/models.py | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/server/models.py b/server/models.py index bb2531784cb..3d3628a7cfc 100644 --- a/server/models.py +++ b/server/models.py @@ -734,21 +734,10 @@ def parent(self): return self.host or self.service -def _make_command_created_related_object(object_type=None, joined=False, extra_where=None): - where_expr = "" - if extra_where: - where_expr += extra_where - - +def _make_command_created_related_object(): query = select([BooleanToIntColumn("(count(*) = 0)")]) - - if joined: - query = query.select_from('command_object as command_object_inner, {0}'.format(object_type.lower())) - where_expr += 'command_object_inner.object_id = {0}.id and '.format(object_type.lower()) - else: - query = query.select_from('command_object as command_object_inner') - - where_expr += " command_object_inner.create_date < command_object.create_date and " \ + query = query.select_from('command_object as command_object_inner') + where_expr = " command_object_inner.create_date < command_object.create_date and " \ " (command_object_inner.object_id = command_object.object_id and " \ " command_object_inner.object_type = command_object.object_type) " query = query.where(text(where_expr)) @@ -774,11 +763,6 @@ class CommandObject(db.Model): # remeber that this table has a row instances per relationship. # this created integer can be used to obtain the total object_type objects created. created = _make_command_created_related_object() - created_vulnerability_critical = _make_command_created_related_object( - 'Vulnerability', - joined=True, - extra_where="vulnerability.severity='critical' and " - ) def _make_created_objects_sum(object_type_filter): @@ -795,7 +779,7 @@ def _make_created_objects_sum_joined(object_type_filter, join_filters): for attr, filter_value in join_filters.items(): where_conditions.append("vulnerability.{0} = {1}".format(attr, filter_value)) return column_property( - select([func.sum(CommandObject.created_vulnerability_critical)]). \ + select([func.sum(CommandObject.created)]). \ select_from(table('command_object')). \ select_from(table('vulnerability')). \ where(' and '.join(where_conditions)) From 446a9cc25078edd35109bb73bf1e000cd328449f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 1 Nov 2017 18:32:54 -0300 Subject: [PATCH 0475/1506] Add more test cases --- test_cases/test_api_commands.py | 86 +++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 3 deletions(-) diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index 8c3270e83d5..738dc598b66 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -11,7 +11,7 @@ from server.api.modules.commandsrun import CommandView from server.api.modules.workspaces import WorkspaceView from test_cases.factories import VulnerabilityFactory, EmptyCommandFactory, CommandObjectFactory, HostFactory, \ - WorkspaceFactory + WorkspaceFactory, ServiceFactory @pytest.mark.usefixtures('logged_user') @@ -75,8 +75,6 @@ def test_activity_feed(self, session, test_client): u'sum_created_vulnerabilities_web': 0, u'sum_created_vulnerability_critical': 0}] - - def test_verify_created_critical_vulns_is_correctly_showing_sum_values(self, session, test_client): workspace = WorkspaceFactory.create() command = EmptyCommandFactory.create(workspace=workspace) @@ -109,3 +107,85 @@ def test_verify_created_critical_vulns_is_correctly_showing_sum_values(self, ses u'sum_created_vulnerabilities_web': 0, u'sum_created_vulnerability_critical': 1 }] + + def test_verify_created_vulns_with_host_and_service_verification(self, session, test_client): + workspace = WorkspaceFactory.create() + command = EmptyCommandFactory.create(workspace=workspace) + host = HostFactory.create(workspace=workspace) + service = ServiceFactory.create(workspace=workspace) + vuln = VulnerabilityFactory.create(severity='critical', workspace=workspace, host=host, service=None) + vuln_med = VulnerabilityFactory.create(severity='medium', workspace=workspace, service=service, host=None) + session.flush() + CommandObjectFactory.create( + command=command, + object_type='Host', + object_id=host.id + ) + CommandObjectFactory.create( + command=command, + object_type='Vulnerability', + object_id=vuln.id + ) + CommandObjectFactory.create( + command=command, + object_type='Service', + object_id=service.id + ) + CommandObjectFactory.create( + command=command, + object_type='Vulnerability', + object_id=vuln_med.id + ) + session.commit() + res = test_client.get(self.url(workspace=command.workspace) + 'activity_feed/') + assert res.status_code == 200 + assert res.json == [{u'command': command.id, + u'sum_created_hosts': 1, + u'sum_created_services': 1, + u'sum_created_vulnerabilities': 2, + u'sum_created_vulnerabilities_web': 0, + u'sum_created_vulnerability_critical': 1 + }] + + def test_multiple_commands_executed_with_same_objects_found(self, session, test_client): + + workspace = WorkspaceFactory.create() + command = EmptyCommandFactory.create(workspace=workspace) + service = ServiceFactory.create(workspace=workspace) + for _ in range(0, 10): + host = HostFactory.create(workspace=workspace) + vuln = VulnerabilityFactory.create(severity='low', workspace=workspace, host=host, service=None) + session.flush() + CommandObjectFactory.create( + command=command, + object_type='Host', + object_id=host.id + ) + CommandObjectFactory.create( + command=command, + object_type='Vulnerability', + object_id=vuln.id + ) + vuln_med = VulnerabilityFactory.create(severity='medium', workspace=workspace, service=service, host=None) + session.flush() + + CommandObjectFactory.create( + command=command, + object_type='Service', + object_id=service.id + ) + CommandObjectFactory.create( + command=command, + object_type='Vulnerability', + object_id=vuln_med.id + ) + session.commit() + res = test_client.get(self.url(workspace=command.workspace) + 'activity_feed/') + assert res.status_code == 200 + assert res.json == [{u'command': command.id, + u'sum_created_hosts': 10, + u'sum_created_services': 1, + u'sum_created_vulnerabilities': 11, + u'sum_created_vulnerabilities_web': 0, + u'sum_created_vulnerability_critical': 0 + }] From 7305d3780d55f295a372ebc26165e90d6cde24f2 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 1 Nov 2017 19:00:22 -0300 Subject: [PATCH 0476/1506] Make the unit test a little more interesting --- test_cases/test_api_commands.py | 40 ++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index 738dc598b66..b4be79ec188 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -150,11 +150,14 @@ def test_verify_created_vulns_with_host_and_service_verification(self, session, def test_multiple_commands_executed_with_same_objects_found(self, session, test_client): workspace = WorkspaceFactory.create() - command = EmptyCommandFactory.create(workspace=workspace) + host = HostFactory.create(workspace=workspace) + vuln = VulnerabilityFactory.create(severity='low', workspace=workspace, host=host, service=None) service = ServiceFactory.create(workspace=workspace) - for _ in range(0, 10): - host = HostFactory.create(workspace=workspace) - vuln = VulnerabilityFactory.create(severity='low', workspace=workspace, host=host, service=None) + commands = [] + for index in range(0, 10): + command = EmptyCommandFactory.create(workspace=workspace) + commands.append(command) + session.flush() CommandObjectFactory.create( command=command, @@ -168,7 +171,7 @@ def test_multiple_commands_executed_with_same_objects_found(self, session, test_ ) vuln_med = VulnerabilityFactory.create(severity='medium', workspace=workspace, service=service, host=None) session.flush() - + command = EmptyCommandFactory.create(workspace=workspace) CommandObjectFactory.create( command=command, object_type='Service', @@ -182,10 +185,27 @@ def test_multiple_commands_executed_with_same_objects_found(self, session, test_ session.commit() res = test_client.get(self.url(workspace=command.workspace) + 'activity_feed/') assert res.status_code == 200 - assert res.json == [{u'command': command.id, - u'sum_created_hosts': 10, - u'sum_created_services': 1, - u'sum_created_vulnerabilities': 11, + assert res.json[0] == {u'command': commands[0].id, + u'sum_created_hosts': 1, + u'sum_created_services': 0, + u'sum_created_vulnerabilities': 1, u'sum_created_vulnerabilities_web': 0, u'sum_created_vulnerability_critical': 0 - }] + } + for index in range(1, 10): + assert res.json[index] == {u'command': commands[index].id, + u'sum_created_hosts': 0, + u'sum_created_services': 0, + u'sum_created_vulnerabilities': 0, + u'sum_created_vulnerabilities_web': 0, + u'sum_created_vulnerability_critical': 0 + } + + # new command must create new service and vuln + assert res.json[10] == {u'command': command.id, + u'sum_created_hosts': 0, + u'sum_created_services': 1, + u'sum_created_vulnerabilities': 1, + u'sum_created_vulnerabilities_web': 0, + u'sum_created_vulnerability_critical': 0 + } From 0ef4100fb23236c8f3a01f474e306503b3466814 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 2 Nov 2017 17:46:27 -0300 Subject: [PATCH 0477/1506] Add evidence relationship to Vulnerability objects Also change the value of object_type in generic relationships. It is now the table name instead of the class name, to better support inherited models and future refactors. I changed some nullable options of the File model. --- server/api/modules/vulns.py | 5 ++-- server/models.py | 16 ++++++++--- test_cases/models/test_file.py | 52 ++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 7 deletions(-) create mode 100644 test_cases/models/test_file.py diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 901ed4a9664..2e9a8be5858 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -128,9 +128,8 @@ def get_metadata(self, obj): def get_attachments(self, obj): res = [] - files = db.session.query(File).filter_by(object_id=obj.id, object_type=obj.__class__.__name__).all() - for file_obj in files: + for file_obj in obj.evidence: ret, errors = EvidenceSchema().dump(file_obj) if errors: raise ValidationError(errors, data=ret) @@ -306,7 +305,7 @@ def _perform_create(self, data, **kwargs): db.session, File, object_id=obj.id, - object_type=obj.__class__.__name__, + object_type='vulnerability', name=os.path.splitext(os.path.basename(filename))[0], filename=os.path.basename(filename), content=faraday_file, diff --git a/server/models.py b/server/models.py index 3d3628a7cfc..ee12d2dd472 100644 --- a/server/models.py +++ b/server/models.py @@ -449,6 +449,13 @@ class VulnerabilityGeneric(VulnerabilityABC): proxy_factory=CustomAssociationSet, creator=_build_associationproxy_creator('PolicyViolation')) + evidence = relationship( + "File", + primaryjoin="and_(File.object_id==VulnerabilityGeneric.id, " + "File.object_type=='vulnerability')", + foreign_keys="File.object_id", + ) + __mapper_args__ = { 'polymorphic_on': type } @@ -991,10 +998,11 @@ class File(Metadata): __tablename__ = 'file' id = Column(Integer, autoincrement=True, primary_key=True) - name = Column(Text, unique=True) - filename = Column(Text, unique=True) - description = Column(Text, unique=True) - content = Column(UploadedFileField(upload_type=FaradayUploadedFile)) # plain attached file + name = Column(Text) + filename = Column(Text, nullable=False) + description = Column(Text) + content = Column(UploadedFileField(upload_type=FaradayUploadedFile), + nullable=False) # plain attached file object_id = Column(Integer, nullable=False) object_type = Column(Text, nullable=False) diff --git a/test_cases/models/test_file.py b/test_cases/models/test_file.py new file mode 100644 index 00000000000..ff6190a2691 --- /dev/null +++ b/test_cases/models/test_file.py @@ -0,0 +1,52 @@ +import os +import pytest +from server.models import File +from depot.manager import DepotManager + +CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) + + +@pytest.fixture +def depotfile(): + depot = DepotManager.get('default') + path = os.path.join(CURRENT_PATH, '../data', 'faraday.png') + with open(path) as fp: + fileid = depot.create(fp, 'faraday.png', 'image/png') + return fileid + + +@pytest.mark.usefixtures('app') # To load depot config +def test_get_vulnweb_evidence(vulnerability_web_factory, depotfile, session): + # Use vuln web to ensure its parent is a service and not a host + all_vulns = vulnerability_web_factory.create_batch(10) + session.commit() + vuln = all_vulns[0] + + correct_file = File(filename='faraday.png', object_id=vuln.id, + object_type='vulnerability', content=depotfile) + session.add(File(filename='faraday.png', + object_id=vuln.service_id, + object_type='service', content=depotfile)) + session.add(correct_file) + + for other_vuln in all_vulns[1:]: + session.add(File(filename='faraday.png', object_id=other_vuln.id, + object_type='vulnerability', content=depotfile)) + session.add(File(filename='faraday.png', + object_id=other_vuln.service_id, + object_type='service', content=depotfile)) + + session.commit() + assert vuln.evidence == [correct_file] + + +@pytest.mark.skip(reason='write to instrumentedlist not implemented') +@pytest.mark.usefixtures('app') # To load depot config +def test_add_vulnweb_evidence(vulnerability_web, depotfile, session): + session.commit() + file_ = File(filename='faraday.png', content=depotfile) + vulnerability_web.evidence.append(file_) + session.commit() + assert len(vulnerability_web.evidence) == 1 + assert vulnerability_web.evidence[0].object_type == 'vulnerability' + assert vulnerability_web.evidence[0].object_id == vulnerability_web.id From 03e5aa26f6eef17f654e633235b6d939e897faee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 2 Nov 2017 18:05:59 -0300 Subject: [PATCH 0478/1506] Fix n+1 issues on vuln evidence Before 0ef4100fb23236c8f3a01f474e306503b3466814 it wasn't detected by nplus one, after the merge it was --- server/api/modules/vulns.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index cdb0a47a949..0fec77c8093 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -327,6 +327,8 @@ def _get_eagerloaded_query(self, *args, **kwargs): joinedload(VulnerabilityWeb.service) .joinedload(Service.host) .joinedload(Host.hostnames), + + joinedload(VulnerabilityGeneric.evidence), ] return query.options(selectin_polymorphic( VulnerabilityGeneric, From 228dd1535fe4e10981c56151a302e65e79810658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 2 Nov 2017 18:53:30 -0300 Subject: [PATCH 0479/1506] Fix Comment model It was coded as it were a many to many, like tag. It should be like File but with Workspace embedded --- server/importer.py | 18 +++--------------- server/models.py | 18 +++++------------- 2 files changed, 8 insertions(+), 28 deletions(-) diff --git a/server/importer.py b/server/importer.py index 568dbae0c64..0575f43dacf 100644 --- a/server/importer.py +++ b/server/importer.py @@ -33,7 +33,6 @@ db, Command, Comment, - CommentObject, Credential, ExecutiveReport, Host, @@ -571,15 +570,9 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation session, Comment, text='{0}\n{1}'.format(document.get('text', ''), document.get('description', '')), - workspace=workspace) - - get_or_create( - session, - CommentObject, object_id=couchdb_relational_map[parent_document.get('_id')], object_type=parent_document['type'], - comment=comment, - ) + workspace=workspace) yield comment @@ -712,17 +705,12 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation session, Comment, text=document.get('text'), - workspace=workspace) - - get_or_create( - session, - CommentObject, object_id=workspace.id, object_type='Workspace', - comment=comment, - ) + workspace=workspace) yield comment + class FaradayEntityImporter(object): # Document Types: [u'Service', u'Communication', u'Vulnerability', u'CommandRunInformation', u'Reports', u'Host', u'Workspace'] diff --git a/server/models.py b/server/models.py index ee12d2dd472..03cf2811f17 100644 --- a/server/models.py +++ b/server/models.py @@ -1151,18 +1151,6 @@ class TagObject(db.Model): tag_id = Column(Integer, ForeignKey('tag.id'), index=True) -class CommentObject(db.Model): - __tablename__ = 'comment_object' - id = Column(Integer, primary_key=True) - - object_id = Column(Integer, nullable=False) - object_type = Column(Text, nullable=False) - - comment = relationship('Comment', backref='comment_objects') - comment_id = Column(Integer, ForeignKey('comment.id'), index=True) - - - class Comment(Metadata): __tablename__ = 'comment' id = Column(Integer, primary_key=True) @@ -1176,9 +1164,13 @@ class Comment(Metadata): foreign_keys=[reply_to_id] ) - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, + nullable=False) workspace = relationship('Workspace', foreign_keys=[workspace_id]) + object_id = Column(Integer, nullable=False) + object_type = Column(Text, nullable=False) + class ExecutiveReport(Metadata): STATUSES = [ From 022134b0e07972af4e29aca95db461491d001bd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 3 Nov 2017 15:51:18 -0300 Subject: [PATCH 0480/1506] Add generic tags relationship to Vulnerability objects --- server/models.py | 8 ++++++++ test_cases/factories.py | 10 ++++++++++ test_cases/models/test_tag.py | 26 ++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 test_cases/models/test_tag.py diff --git a/server/models.py b/server/models.py index 03cf2811f17..e3b72741deb 100644 --- a/server/models.py +++ b/server/models.py @@ -456,6 +456,14 @@ class VulnerabilityGeneric(VulnerabilityABC): foreign_keys="File.object_id", ) + tags = relationship( + "Tag", + secondary="tag_object", + primaryjoin="and_(TagObject.object_id==VulnerabilityGeneric.id, " + "TagObject.object_type=='vulnerability')", + collection_class=set, + ) + __mapper_args__ = { 'polymorphic_on': type } diff --git a/test_cases/factories.py b/test_cases/factories.py index 057374d361f..7cc7df61bb3 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -22,6 +22,7 @@ Reference, Service, SourceCode, + Tag, User, Vulnerability, VulnerabilityCode, @@ -325,3 +326,12 @@ def build_dict(cls, **kwargs): ret['start_date'] = ret['start_date'].isoformat() ret['end_date'] = ret['end_date'].isoformat() return ret + + +class TagFactory(FaradayFactory): + name = FuzzyText() + slug = FuzzyText() + + class Meta: + model = Tag + sqlalchemy_session = db.session diff --git a/test_cases/models/test_tag.py b/test_cases/models/test_tag.py new file mode 100644 index 00000000000..9c3dea4a8fd --- /dev/null +++ b/test_cases/models/test_tag.py @@ -0,0 +1,26 @@ +import pytest +from server.models import TagObject + + +def test_vulnweb_tags(vulnerability_web_factory, tag_factory, session): + # Use vuln web to ensure its parent is a service and not a host + all_vulns = vulnerability_web_factory.create_batch(10) + session.commit() + vuln = all_vulns[0] + + correct_tags = tag_factory.create_batch(3) + for tag in correct_tags: + session.add(TagObject(tag=tag, object_type='vulnerability', + object_id=vuln.id)) + session.add(TagObject(tag=tag_factory.create(), + object_type='service', + object_id=vuln.service_id)) + for other_vuln in all_vulns[5:]: + session.add(TagObject(tag=correct_tags[1], object_type='vulnerability', + object_id=other_vuln.id)) + session.add(TagObject(tag=tag_factory.create(), + object_type='vulnerability', + object_id=other_vuln.id)) + + session.commit() + assert vuln.tags == set(correct_tags) From ac7d1ccea23ec05461411d74440e4861c86caad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 3 Nov 2017 16:05:50 -0300 Subject: [PATCH 0481/1506] Fix n+1 issues on vuln tags I'm not sure why this wasn't undetected by nplusone and evidence was. Since now the vulnerability API does a constant number of queries instead of proportional with the number of vulns in the list --- server/api/modules/vulns.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 0fec77c8093..ec25fc6e9ea 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -87,7 +87,7 @@ class VulnerabilitySchema(AutoSchema): parent_type = MutableField(fields.Method('get_parent_type'), fields.String(), required=True) - tags = fields.Method(serialize='get_tags') + tags = PrimaryKeyRelatedField('name', dump_only=True, many=True) easeofresolution = fields.String(attribute='ease_of_resolution', validate=OneOf(Vulnerability.EASE_OF_RESOLUTIONS),) hostnames = PrimaryKeyRelatedField('name', many=True, dump_only=True) metadata = fields.Method(serialize='get_metadata') @@ -141,13 +141,6 @@ def get_attachments(self, obj): def load_attachments(self, value): return value - def get_tags(self, obj): - # TODO: improve performance here - return [tag.name for tag in db.session.query(TagObject, Tag).filter_by( - object_type=obj.__class__.__name__, - object_id=obj.id - ).all()] - def get_parent(self, obj): return obj.service_id or obj.host_id @@ -329,6 +322,7 @@ def _get_eagerloaded_query(self, *args, **kwargs): .joinedload(Host.hostnames), joinedload(VulnerabilityGeneric.evidence), + joinedload(VulnerabilityGeneric.tags), ] return query.options(selectin_polymorphic( VulnerabilityGeneric, From a96a59c7badf026d7cd1377ecfbc44932517ecb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 3 Nov 2017 16:35:20 -0300 Subject: [PATCH 0482/1506] Fix bug with pytest that was unexpectedly skipping some tests Because of a bug with pytest, I can't simply mark TestListCommandView with @pytest.mark.skip. I had to made it inherit from object instad of ReadOnlyAPITests, and to manually skip the extra tests inside the class. See https://docs.pytest.org/en/latest/skipping.html#skip-all-test-functions-of-a-class-or-module and https://github.com/pytest-dev/pytest/issues/568 for more information --- test_cases/test_api_commands.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index cddde81ab87..82a048ef560 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -14,14 +14,21 @@ WorkspaceFactory, ServiceFactory -@pytest.mark.skip(reason='refactor needed to adapt new m2m model') +# Note: because of a bug with pytest, I can't simply mark TestListCommandView +# with @pytest.mark.skip. I had to made it inherit from object instad of +# ReadOnlyAPITests, and to manually skip the extra tests inside the class. +# See https://docs.pytest.org/en/latest/skipping.html#skip-all-test-functions-of-a-class-or-module +# and https://github.com/pytest-dev/pytest/issues/568 for more information + @pytest.mark.usefixtures('logged_user') -class TestListCommandView(ReadOnlyAPITests): +# class TestListCommandView(ReadOnlyAPITests): # TODO: change to this!!! +class TestListCommandView(object): model = Command factory = factories.CommandFactory api_endpoint = 'commands' view_class = CommandView + @pytest.mark.skip(reason='refactor needed to adapt new m2m model') def test_backwards_compatibility_list(self, test_client, second_workspace, session): self.factory.create(workspace=second_workspace) session.commit() @@ -43,6 +50,7 @@ def test_backwards_compatibility_list(self, test_client, second_workspace, sessi ] assert set(object_properties) == set(command['value'].keys()) + @pytest.mark.skip(reason='refactor needed to adapt new m2m model') def test_activity_feed(self, session, test_client): command = self.factory.create() another_command = EmptyCommandFactory.create(workspace=command.workspace) @@ -76,6 +84,7 @@ def test_activity_feed(self, session, test_client): u'sum_created_vulnerabilities_web': 0, u'sum_created_vulnerability_critical': 0}] + @pytest.mark.skip(reason='refactor needed to adapt new m2m model') def test_verify_created_critical_vulns_is_correctly_showing_sum_values(self, session, test_client): workspace = WorkspaceFactory.create() command = EmptyCommandFactory.create(workspace=workspace) @@ -109,6 +118,7 @@ def test_verify_created_critical_vulns_is_correctly_showing_sum_values(self, ses u'sum_created_vulnerability_critical': 1 }] + @pytest.mark.skip(reason='refactor needed to adapt new m2m model') def test_verify_created_vulns_with_host_and_service_verification(self, session, test_client): workspace = WorkspaceFactory.create() command = EmptyCommandFactory.create(workspace=workspace) @@ -148,6 +158,7 @@ def test_verify_created_vulns_with_host_and_service_verification(self, session, u'sum_created_vulnerability_critical': 1 }] + @pytest.mark.skip(reason='refactor needed to adapt new m2m model') def test_multiple_commands_executed_with_same_objects_found(self, session, test_client): workspace = WorkspaceFactory.create() From 655397419a9c8a7aa72f41ec866ccd0fb6f59f63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 3 Nov 2017 17:06:05 -0300 Subject: [PATCH 0483/1506] Add ignore_nplusone test fixture to improve TDD process --- test_cases/conftest.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 093bd1c3d34..330283571bf 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -5,6 +5,7 @@ import pytest from factory import Factory from flask.testing import FlaskClient +from nplusone.core import signals from sqlalchemy import event from pytest_factoryboy import register @@ -199,12 +200,15 @@ def create_user(app, session, username, email, password, **kwargs): @pytest.fixture def user(app, database, session): # print 'user', id(session), session - return create_user(app, session, 'test', 'user@test.com', 'password', is_ldap=False) + return create_user(app, session, 'test', 'user@test.com', 'password', + is_ldap=False) @pytest.fixture def ldap_user(app, session): - return create_user(app, session, 'ldap', 'ldap@test.com', 'password', is_ldap=True) + return create_user(app, session, 'ldap', 'ldap@test.com', 'password', + is_ldap=True) + @pytest.fixture def host_with_hostnames(host, hostname_factory): @@ -219,7 +223,16 @@ def login_as(test_client, user): db.session.add(user) sess['user_id'] = user.id + @pytest.fixture def logged_user(test_client, user): login_as(test_client, user) return user + + +@pytest.fixture +def ignore_nplusone(app): + old = app.config['NPLUSONE_RAISE'] + app.config['NPLUSONE_RAISE'] = False + yield + app.config['NPLUSONE_RAISE'] = old From 5b41950141a540516c61a1e95388e8773a659f1f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 2 Nov 2017 16:11:27 -0300 Subject: [PATCH 0484/1506] Update web ui to use the new endpoint activity feed. update unit test also --- server/api/modules/commandsrun.py | 16 ++- server/importer.py | 2 +- server/models.py | 1 - .../www/scripts/commons/providers/server.js | 5 + .../dashboard/controllers/activityFeedCtrl.js | 66 +--------- .../scripts/dashboard/providers/dashboard.js | 12 ++ test_cases/test_api_commands.py | 115 +++++++++++------- 7 files changed, 97 insertions(+), 120 deletions(-) diff --git a/server/api/modules/commandsrun.py b/server/api/modules/commandsrun.py index 25fdbf5ea68..89b19712c00 100644 --- a/server/api/modules/commandsrun.py +++ b/server/api/modules/commandsrun.py @@ -65,12 +65,16 @@ def activity_feed(self, workspace_name): query = Command.query.join(Workspace).filter_by(name=workspace_name) for command in query.all(): res.append({ - 'command': command.id, - 'sum_created_vulnerabilities': command.sum_created_vulnerabilities or 0, - 'sum_created_vulnerabilities_web': command.sum_created_vulnerabilities_web or 0, - 'sum_created_hosts': command.sum_created_hosts or 0, - 'sum_created_services': command.sum_created_services or 0, - 'sum_created_vulnerability_critical': command.sum_created_vulnerability_critical or 0 + '_id': command.id, + 'command_type': '', + 'command': command.command, + 'params': command.params, + 'vulnerabilities_count': (command.sum_created_vulnerabilities or 0) + (command.sum_created_vulnerabilities_web or 0), + 'hosts_count': command.sum_created_hosts or 0, + 'services_count': command.sum_created_services or 0, + 'criticalIssue': command.sum_created_vulnerability_critical or 0, + 'date': time.mktime(command.start_date.timetuple()) * 1000, + }) return res diff --git a/server/importer.py b/server/importer.py index 0575f43dacf..a47ec3280ba 100644 --- a/server/importer.py +++ b/server/importer.py @@ -79,10 +79,10 @@ ]) OBJ_TYPES = [ + (1, 'CommandRunInformation'), (1, 'Host'), (1, 'EntityMetadata'), (1, 'Note'), - (1, 'CommandRunInformation'), (1, 'TaskGroup'), (1, 'Task'), (1, 'Workspace'), diff --git a/server/models.py b/server/models.py index 72acf883bf9..9a07d25e596 100644 --- a/server/models.py +++ b/server/models.py @@ -781,7 +781,6 @@ class CommandObject(db.Model): command_id = Column(Integer, ForeignKey('command.id'), index=True) create_date = Column(DateTime, default=datetime.utcnow) - update_date = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) # the following properties are used to know if the command created the specified objects_type # remeber that this table has a row instances per relationship. diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index 977d839e373..3b8b6c210b4 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -271,6 +271,11 @@ angular.module("faradayApp") return get(getUrl, data); } + ServerAPI.getActivityFeed = function(wsName, data) { + var getUrl = createGetUrl(wsName, 'commands') + 'activity_feed/'; + return get(getUrl, data); + } + ServerAPI.getWorkspacesNames = function() { return get(APIURL + "ws/"); } diff --git a/server/www/scripts/dashboard/controllers/activityFeedCtrl.js b/server/www/scripts/dashboard/controllers/activityFeedCtrl.js index 13b0be75307..b3a1873c0d8 100644 --- a/server/www/scripts/dashboard/controllers/activityFeedCtrl.js +++ b/server/www/scripts/dashboard/controllers/activityFeedCtrl.js @@ -11,79 +11,15 @@ angular.module('faradayApp') var vm = this; vm.commands = []; - - // Get a count of all hosts created by this command. - vm.getHostCount = function(command){ - dashboardSrv.getHostsCountByCommandId($scope.workspace, command._id) - .then(function(hosts) { - - if( !isNaN(hosts['total_rows']) ) - command.hosts_count = hosts['total_rows']; - else - command.hosts_count = 0; - }); - }; - - // Get a count of all services created by this command. - vm.getServiceCount = function(command){ - dashboardSrv.getServicesByCommandId($scope.workspace, command._id) - .then(function(services) { - if( services.services.length != 0 ) - command.services_count = services.services.length; - else - command.services_count = 0; - }); - }; - - // Get a count of all vulns created by this command. - vm.getVulnsCount = function(command){ - dashboardSrv.getVulnsByCommandId($scope.workspace, command._id) - .then(function(vulnerabilities) { - vm.checkCriticalIssue(vulnerabilities, command); - - if(!isNaN(vulnerabilities.count)) - command.vulnerabilities_count = vulnerabilities.count; - else - command.vulnerabilities_count = 0; - }); - }; - - //Check if is a command or Import report. - vm.setCommandType = function(command){ - - if(command.command.indexOf('Import') >= 0) - command.command_type = 'import'; - else - command.command_type = 'command'; - }; - - vm.checkCriticalIssue = function(vulnerabilities, command){ - command.criticalIssue = 0; - - vulnerabilities.vulnerabilities.forEach(function(vuln){ - if(vuln.value.severity == 'critical'){ - command.criticalIssue += 1; - } - }); - }; - // Get last 5 commands var init = function() { if($routeParams.wsId != undefined) { $scope.workspace = $routeParams.wsId; - dashboardSrv.getCommands($scope.workspace) + dashboardSrv.getActivityFeed($scope.workspace) .then(function(commands) { - vm.commands = commands; - vm.commands.forEach(function(command){ - - vm.getHostCount(command); - vm.getServiceCount(command); - vm.getVulnsCount(command); - vm.setCommandType(command); - }); }); } }; diff --git a/server/www/scripts/dashboard/providers/dashboard.js b/server/www/scripts/dashboard/providers/dashboard.js index 6c2741dab5f..84e65dc1d4f 100644 --- a/server/www/scripts/dashboard/providers/dashboard.js +++ b/server/www/scripts/dashboard/providers/dashboard.js @@ -228,6 +228,18 @@ angular.module('faradayApp') return deferred.promise; }; + dashboardSrv.getActivityFeed = function(ws) { + var deferred = $q.defer(); + + ServerAPI.getActivityFeed(ws).then(function(res) { + deferred.resolve(res.data); + }, function() { + deferred.reject(); + }); + + return deferred.promise; + }; + dashboardSrv.getCommands = function(ws) { var deferred = $q.defer(); diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index 82a048ef560..480b473a01c 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -70,19 +70,26 @@ def test_activity_feed(self, session, test_client): res = test_client.get(self.url(workspace=command.workspace) + 'activity_feed/') assert res.status_code == 200 - assert filter(lambda stats: stats['command'] == command.id, res.json) == [{u'command': command.id, - u'sum_created_hosts': 1, - u'sum_created_services': 0, - u'sum_created_vulnerabilities': 1, - u'sum_created_vulnerabilities_web': 0, - u'sum_created_vulnerability_critical': 0}] + assert filter(lambda stats: stats['_id'] == command.id, res.json) == [{u'_id': command.id, + u'command': command.command, + u'command_type': '', + u'date': time.mktime(command.start_date.timetuple()) * 1000, + u'params': command.params, + u'hosts_count': 1, + u'services_count': 0, + u'vulnerabilities_count': 1, + u'criticalIssue': 0}] - assert filter(lambda stats: stats['command'] == another_command.id, res.json) == [{u'command': another_command.id, - u'sum_created_hosts': 0, - u'sum_created_services': 0, - u'sum_created_vulnerabilities': 0, - u'sum_created_vulnerabilities_web': 0, - u'sum_created_vulnerability_critical': 0}] + assert filter(lambda stats: stats['_id'] == another_command.id, res.json) == [{ + u'_id': another_command.id, + u'command': another_command.command, + u'command_type': '', + u'date': time.mktime(another_command.start_date.timetuple()) * 1000, + u'params': another_command.params, + u'hosts_count': 0, + u'services_count': 0, + u'vulnerabilities_count': 0, + u'criticalIssue': 0}] @pytest.mark.skip(reason='refactor needed to adapt new m2m model') def test_verify_created_critical_vulns_is_correctly_showing_sum_values(self, session, test_client): @@ -110,13 +117,17 @@ def test_verify_created_critical_vulns_is_correctly_showing_sum_values(self, ses session.commit() res = test_client.get(self.url(workspace=command.workspace) + 'activity_feed/') assert res.status_code == 200 - assert res.json == [{u'command': command.id, - u'sum_created_hosts': 1, - u'sum_created_services': 0, - u'sum_created_vulnerabilities': 2, - u'sum_created_vulnerabilities_web': 0, - u'sum_created_vulnerability_critical': 1 - }] + assert res.json == [ + {u'_id': command.id, + u'command': command.command, + u'command_type': '', + u'date': time.mktime(command.start_date.timetuple()) * 1000, + u'params': command.params, + u'hosts_count': 1, + u'services_count': 0, + u'vulnerabilities_count': 2, + u'criticalIssue': 1} + ] @pytest.mark.skip(reason='refactor needed to adapt new m2m model') def test_verify_created_vulns_with_host_and_service_verification(self, session, test_client): @@ -150,13 +161,16 @@ def test_verify_created_vulns_with_host_and_service_verification(self, session, session.commit() res = test_client.get(self.url(workspace=command.workspace) + 'activity_feed/') assert res.status_code == 200 - assert res.json == [{u'command': command.id, - u'sum_created_hosts': 1, - u'sum_created_services': 1, - u'sum_created_vulnerabilities': 2, - u'sum_created_vulnerabilities_web': 0, - u'sum_created_vulnerability_critical': 1 - }] + assert res.json == [{u'_id': command.id, + u'command': command.command, + u'command_type': '', + u'date': time.mktime(command.start_date.timetuple()) * 1000, + u'params': command.params, + u'hosts_count': 1, + u'services_count': 1, + u'vulnerabilities_count': 2, + u'criticalIssue': 1} + ] @pytest.mark.skip(reason='refactor needed to adapt new m2m model') def test_multiple_commands_executed_with_same_objects_found(self, session, test_client): @@ -197,27 +211,34 @@ def test_multiple_commands_executed_with_same_objects_found(self, session, test_ session.commit() res = test_client.get(self.url(workspace=command.workspace) + 'activity_feed/') assert res.status_code == 200 - assert res.json[0] == {u'command': commands[0].id, - u'sum_created_hosts': 1, - u'sum_created_services': 0, - u'sum_created_vulnerabilities': 1, - u'sum_created_vulnerabilities_web': 0, - u'sum_created_vulnerability_critical': 0 - } + assert res.json[0] == {u'_id': commands[0].id, + u'command': commands[0].command, + u'command_type': '', + u'date': time.mktime(commands[0].start_date.timetuple()) * 1000, + u'params': commands[0].params, + u'hosts_count': 1, + u'services_count': 0, + u'vulnerabilities_count': 1, + u'criticalIssue': 0} + for index in range(1, 10): - assert res.json[index] == {u'command': commands[index].id, - u'sum_created_hosts': 0, - u'sum_created_services': 0, - u'sum_created_vulnerabilities': 0, - u'sum_created_vulnerabilities_web': 0, - u'sum_created_vulnerability_critical': 0 - } + assert res.json[index] == {u'_id': commands[index].id, + u'command': commands[index].command, + u'command_type': '', + u'date': time.mktime(commands[index].start_date.timetuple()) * 1000, + u'params': commands[index].params, + u'hosts_count': 0, + u'services_count': 0, + u'vulnerabilities_count': 0, + u'criticalIssue': 0} # new command must create new service and vuln - assert res.json[10] == {u'command': command.id, - u'sum_created_hosts': 0, - u'sum_created_services': 1, - u'sum_created_vulnerabilities': 1, - u'sum_created_vulnerabilities_web': 0, - u'sum_created_vulnerability_critical': 0 - } + assert res.json[10] == {u'_id': command.id, + u'command': command.command, + u'command_type': '', + u'date': time.mktime(command.start_date.timetuple()) * 1000, + u'params': command.params, + u'hosts_count': 0, + u'services_count': 1, + u'vulnerabilities_count': 1, + u'criticalIssue': 0} From 2b32acf5e674926e0ec4a48fc47e138c87e5b56a Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 2 Nov 2017 20:08:14 -0300 Subject: [PATCH 0485/1506] Add more vuln status (extracted from the app.js). Log a warning msg when status can not be mapped --- server/importer.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index a47ec3280ba..7566f70ba7b 100644 --- a/server/importer.py +++ b/server/importer.py @@ -391,6 +391,7 @@ class VulnerabilityImporter(object): DOC_TYPE = ['Vulnerability', 'VulnerabilityWeb'] def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): + vulnerability = None couch_parent_id = document.get('parent', None) if not couch_parent_id: couch_parent_id = '.'.join(document['_id'].split('.')[:-1]) @@ -467,8 +468,14 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation 'opened': 'open', 'open': 'open', 'closed': 'closed', + 're-opened': 're-opened', + 'risk-accepted': 'risk-accepted', } - status = status_map[document.get('status', 'opened')] + try: + status = status_map[document.get('status', 'opened')] + except KeyError: + logger.warn('Could not map vulnerability status {0}'.format(document['status'])) + continue vulnerability.status = status vulnerability.reference_instances.update( From 0895e7657b5338012d3138205e8b8d79aa7944fb Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 2 Nov 2017 20:08:53 -0300 Subject: [PATCH 0486/1506] Add CommandObject to Host importer --- server/importer.py | 16 ++++++++++++---- server/models.py | 17 +++++++++++++++++ test_cases/test_api_commands.py | 2 ++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/server/importer.py b/server/importer.py index 7566f70ba7b..f5ca66dc9c1 100644 --- a/server/importer.py +++ b/server/importer.py @@ -55,6 +55,7 @@ VulnerabilityWeb, Workspace, File, + log_command_object_found, ) from server.utils import invalid_chars from server.utils.database import get_or_create @@ -268,6 +269,10 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation hosts = [] host_ips = [name_or_ip for name_or_ip in self.retrieve_ips_from_host_document(document)] interfaces = get_children_from_couch(workspace, document.get('_id'), 'Interface') + command = None + if 'command_id' in document['metadata'] and document['metadata']['command_id']: + command = session.query(Command).get(couchdb_relational_map[document['metadata']['command_id']][0]) + for interface in interfaces: interface = interface['value'] if check_ip_address(interface['ipv4']['address']): @@ -275,27 +280,30 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation host, created = get_or_create(session, Host, ip=interface_ip, workspace=workspace) host.default_gateway_ip = interface['ipv4']['gateway'] self.merge_with_host(host, interface, workspace) - hosts.append(host) + hosts.append((host, created)) if check_ip_address(interface['ipv6']['address']): interface_ip = interface['ipv6']['address'] host, created = get_or_create(session, Host, ip=interface_ip, workspace=workspace) host.default_gateway_ip = interface['ipv6']['gateway'] self.merge_with_host(host, interface, workspace) - hosts.append(host) + hosts.append((host, created)) if not hosts: # if not host were created after inspecting interfaces # we create a host with "name" as ip to avoid losing hosts. # some hosts lacks of interface for name_or_ip in host_ips: host, created = get_or_create(session, Host, ip=name_or_ip, workspace=workspace) - hosts.append(host) + hosts.append((host, created)) if len(hosts) > 1: logger.warning('Total hosts found {0} for couchdb id {1}'.format(len(hosts), document.get('_id'))) - for host in hosts: + for host, created in hosts: # we update or set other host attributes in this cycle # Ticket #3387: if the 'os' field is None, we default to 'unknown + if command: + log_command_object_found(command, host, created) + if not document.get('os'): document['os'] = 'unknown' diff --git a/server/models.py b/server/models.py index 9a07d25e596..60d91c56caf 100644 --- a/server/models.py +++ b/server/models.py @@ -787,6 +787,11 @@ class CommandObject(db.Model): # this created integer can be used to obtain the total object_type objects created. created = _make_command_created_related_object() + # We are currently using the column property created. however to avoid losing information + # we also store the a boolean to know if at the moment of created the object related to the + # Command was created. + created_persistent = Column(Boolean, default=False) + def _make_created_objects_sum(object_type_filter): where_condition = "command_object.command_id = command.id and command_object.object_type= '%s'" % object_type_filter @@ -1236,3 +1241,15 @@ class ExecutiveReport(Metadata): 'after_create', vulnerability_uniqueness.execute_if(dialect='postgresql') ) + + +def log_command_object_found(command, object, created): + db.session.flush() + get_or_create( + db.session, + CommandObject, + command=command, + object_id=object.id, + object_type=object.__tablename__, + created_persistent=created, + ) diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index 480b473a01c..c1e08acbf17 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -2,6 +2,8 @@ """Tests for many API endpoints that do not depend on workspace_name""" import pytest +import time + from test_cases import factories from test_api_workspaced_base import API_PREFIX, ReadOnlyAPITests from server.models import ( From be3ac24c705447599c4f45f21ab0d9417d906de3 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 3 Nov 2017 11:06:23 -0300 Subject: [PATCH 0487/1506] Add command object logger to importer --- server/importer.py | 21 +++++++++++++++++++-- server/models.py | 14 +++++++++----- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/server/importer.py b/server/importer.py index f5ca66dc9c1..c2d19847649 100644 --- a/server/importer.py +++ b/server/importer.py @@ -270,7 +270,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation host_ips = [name_or_ip for name_or_ip in self.retrieve_ips_from_host_document(document)] interfaces = get_children_from_couch(workspace, document.get('_id'), 'Interface') command = None - if 'command_id' in document['metadata'] and document['metadata']['command_id']: + if 'metadata' in document and 'command_id' in document['metadata'] and document['metadata']['command_id']: command = session.query(Command).get(couchdb_relational_map[document['metadata']['command_id']][0]) for interface in interfaces: @@ -351,6 +351,9 @@ class ServiceImporter(object): def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): # service was always below interface, not it's below host. + command = None + if 'command_id' in document['metadata'] and document['metadata']['command_id']: + command = session.query(Command).get(couchdb_relational_map[document['metadata']['command_id']][0]) try: parent_id = document['parent'].split('.')[0] except KeyError: @@ -358,6 +361,8 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation parent_id = document['_id'].split('.')[0] for relational_parent_id in couchdb_relational_map[parent_id]: host, created = get_or_create(session, Host, id=relational_parent_id) + if command: + log_command_object_found(command, host, created) ports = document.get('ports') if len(ports) > 2: logger.warn('More than one port found in services!') @@ -391,6 +396,8 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation service.status = status_mapper.get(couchdb_status, 'open') service.version = document.get('version') service.workspace = workspace + if command: + log_command_object_found(command, service, created) yield service @@ -399,6 +406,9 @@ class VulnerabilityImporter(object): DOC_TYPE = ['Vulnerability', 'VulnerabilityWeb'] def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): + command = None + if 'command_id' in document['metadata'] and document['metadata']['command_id']: + command = session.query(Command).get(couchdb_relational_map[document['metadata']['command_id']][0]) vulnerability = None couch_parent_id = document.get('parent', None) if not couch_parent_id: @@ -435,6 +445,8 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation website=website, workspace=workspace, ) + if command: + log_command_object_found(command, vulnerability, created) if document['type'] == 'Vulnerability': vuln_params = { 'name': document.get('name'), @@ -492,7 +504,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation self.add_policy_violations(document, vulnerability, workspace)) # need the vuln ID before creating Tags for it - session.commit() + session.flush() tags = document.get('tags', []) if len(tags): create_tags(tags, vulnerability.id, document['type']) @@ -595,6 +607,9 @@ class CredentialImporter(object): DOC_TYPE = 'Cred' def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): + command = None + if 'command_id' in document['metadata'] and document['metadata']['command_id']: + command = session.query(Command).get(couchdb_relational_map[document['metadata']['command_id']][0]) parents = [] if level == 2: parent_ids = couchdb_relational_map[document['_id'].split('.')[0]] @@ -612,6 +627,8 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation if isinstance(parent, Service): service = parent credential, created = get_or_create(session, Credential, username=document.get('username'), host=host, service=service) + if command: + log_command_object_found(command, credential, created) credential.password = document.get('password', None) credential.owned = document.get('owned', False) credential.description = document.get('description', None) diff --git a/server/models.py b/server/models.py index 60d91c56caf..db2759861f2 100644 --- a/server/models.py +++ b/server/models.py @@ -554,10 +554,10 @@ class VulnerabilityCode(VulnerabilityGeneric): source_code_id = Column(Integer, ForeignKey(SourceCode.id), index=True) source_code = relationship( - SourceCode, - backref='vulnerabilities', - foreign_keys=[source_code_id] - ) + SourceCode, + backref='vulnerabilities', + foreign_keys=[source_code_id] + ) __mapper_args__ = { 'polymorphic_identity': VulnerabilityGeneric.VULN_TYPES[2] @@ -1244,12 +1244,16 @@ class ExecutiveReport(Metadata): def log_command_object_found(command, object, created): + object_type = object.__tablename__ + if object.__class__.__name__ in ['Vulnerability', 'VulnerabilityWeb', 'VulnerabilityCode']: + object_type = 'vulnerability' + db.session.flush() get_or_create( db.session, CommandObject, command=command, object_id=object.id, - object_type=object.__tablename__, + object_type=object_type, created_persistent=created, ) From c1e87cbf26c0836ca06ccf691c5178410828408b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 3 Nov 2017 20:05:13 -0300 Subject: [PATCH 0488/1506] Fix some importer bugs. Update File unique constraint --- server/importer.py | 30 +++++++++++++++++++++++------- server/models.py | 4 ++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/server/importer.py b/server/importer.py index c2d19847649..a6ca6770f95 100644 --- a/server/importer.py +++ b/server/importer.py @@ -531,6 +531,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation filename=attachment_name, object_id=vulnerability.id, object_type=vulnerability.__class__.__name__) + session.flush() file.content = attachment_file.read() attachment_file.close() @@ -539,27 +540,35 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation yield vulnerability def add_policy_violations(self, document, vulnerability, workspace): - policy_violations = list() + policy_violations = set() for policy_violation in document.get('policyviolations', []): - pv, _ = get_or_create( + if not policy_violation: + continue + pv, created = get_or_create( session, PolicyViolation, name=policy_violation, workspace=workspace ) - policy_violations.append(pv) + session.flush() + if created and pv.name not in map(lambda pva: pva.name, policy_violations): + policy_violations.add(pv) return policy_violations def add_references(self, document, vulnerability, workspace): - references = list() + references = set() for ref in document.get('refs', []): - reference, _ = get_or_create( + if not ref: + continue + reference, created = get_or_create( session, Reference, name=ref, workspace=workspace ) - references.append(reference) + session.flush() + if created and reference not in map(lambda ref: ref.name, references): + references.add(reference) return references @@ -626,7 +635,14 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation host = parent if isinstance(parent, Service): service = parent - credential, created = get_or_create(session, Credential, username=document.get('username'), host=host, service=service) + credential, created = get_or_create( + session, + Credential, + username=document.get('username'), + host=host, + service=service, + workspace=workspace, + ) if command: log_command_object_found(command, credential, created) credential.password = document.get('password', None) diff --git a/server/models.py b/server/models.py index db2759861f2..97f7d234f76 100644 --- a/server/models.py +++ b/server/models.py @@ -1028,6 +1028,10 @@ class File(Metadata): object_id = Column(Integer, nullable=False) object_type = Column(Text, nullable=False) + __table_args__ = ( + UniqueConstraint('object_id', 'object_type', 'filename', name='uix_file_obj_id_type_and_filename'), + ) + class UserAvatar(Metadata): __tablename_ = 'user_avatar' From 2f85ceabae70c49fb8fb83d4eb1e102498bf01ba Mon Sep 17 00:00:00 2001 From: Mike Zhong Date: Mon, 6 Nov 2017 12:14:08 -0300 Subject: [PATCH 0489/1506] added logger to record errors --- plugins/repo/dig/plugin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/repo/dig/plugin.py b/plugins/repo/dig/plugin.py index d2ac1ecace1..79dcba0f352 100644 --- a/plugins/repo/dig/plugin.py +++ b/plugins/repo/dig/plugin.py @@ -156,8 +156,9 @@ def parseOutputString(self, output): name="TXT Information", text=text.encode('ascii', 'ignore')) - except: + except Exception as ex: print("some part of the dig plug-in caused an error! Please check repo/dig/plugin.py") + logger.error("Error from dig plugin: %s" % (ex)) return False From e85532cb550ab1051e32c510fd92d80931dbbbcc Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 6 Nov 2017 14:43:32 -0300 Subject: [PATCH 0490/1506] Use table name and not object class name in the object_type field --- server/importer.py | 7 +++--- server/models.py | 16 +++++------- test_cases/factories.py | 4 +-- test_cases/test_api_commands.py | 35 +++++++++++--------------- test_cases/test_api_workspaced_base.py | 1 + 5 files changed, 28 insertions(+), 35 deletions(-) diff --git a/server/importer.py b/server/importer.py index a6ca6770f95..33bc6f407f8 100644 --- a/server/importer.py +++ b/server/importer.py @@ -639,18 +639,19 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation session, Credential, username=document.get('username'), + password=document.get('password'), host=host, service=service, workspace=workspace, ) - if command: - log_command_object_found(command, credential, created) credential.password = document.get('password', None) credential.owned = document.get('owned', False) credential.description = document.get('description', None) credential.name = document.get('name', None) credential.workspace = workspace - yield credential + if command: + log_command_object_found(command, credential, created) + yield credential class WorkspaceImporter(object): diff --git a/server/models.py b/server/models.py index 97f7d234f76..66cf5c2e88f 100644 --- a/server/models.py +++ b/server/models.py @@ -829,15 +829,15 @@ class Command(Metadata): workspace = relationship('Workspace', foreign_keys=[workspace_id]) # TODO: add Tool relationship and report_attachment - sum_created_vulnerabilities = _make_created_objects_sum('Vulnerability') + sum_created_vulnerabilities = _make_created_objects_sum('vulnerability') - sum_created_vulnerabilities_web = _make_created_objects_sum('VulnerabilityWeb') + sum_created_vulnerabilities_web = _make_created_objects_sum_joined('vulnerability', {'type': '\'vulnerability_web\''}) - sum_created_hosts = _make_created_objects_sum('Host') + sum_created_hosts = _make_created_objects_sum('host') - sum_created_services = _make_created_objects_sum('Service') + sum_created_services = _make_created_objects_sum('service') - sum_created_vulnerability_critical = _make_created_objects_sum_joined('Vulnerability', {'severity': '\'critical\''}) + sum_created_vulnerability_critical = _make_created_objects_sum_joined('vulnerability', {'severity': '\'critical\''}) @property def parent(self): @@ -1024,14 +1024,10 @@ class File(Metadata): filename = Column(Text, nullable=False) description = Column(Text) content = Column(UploadedFileField(upload_type=FaradayUploadedFile), - nullable=False) # plain attached file + nullable=True) # plain attached file object_id = Column(Integer, nullable=False) object_type = Column(Text, nullable=False) - __table_args__ = ( - UniqueConstraint('object_id', 'object_type', 'filename', name='uix_file_obj_id_type_and_filename'), - ) - class UserAvatar(Metadata): __tablename_ = 'user_avatar' diff --git a/test_cases/factories.py b/test_cases/factories.py index 7cc7df61bb3..ddbbc308f03 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -285,12 +285,12 @@ def attach_vuln_object(self, create, extracted, **kwargs): vuln = VulnerabilityFactory.create(workspace=self.workspace, host=host, service=None, severity='low') db.session.flush() CommandObjectFactory.create( - object_type='Vulnerability', + object_type='vulnerability', object_id=vuln.id, command=self, ) CommandObjectFactory.create( - object_type='Host', + object_type='host', object_id=host.id, command=self, ) diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index c1e08acbf17..e4e5a06c6aa 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -23,14 +23,13 @@ # and https://github.com/pytest-dev/pytest/issues/568 for more information @pytest.mark.usefixtures('logged_user') -# class TestListCommandView(ReadOnlyAPITests): # TODO: change to this!!! -class TestListCommandView(object): +class TestListCommandView(ReadOnlyAPITests): model = Command factory = factories.CommandFactory api_endpoint = 'commands' view_class = CommandView - @pytest.mark.skip(reason='refactor needed to adapt new m2m model') + @pytest.mark.usefixtures('ignore_nplusone') def test_backwards_compatibility_list(self, test_client, second_workspace, session): self.factory.create(workspace=second_workspace) session.commit() @@ -52,7 +51,6 @@ def test_backwards_compatibility_list(self, test_client, second_workspace, sessi ] assert set(object_properties) == set(command['value'].keys()) - @pytest.mark.skip(reason='refactor needed to adapt new m2m model') def test_activity_feed(self, session, test_client): command = self.factory.create() another_command = EmptyCommandFactory.create(workspace=command.workspace) @@ -60,12 +58,12 @@ def test_activity_feed(self, session, test_client): session.flush() CommandObjectFactory.create( command=another_command, - object_type='Vulnerability', + object_type='vulnerability', object_id=vuln.id ) CommandObjectFactory.create( command=another_command, - object_type='Host', + object_type='host', object_id=vuln.host.id ) session.commit() @@ -93,7 +91,6 @@ def test_activity_feed(self, session, test_client): u'vulnerabilities_count': 0, u'criticalIssue': 0}] - @pytest.mark.skip(reason='refactor needed to adapt new m2m model') def test_verify_created_critical_vulns_is_correctly_showing_sum_values(self, session, test_client): workspace = WorkspaceFactory.create() command = EmptyCommandFactory.create(workspace=workspace) @@ -103,17 +100,17 @@ def test_verify_created_critical_vulns_is_correctly_showing_sum_values(self, ses session.flush() CommandObjectFactory.create( command=command, - object_type='Host', + object_type='host', object_id=host.id ) CommandObjectFactory.create( command=command, - object_type='Vulnerability', + object_type='vulnerability', object_id=vuln.id ) CommandObjectFactory.create( command=command, - object_type='Vulnerability', + object_type='vulnerability', object_id=vuln_med.id ) session.commit() @@ -131,7 +128,6 @@ def test_verify_created_critical_vulns_is_correctly_showing_sum_values(self, ses u'criticalIssue': 1} ] - @pytest.mark.skip(reason='refactor needed to adapt new m2m model') def test_verify_created_vulns_with_host_and_service_verification(self, session, test_client): workspace = WorkspaceFactory.create() command = EmptyCommandFactory.create(workspace=workspace) @@ -142,22 +138,22 @@ def test_verify_created_vulns_with_host_and_service_verification(self, session, session.flush() CommandObjectFactory.create( command=command, - object_type='Host', + object_type='host', object_id=host.id ) CommandObjectFactory.create( command=command, - object_type='Vulnerability', + object_type='vulnerability', object_id=vuln.id ) CommandObjectFactory.create( command=command, - object_type='Service', + object_type='service', object_id=service.id ) CommandObjectFactory.create( command=command, - object_type='Vulnerability', + object_type='vulnerability', object_id=vuln_med.id ) session.commit() @@ -174,7 +170,6 @@ def test_verify_created_vulns_with_host_and_service_verification(self, session, u'criticalIssue': 1} ] - @pytest.mark.skip(reason='refactor needed to adapt new m2m model') def test_multiple_commands_executed_with_same_objects_found(self, session, test_client): workspace = WorkspaceFactory.create() @@ -189,12 +184,12 @@ def test_multiple_commands_executed_with_same_objects_found(self, session, test_ session.flush() CommandObjectFactory.create( command=command, - object_type='Host', + object_type='host', object_id=host.id ) CommandObjectFactory.create( command=command, - object_type='Vulnerability', + object_type='vulnerability', object_id=vuln.id ) vuln_med = VulnerabilityFactory.create(severity='medium', workspace=workspace, service=service, host=None) @@ -202,12 +197,12 @@ def test_multiple_commands_executed_with_same_objects_found(self, session, test_ command = EmptyCommandFactory.create(workspace=workspace) CommandObjectFactory.create( command=command, - object_type='Service', + object_type='service', object_id=service.id ) CommandObjectFactory.create( command=command, - object_type='Vulnerability', + object_type='vulnerability', object_id=vuln_med.id ) session.commit() diff --git a/test_cases/test_api_workspaced_base.py b/test_cases/test_api_workspaced_base.py index a244ec227d7..048528a1f9d 100644 --- a/test_cases/test_api_workspaced_base.py +++ b/test_cases/test_api_workspaced_base.py @@ -62,6 +62,7 @@ def _envelope_list(self, objects, pagination_metadata=None): monkeypatch.setattr(self.view_class, '_envelope_list', _envelope_list) @pytest.mark.usefixtures('mock_envelope_list') + @pytest.mark.usefixtures('ignore_nplusone') def test_list_retrieves_all_items_from_workspace(self, test_client, second_workspace, session): From 8803f57f81e9c201aae1ea8335273c2e13955cc9 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 6 Nov 2017 14:49:12 -0300 Subject: [PATCH 0491/1506] Add test case for vuln API --- test_cases/test_api_vulnerability.py | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index c36869066d5..92a979d858a 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -261,6 +261,36 @@ def test_create_vuln_props(self, host_with_hostnames, test_client, session): if prop not in vuln_props_excluded: assert res.json[prop] == value, prop + @pytest.mark.skip(reason="Known bug") + def test_create_idempotent(self, host_with_hostnames, test_client, session): + """ + This test makes sure that creating the same vuln twice doesn't duplicate the entry or has any other collateral effects + :param host_with_hostnames: + :param test_client: + :param session: + :return: + """ + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='Vulnerability name goes here', + vuln_type='Vulnerability', + parent_id=host_with_hostnames.id, + parent_type='Host', + refs=[], + policyviolations=[], + description='Description goes here', + severity='critical', + ) + ws_name = host_with_hostnames.workspace.name + vuln_count_previous = session.query(Vulnerability).count() + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) + assert res.status_code == 201 + assert vuln_count_previous + 1 == session.query(Vulnerability).count() + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) + assert res.status_code == 400 + assert vuln_count_previous + 1 == session.query(Vulnerability).count() + + def test_create_vuln_with_closed_status(self, host_with_hostnames, test_client, session): session.commit() # flush host_with_hostnames raw_data = self._create_post_data_vulnerability( From ea085b48aca9796dd46ffe30f100abd52837affb Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 6 Nov 2017 14:49:53 -0300 Subject: [PATCH 0492/1506] Remove easeofresolution empty field from vuln --- server/www/scripts/statusReport/controllers/modalNew.js | 1 - server/www/scripts/vulns/providers/vuln.js | 1 - 2 files changed, 2 deletions(-) diff --git a/server/www/scripts/statusReport/controllers/modalNew.js b/server/www/scripts/statusReport/controllers/modalNew.js index 47b1b2f8edb..8f44d7d87ab 100644 --- a/server/www/scripts/statusReport/controllers/modalNew.js +++ b/server/www/scripts/statusReport/controllers/modalNew.js @@ -64,7 +64,6 @@ angular.module('faradayApp') _attachments: {}, data: "", desc: "", - easeofresolution: undefined, impact: { accountability: false, availability: false, diff --git a/server/www/scripts/vulns/providers/vuln.js b/server/www/scripts/vulns/providers/vuln.js index 8a797019260..23380ceaace 100644 --- a/server/www/scripts/vulns/providers/vuln.js +++ b/server/www/scripts/vulns/providers/vuln.js @@ -15,7 +15,6 @@ angular.module('faradayApp') this.confirmed = true; this.data = ""; this.desc = ""; - this.easeofresolution = ""; this.hostnames = ""; this.impact = { accountability: false, From 5a321e3ceb1308a404a2f96247882d0cdfca4002 Mon Sep 17 00:00:00 2001 From: Mike Zhong Date: Mon, 6 Nov 2017 14:51:59 -0300 Subject: [PATCH 0493/1506] adapted get_private_ip() for OS X --- model/commands_history.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/model/commands_history.py b/model/commands_history.py index 268c5146d3d..21b17015778 100644 --- a/model/commands_history.py +++ b/model/commands_history.py @@ -12,6 +12,8 @@ import subprocess import getpass +from sys import platform as _platform + def get_private_ip(): """ @@ -20,7 +22,6 @@ def get_private_ip(): TODO: The problem is what happens when the machine has more than one private ip """ - # What's the best way to do this? ip = socket.gethostbyname(socket.gethostname()) if ip: if not ip.startswith('127'): @@ -29,7 +30,11 @@ def get_private_ip(): if ip: if not ip.startswith('127'): return ip - ip = subprocess.check_output(["ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/'"], shell=True) + if _platform == "linux" or _platform == "linux2": # linux + ip = subprocess.check_output(["ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/'"], shell=True) + elif _platform == "darwin": # MAC OS X + ip = subprocess.check_output(["ipconfig getifaddr en0 for"], shell=True) + ip = ip.rstrip() # removes '\n' return ip From 38f49ab39be94babbd3f685ab5cd5fb1225c4592 Mon Sep 17 00:00:00 2001 From: Mike Zhong Date: Mon, 6 Nov 2017 14:53:50 -0300 Subject: [PATCH 0494/1506] uses a more general command for OS X --- model/commands_history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/commands_history.py b/model/commands_history.py index 21b17015778..8665fa67e57 100644 --- a/model/commands_history.py +++ b/model/commands_history.py @@ -33,7 +33,7 @@ def get_private_ip(): if _platform == "linux" or _platform == "linux2": # linux ip = subprocess.check_output(["ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/'"], shell=True) elif _platform == "darwin": # MAC OS X - ip = subprocess.check_output(["ipconfig getifaddr en0 for"], shell=True) + ip = subprocess.check_output(["ifconfig | grep 'inet ' | grep -Fv 127.0.0.1 | awk '{print $2}' "], shell=True) ip = ip.rstrip() # removes '\n' return ip From 9e401a98014263e2ce8dbbb26ac0148732d14a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 6 Nov 2017 16:25:14 -0300 Subject: [PATCH 0495/1506] Fix issues with service count in the dashboard Related to wether to count service with status in closed or not --- server/api/base.py | 6 +++++- server/api/modules/services.py | 1 + server/api/modules/workspaces.py | 3 ++- server/models.py | 8 +++++--- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 70ec9847cca..51c3d73526e 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -397,6 +397,9 @@ class DeleteWorkspacedMixin(DeleteMixin): class CountWorkspacedMixin(object): + #: List of SQLAlchemy query filters to apply when counting + count_extra_filters = [] + def count(self, **kwargs): res = { 'groups': [], @@ -419,7 +422,8 @@ def count(self, **kwargs): db.session.query(self.model_class) .join(Workspace) .group_by(group_by) - .filter(Workspace.name == workspace_name)) + .filter(Workspace.name == workspace_name, + *self.count_extra_filters)) for key, count in count.values(group_by, func.count(group_by)): res['groups'].append( {'count': count, diff --git a/server/api/modules/services.py b/server/api/modules/services.py index 2a88ddf5419..239fe402298 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -59,6 +59,7 @@ class ServiceView(ReadWriteWorkspacedView): route_base = 'services' model_class = Service schema_class = ServiceSchema + count_extra_filters = [Service.status == 'open'] def _envelope_list(self, objects, pagination_metadata=None): services = [] diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index ac4fcb57845..f3e2299f603 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -36,7 +36,8 @@ class WorkspaceSummarySchema(Schema): credentials = fields.Integer(dump_only=True, attribute='credential_count') hosts = fields.Integer(dump_only=True, attribute='host_count') - services = fields.Integer(dump_only=True, attribute='service_count') + services = fields.Integer(dump_only=True, + attribute='total_service_count') web_vulns = fields.Integer(dump_only=True, allow_none=False, attribute='vulnerability_web_count') code_vulns = fields.Integer(dump_only=True, allow_none=False, diff --git a/server/models.py b/server/models.py index 97f7d234f76..25be1a07d1b 100644 --- a/server/models.py +++ b/server/models.py @@ -883,7 +883,9 @@ class Workspace(Metadata): credential_count = _make_generic_count_property('workspace', 'credential') host_count = _make_generic_count_property('workspace', 'host') - service_count = _make_generic_count_property('workspace', 'service') + open_service_count = _make_generic_count_property( + 'workspace', 'service', where=text("service.status = 'open'")) + total_service_count = _make_generic_count_property('workspace', 'service') vulnerability_web_count = query_expression() vulnerability_code_count = query_expression() @@ -898,11 +900,11 @@ def query_with_count(cls, only_confirmed): If only_confirmed is True, it will only show the count for confirmed vulnerabilities. Otherwise, it will show the count of all of them """ - from sqlalchemy.sql.expression import literal_column return cls.query.options( undefer(cls.host_count), undefer(cls.credential_count), - undefer(cls.service_count), + undefer(cls.open_service_count), + undefer(cls.total_service_count), with_expression( cls.vulnerability_web_count, _make_vuln_count_property('vulnerability_web', From 7c48224ec3e8969f9acc0bfa12f1860d59c2f5d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 6 Nov 2017 16:45:26 -0300 Subject: [PATCH 0496/1506] Fix order of boxes in workspace summarized report (dashboard) --- .../scripts/dashboard/controllers/summarizedCtrl.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/server/www/scripts/dashboard/controllers/summarizedCtrl.js b/server/www/scripts/dashboard/controllers/summarizedCtrl.js index b6427f12ad5..cc62a1f3c87 100644 --- a/server/www/scripts/dashboard/controllers/summarizedCtrl.js +++ b/server/www/scripts/dashboard/controllers/summarizedCtrl.js @@ -24,11 +24,16 @@ angular.module('faradayApp') } }; + // Used to set the order ob the items + countFields = ['hosts', 'credentials', 'services', 'std_vulns', + 'total_vulns', 'web_vulns'] + $scope.loadData = function() { dashboardSrv.getObjectsCount($scope.workspace) .then(function(obj_count) { $scope.objectsCount = []; - for (var property in obj_count) { + // for (var property in obj_count) { + countFields.forEach(function(property){ if (obj_count.hasOwnProperty(property) && obj_count[property] > 0) { var tmp_obj = {}; tmp_obj["value"] = obj_count[property]; @@ -38,11 +43,11 @@ angular.module('faradayApp') tmp_obj["key"] = property.replace("_", " "); $scope.objectsCount.push(tmp_obj); } - } + }); }); }; dashboardSrv.registerCallback($scope.loadData); init(); - }]); \ No newline at end of file + }]); From 447b52544b64dba30f2415dff5335bb60f54a19f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 6 Nov 2017 16:50:12 -0300 Subject: [PATCH 0497/1506] Update test after adding new fields to activity feed --- server/api/modules/commandsrun.py | 3 ++- server/importer.py | 5 +++++ server/models.py | 9 +++++++- .../dashboard/partials/activityFeed.html | 9 ++++---- test_cases/factories.py | 2 ++ test_cases/test_api_commands.py | 21 ++++++++++++------- 6 files changed, 35 insertions(+), 14 deletions(-) diff --git a/server/api/modules/commandsrun.py b/server/api/modules/commandsrun.py index 89b19712c00..18630d4bfe4 100644 --- a/server/api/modules/commandsrun.py +++ b/server/api/modules/commandsrun.py @@ -66,7 +66,8 @@ def activity_feed(self, workspace_name): for command in query.all(): res.append({ '_id': command.id, - 'command_type': '', + 'user': command.user, + 'import_source': command.import_source, 'command': command.command, 'params': command.params, 'vulnerabilities_count': (command.sum_created_vulnerabilities or 0) + (command.sum_created_vulnerabilities_web or 0), diff --git a/server/importer.py b/server/importer.py index 33bc6f407f8..be036263916 100644 --- a/server/importer.py +++ b/server/importer.py @@ -576,6 +576,10 @@ class CommandImporter(object): DOC_TYPE = 'CommandRunInformation' def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): + import_source = 'shell' + if document.get('Command', '').startswith('Import'): + import_source = 'shell' + start_date = datetime.datetime.fromtimestamp(document.get('itime')) command, instance = get_or_create( @@ -583,6 +587,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation Command, command=document.get('command', None), start_date=start_date, + import_source=import_source, ) if document.get('duration'): command.end_date = start_date + datetime.timedelta(seconds=document.get('duration')) diff --git a/server/models.py b/server/models.py index 91d863d3969..607d0ca5437 100644 --- a/server/models.py +++ b/server/models.py @@ -815,6 +815,12 @@ def _make_created_objects_sum_joined(object_type_filter, join_filters): class Command(Metadata): + + IMPORT_SOURCE = [ + 'report', # all the files the tools export and faraday imports it from the resports directory, gtk manual import or web import. + 'shell', # command executed on the shell or webshell with hooks connected to faraday. + ] + __tablename__ = 'command' id = Column(Integer, primary_key=True) command = Column(Text(), nullable=False) @@ -824,6 +830,7 @@ class Command(Metadata): hostname = Column(String(250), nullable=False) # where the command was executed params = Column(Text(), nullable=True) user = Column(String(250), nullable=True) # os username where the command was executed + import_source = Column(Enum(*IMPORT_SOURCE, name='import_source_enum')) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship('Workspace', foreign_keys=[workspace_id]) @@ -1026,7 +1033,7 @@ class File(Metadata): filename = Column(Text, nullable=False) description = Column(Text) content = Column(UploadedFileField(upload_type=FaradayUploadedFile), - nullable=True) # plain attached file + nullable=False) # plain attached file object_id = Column(Integer, nullable=False) object_type = Column(Text, nullable=False) diff --git a/server/www/scripts/dashboard/partials/activityFeed.html b/server/www/scripts/dashboard/partials/activityFeed.html index 8b5106e66b7..bc907d5a349 100644 --- a/server/www/scripts/dashboard/partials/activityFeed.html +++ b/server/www/scripts/dashboard/partials/activityFeed.html @@ -18,11 +18,10 @@

    Activity Feed

    - +
    - - - - {{cmd.user || Anonymous}} ran {{cmd.command}} - {{cmd.command.replace('Import', 'imported').replace(':', '') || Unknown}} + + + {{cmd.user || Anonymous}} ran {{cmd.command}} + {{cmd.command.replace('Import', 'imported').replace(':', '') || Unknown}} and found nothing : diff --git a/test_cases/factories.py b/test_cases/factories.py index ddbbc308f03..1d0bf34c1e7 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -273,6 +273,7 @@ class CommandFactory(WorkspaceObjectFactory): ip = FuzzyText() user = FuzzyText() hostname = FuzzyText() + import_source = 'shell' class Meta: model = Command @@ -302,6 +303,7 @@ class EmptyCommandFactory(FaradayFactory): ip = FuzzyText() user = FuzzyText() hostname = FuzzyText() + import_source = 'shell' class Meta: model = Command diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index e4e5a06c6aa..648f7e6e0f5 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -72,7 +72,8 @@ def test_activity_feed(self, session, test_client): assert filter(lambda stats: stats['_id'] == command.id, res.json) == [{u'_id': command.id, u'command': command.command, - u'command_type': '', + u'import_source': u'shell', + u'user': command.user, u'date': time.mktime(command.start_date.timetuple()) * 1000, u'params': command.params, u'hosts_count': 1, @@ -83,7 +84,8 @@ def test_activity_feed(self, session, test_client): assert filter(lambda stats: stats['_id'] == another_command.id, res.json) == [{ u'_id': another_command.id, u'command': another_command.command, - u'command_type': '', + u'import_source': u'shell', + u'user': another_command.user, u'date': time.mktime(another_command.start_date.timetuple()) * 1000, u'params': another_command.params, u'hosts_count': 0, @@ -119,7 +121,8 @@ def test_verify_created_critical_vulns_is_correctly_showing_sum_values(self, ses assert res.json == [ {u'_id': command.id, u'command': command.command, - u'command_type': '', + u'import_source': u'shell', + u'user': command.user, u'date': time.mktime(command.start_date.timetuple()) * 1000, u'params': command.params, u'hosts_count': 1, @@ -161,7 +164,8 @@ def test_verify_created_vulns_with_host_and_service_verification(self, session, assert res.status_code == 200 assert res.json == [{u'_id': command.id, u'command': command.command, - u'command_type': '', + u'import_source': u'shell', + u'user': command.user, u'date': time.mktime(command.start_date.timetuple()) * 1000, u'params': command.params, u'hosts_count': 1, @@ -210,7 +214,8 @@ def test_multiple_commands_executed_with_same_objects_found(self, session, test_ assert res.status_code == 200 assert res.json[0] == {u'_id': commands[0].id, u'command': commands[0].command, - u'command_type': '', + u'import_source': u'shell', + u'user': commands[0].user, u'date': time.mktime(commands[0].start_date.timetuple()) * 1000, u'params': commands[0].params, u'hosts_count': 1, @@ -221,7 +226,8 @@ def test_multiple_commands_executed_with_same_objects_found(self, session, test_ for index in range(1, 10): assert res.json[index] == {u'_id': commands[index].id, u'command': commands[index].command, - u'command_type': '', + u'import_source': u'shell', + u'user': commands[index].user, u'date': time.mktime(commands[index].start_date.timetuple()) * 1000, u'params': commands[index].params, u'hosts_count': 0, @@ -232,7 +238,8 @@ def test_multiple_commands_executed_with_same_objects_found(self, session, test_ # new command must create new service and vuln assert res.json[10] == {u'_id': command.id, u'command': command.command, - u'command_type': '', + u'import_source': u'shell', + u'user': command.user, u'date': time.mktime(command.start_date.timetuple()) * 1000, u'params': command.params, u'hosts_count': 0, From c64f598e7734681451f55de63ccefc38e768b86b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 6 Nov 2017 16:56:14 -0300 Subject: [PATCH 0498/1506] Update import_source instead of setting the value in creation time --- server/importer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index be036263916..fa14a967eee 100644 --- a/server/importer.py +++ b/server/importer.py @@ -587,11 +587,12 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation Command, command=document.get('command', None), start_date=start_date, - import_source=import_source, + ) if document.get('duration'): command.end_date = start_date + datetime.timedelta(seconds=document.get('duration')) + command.import_source = import_source command.command = document.get('command', None) command.ip = document.get('ip', None) command.hostname = document.get('hostname', None) From 561e9220723162fa45386df8080843b42cff7d12 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 6 Nov 2017 18:10:59 -0300 Subject: [PATCH 0499/1506] Se import source as report when it starts with Import --- server/importer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/importer.py b/server/importer.py index fa14a967eee..6f7973af38b 100644 --- a/server/importer.py +++ b/server/importer.py @@ -577,8 +577,8 @@ class CommandImporter(object): DOC_TYPE = 'CommandRunInformation' def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): import_source = 'shell' - if document.get('Command', '').startswith('Import'): - import_source = 'shell' + if document.get('command', '').startswith('Import'): + import_source = 'report' start_date = datetime.datetime.fromtimestamp(document.get('itime')) From 916a89a6cc5ccac00c3461a0fac3ecd18e3a34c5 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 6 Nov 2017 18:34:29 -0300 Subject: [PATCH 0500/1506] Fix sqlalchemy warnings --- server/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/models.py b/server/models.py index 607d0ca5437..219e6d27711 100644 --- a/server/models.py +++ b/server/models.py @@ -760,7 +760,7 @@ def parent(self): def _make_command_created_related_object(): query = select([BooleanToIntColumn("(count(*) = 0)")]) - query = query.select_from('command_object as command_object_inner') + query = query.select_from(text('command_object as command_object_inner')) where_expr = " command_object_inner.create_date < command_object.create_date and " \ " (command_object_inner.object_id = command_object.object_id and " \ " command_object_inner.object_type = command_object.object_type) " @@ -798,7 +798,7 @@ def _make_created_objects_sum(object_type_filter): return column_property( select([func.sum(CommandObject.created)]).\ select_from(table('command_object')). \ - where(where_condition) + where(text(where_condition)) ) @@ -810,7 +810,7 @@ def _make_created_objects_sum_joined(object_type_filter, join_filters): select([func.sum(CommandObject.created)]). \ select_from(table('command_object')). \ select_from(table('vulnerability')). \ - where(' and '.join(where_conditions)) + where(text(' and '.join(where_conditions))) ) From a806326e3458879853e40b20e220bb5449a99b9a Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 6 Nov 2017 18:34:43 -0300 Subject: [PATCH 0501/1506] Fix a bug on importer when logging creations --- server/importer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/importer.py b/server/importer.py index 6f7973af38b..9f53773733f 100644 --- a/server/importer.py +++ b/server/importer.py @@ -445,8 +445,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation website=website, workspace=workspace, ) - if command: - log_command_object_found(command, vulnerability, created) + if document['type'] == 'Vulnerability': vuln_params = { 'name': document.get('name'), @@ -474,6 +473,8 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation vulnerability.impact_availability = document.get('impact', {}).get('availability') vulnerability.impact_confidentiality = document.get('impact', {}).get('confidentiality') vulnerability.impact_integrity = document.get('impact', {}).get('integrity') + if command: + log_command_object_found(command, vulnerability, created) if document['type'] == 'VulnerabilityWeb': vulnerability.query_string = document.get('query') vulnerability.request = document.get('request') From a5f15822400e77987730eeea6bc9827d4fe03cbc Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 6 Nov 2017 18:37:46 -0300 Subject: [PATCH 0502/1506] remove flush --- server/importer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index 9f53773733f..41e8fd83858 100644 --- a/server/importer.py +++ b/server/importer.py @@ -532,7 +532,6 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation filename=attachment_name, object_id=vulnerability.id, object_type=vulnerability.__class__.__name__) - session.flush() file.content = attachment_file.read() attachment_file.close() From 14c2218937649ad625f7e61ac4cb7a08af54e6ee Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 6 Nov 2017 19:17:43 -0300 Subject: [PATCH 0503/1506] Fix vuln severity for the Web UI --- server/api/modules/vulns.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index ec25fc6e9ea..22aa90bc486 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -95,6 +95,7 @@ class VulnerabilitySchema(AutoSchema): '_id', 'ports', 'status', 'protocol', 'name', 'version', 'summary' ]), dump_only=True) host = fields.Integer(dump_only=True, attribute='host_id') + severity = fields.Method(serialize='get_severity', deserialize='load_severity') status = fields.Method(serialize='get_status', deserialize='load_status') # TODO: this breaks enum validation. type = fields.Method(serialize='get_type', deserialize='load_type') obj_id = fields.String(dump_only=True, attribute='id') @@ -148,6 +149,13 @@ def get_parent_type(self, obj): assert obj.service_id is not None or obj.host_id is not None return 'Service' if obj.service_id is not None else 'Host' + def get_severity(self, obj): + if obj.severity == 'medium': + return 'med' + if obj.severity == 'informational': + return 'info' + return obj.severity + def get_status(self, obj): if obj.status == 'open': return 'opened' @@ -156,6 +164,13 @@ def get_status(self, obj): def get_issuetracker(self, obj): return {} + def load_severity(self, value): + if value == 'med': + return 'medium' + if value == 'info': + return 'informational' + return value + def load_status(self, value): if value == 'opened': return 'open' From 244e485fbcd406f258928404915a7d64c4b85591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 6 Nov 2017 19:43:55 -0300 Subject: [PATCH 0504/1506] Fix copy-paste typo and change sorting in hosts API Sort by host ip by default --- server/api/base.py | 6 ++++++ server/api/modules/hosts.py | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/server/api/base.py b/server/api/base.py index 51c3d73526e..d2d8e61ba89 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -214,6 +214,10 @@ def _validate_uniqueness(self, obj, object_id=None): class ListMixin(object): """Add GET / route""" + #: If set (to a SQLAlchemy attribute instance) use this field to order the + #: query by default + order_field = None + def _envelope_list(self, objects, pagination_metadata=None): """Override this method to define how a list of objects is rendered""" @@ -224,6 +228,8 @@ def _paginate(self, query): def index(self, **kwargs): query = self._filter_query(self._get_eagerloaded_query(**kwargs)) + if self.order_field is not None: + query = query.order_by(self.order_field) objects, pagination_metadata = self._paginate(query) return self._envelope_list(self._dump(objects, many=True), pagination_metadata) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 622476a1164..d961d74e7bc 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -40,7 +40,7 @@ class HostSchema(AutoSchema): owner = PrimaryKeyRelatedField('username', attribute='creator', dump_only=True) services = fields.Integer(attribute='open_service_count', dump_only=True) vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) - credentials = fields.Integer(attribute='vulnerability_count', dump_only=True) + credentials = fields.Integer(attribute='credentials_count', dump_only=True) hostnames = PrimaryKeyRelatedField('name', many=True, attribute="hostnames", # TODO migration: make it writable @@ -88,6 +88,7 @@ class HostsView(PaginatedMixin, ReadWriteWorkspacedView): route_base = 'hosts' model_class = Host + order_field = Host.ip.asc() schema_class = HostSchema unique_fields = ['ip'] filterset_class = HostFilterSet From 4ac67782f42d591e14bf1cbc6946677f43ccb5d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 6 Nov 2017 19:46:27 -0300 Subject: [PATCH 0505/1506] Fix Host.vulnerability_count It was only counting the vulnerabilities with parent Host. It should also consider the vulns whose parent is a service of the host --- server/models.py | 23 +++++++++++++++++++++-- test_cases/test_api_hosts.py | 2 +- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/server/models.py b/server/models.py index 91d863d3969..4c9e1ec486f 100644 --- a/server/models.py +++ b/server/models.py @@ -167,7 +167,8 @@ class Host(Metadata): mac = Column(Text, nullable=True) net_segment = Column(Text, nullable=True) - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, + nullable=False) workspace = relationship( 'Workspace', backref='hosts', @@ -177,7 +178,25 @@ class Host(Metadata): open_service_count = _make_generic_count_property( 'host', 'service', where=text("service.status = 'open'")) total_service_count = _make_generic_count_property('host', 'service') - vulnerability_count = _make_generic_count_property('host', 'vulnerability') + + __host_vulnerabilities = ( + select([func.count(text('vulnerability.id'))]). + select_from('vulnerability'). + where(text('vulnerability.host_id = host.id')). + as_scalar() + ) + __service_vulnerabilities = ( + select([func.count(text('vulnerability.id'))]). + select_from(text('vulnerability, service')). + where(text('vulnerability.service_id = service.id and ' + 'service.host_id = host.id')). + as_scalar() + ) + vulnerability_count = column_property( + # select(text('count(*)')).select_from(__host_vulnerabilities.subquery()), + __host_vulnerabilities + __service_vulnerabilities, + deferred=True) + credentials_count = _make_generic_count_property('host', 'credential') __table_args__ = ( diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 74ead2004fa..6062a10e666 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -285,7 +285,7 @@ def test_host_with_open_vuln_count_verification(self, test_client, session, assert res.status_code == 200 json_host = filter(lambda json_host: json_host['value']['id'] == host.id, res.json['rows'])[0] # the host has one vuln associated. another one via service. - assert json_host['value']['vulns'] == 1 + assert json_host['value']['vulns'] == 2 def test_host_services_vuln_count_verification(self, test_client, session, workspace, host_factory, vulnerability_factory, From f779ea3191fbccbfcb1033614cf627d792c0b88b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 7 Nov 2017 12:27:34 -0300 Subject: [PATCH 0506/1506] Add unique constraint for Command Object. Fix a bug in importer --- server/models.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/server/models.py b/server/models.py index 219e6d27711..63d62f38f33 100644 --- a/server/models.py +++ b/server/models.py @@ -780,6 +780,9 @@ class CommandObject(db.Model): command = relationship('Command', backref='command_objects') command_id = Column(Integer, ForeignKey('command.id'), index=True) + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) + workspace = relationship('Workspace', foreign_keys=[workspace_id]) + create_date = Column(DateTime, default=datetime.utcnow) # the following properties are used to know if the command created the specified objects_type @@ -792,6 +795,11 @@ class CommandObject(db.Model): # Command was created. created_persistent = Column(Boolean, default=False) + __table_args__ = ( + UniqueConstraint('object_id', 'object_type', 'command_id', 'workspace_id', + name='uix_command_object_object_id_object_type_command_id_workspace_id'), + ) + def _make_created_objects_sum(object_type_filter): where_condition = "command_object.command_id = command.id and command_object.object_type= '%s'" % object_type_filter @@ -1258,11 +1266,12 @@ def log_command_object_found(command, object, created): object_type = 'vulnerability' db.session.flush() - get_or_create( + log, _ = get_or_create( db.session, CommandObject, command=command, object_id=object.id, object_type=object_type, - created_persistent=created, + workspace=object.workspace, ) + log.created_persistent=created From 4a90b92bf6e6f33f5817e53c3cd452605dfb1359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 7 Nov 2017 17:20:25 -0300 Subject: [PATCH 0507/1506] Add target field to vulns API It is used in the Last Vulnerabilities widget in the dashboard --- server/api/modules/vulns.py | 8 +++++++- test_cases/test_api_vulnerability.py | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 22aa90bc486..47950466898 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -99,7 +99,7 @@ class VulnerabilitySchema(AutoSchema): status = fields.Method(serialize='get_status', deserialize='load_status') # TODO: this breaks enum validation. type = fields.Method(serialize='get_type', deserialize='load_type') obj_id = fields.String(dump_only=True, attribute='id') - target = fields.String(default='') # TODO: review this attribute + target = fields.Method('get_target') class Meta: model = Vulnerability @@ -164,6 +164,12 @@ def get_status(self, obj): def get_issuetracker(self, obj): return {} + def get_target(self, obj): + if obj.service is not None: + return obj.service.host.ip + else: + return obj.host.ip + def load_severity(self, value): if value == 'med': return 'medium' diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 92a979d858a..f60af906a63 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -683,3 +683,24 @@ def test_count_confirmed(self, test_client, session): {"name": "high", "severity": "high", "count": 2}, {"name": "critical", "severity": "critical", "count": 1}, ]) + + @pytest.mark.usefixtures('mock_envelope_list') + def test_target(self, test_client, session, second_workspace, + host_factory, service_factory, + vulnerability_factory, vulnerability_web_factory): + host = host_factory.create(workspace=second_workspace) + service = service_factory.create(host=host, + workspace=second_workspace) + vulns = [ + vulnerability_factory.create(host=host, service=None, + workspace=second_workspace), + vulnerability_factory.create(service=service, host=None, + workspace=second_workspace), + vulnerability_web_factory.create(service=service, + workspace=second_workspace), + ] + + res = test_client.get(self.url(workspace=second_workspace)) + assert res.status_code == 200 + for v in res.json['data']: + assert v['target'] == host.ip From 3421d7d47463ce3c7ff4e9e6d3c4b07e4651c876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 7 Nov 2017 18:36:25 -0300 Subject: [PATCH 0508/1506] Fix count vulns grouped by severity Map "informational" to "info" and "medium" to "med". Extends the changes done in commit 14c2218937649ad625f7e61ac4cb7a08af54e6ee --- server/api/modules/vulns.py | 19 +++++++++++++++++++ test_cases/test_api_vulnerability.py | 21 ++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 47950466898..44657ae30f5 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -379,5 +379,24 @@ def _envelope_list(self, objects, pagination_metadata=None): 'vulnerabilities': vulns, } + def count(self, **kwargs): + """Override to change severity values""" + res = super(VulnerabilityView, self).count(**kwargs) + + def convert_group(group): + group = group.copy() + severity_map = { + "informational": "info", + "medium": "med" + } + severity = group['severity'] + group['severity'] = group['name'] = severity_map.get( + severity, severity) + return group + + if request.args.get('group_by') == 'severity': + res['groups'] = [convert_group(group) for group in res['groups']] + return res + VulnerabilityView.register(vulns_api) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index f60af906a63..c7d9abb4fe9 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -676,7 +676,7 @@ def test_count_confirmed(self, test_client, session): session.commit() res = test_client.get(self.url() + - 'count/?confirmed=1&group_by=severity') + 'count/?confirmed=1&group_by=severity') assert res.status_code == 200 assert res.json['total_count'] == 3 assert sorted(res.json['groups']) == sorted([ @@ -684,6 +684,25 @@ def test_count_confirmed(self, test_client, session): {"name": "critical", "severity": "critical", "count": 1}, ]) + def test_count_severity_map(self, test_client, second_workspace, session): + self.factory.create_batch(4, severity='informational', + workspace=second_workspace) + self.factory.create_batch(3, severity='medium', + workspace=second_workspace) + self.factory.create_batch(2, severity='low', + workspace=second_workspace) + session.commit() + + res = test_client.get(self.url(workspace=second_workspace) + + 'count/?group_by=severity') + assert res.status_code == 200 + assert res.json['total_count'] == 9 + assert sorted(res.json['groups']) == sorted([ + {"name": "med", "severity": "med", "count": 3}, + {"name": "low", "severity": "low", "count": 2}, + {"name": "info", "severity": "info", "count": 4}, + ]) + @pytest.mark.usefixtures('mock_envelope_list') def test_target(self, test_client, session, second_workspace, host_factory, service_factory, From 589171ab5f43e0e9a932ad1ceb3a20f09f753ae8 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 7 Nov 2017 19:19:41 -0300 Subject: [PATCH 0509/1506] Fix a bug on activity feed. Use workspace_id in queries when joining. Update unit tests --- server/api/modules/commandsrun.py | 2 +- server/api/modules/vulns.py | 37 ++++- server/models.py | 223 ++++++++++++++++-------------- test_cases/factories.py | 8 +- test_cases/test_api_commands.py | 39 ++++-- 5 files changed, 184 insertions(+), 125 deletions(-) diff --git a/server/api/modules/commandsrun.py b/server/api/modules/commandsrun.py index 18630d4bfe4..ab555235024 100644 --- a/server/api/modules/commandsrun.py +++ b/server/api/modules/commandsrun.py @@ -70,7 +70,7 @@ def activity_feed(self, workspace_name): 'import_source': command.import_source, 'command': command.command, 'params': command.params, - 'vulnerabilities_count': (command.sum_created_vulnerabilities or 0) + (command.sum_created_vulnerabilities_web or 0), + 'vulnerabilities_count': (command.sum_created_vulnerabilities or 0), 'hosts_count': command.sum_created_hosts or 0, 'services_count': command.sum_created_services or 0, 'criticalIssue': command.sum_created_vulnerability_critical or 0, diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 22aa90bc486..38d058469c6 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -7,10 +7,11 @@ from base64 import b64encode, b64decode from filteralchemy import FilterSet, operators -from flask import request +from flask import request, current_app from flask import Blueprint from marshmallow import Schema, fields, post_load, ValidationError from marshmallow.validate import OneOf +from sqlalchemy import and_ from sqlalchemy.orm import joinedload, selectin_polymorphic from depot.manager import DepotManager @@ -29,7 +30,7 @@ Vulnerability, VulnerabilityWeb, VulnerabilityGeneric, - Host, Service, File) + Host, Service, File, CommandObject) from server.utils.database import get_or_create from server.api.modules.services import ServiceSchema @@ -118,12 +119,17 @@ def get_type(self, obj): return obj.__class__.__name__ def get_metadata(self, obj): + command_id = None + command_obj = CommandObject.query.filter_by(object_type='vulnerability', object_id=obj.id, workspace=obj.workspace).first() + if command_obj: + command_id = command_obj.command_id return { - "command_id": "e1a042dd0e054c1495e1c01ced856438", + "command_id": command_id, "create_time": time.mktime(obj.create_date.utctimetuple()), - "creator": "Metasploit", - "owner": "", "update_action": 0, - "update_controller_action": "No model controller call", + "creator": "", + "owner": obj.creator and obj.creator.username or '', + "update_action": 0, + "update_controller_action": "", "update_time": time.mktime(obj.update_date.utctimetuple()), "update_user": "" } @@ -260,15 +266,32 @@ class Meta(FilterSetMeta): "status", "website", "parameter_name", "query_string", "path", "data", "severity", "confirmed", "name", "request", "response", "parameters", "resolution", "method", "ease_of_resolution", - "description") + "description", "command_id") + strict_fields = ( "severity", "confirmed", "method" ) + default_operator = operators.ILike column_overrides = { field: _strict_filtering for field in strict_fields} operators = (operators.ILike, operators.Equal) + def filter(self): + """Generate a filtered query from request parameters. + + :returns: Filtered SQLALchemy query + """ + command_id = request.args.get('command_id') + if command_id: + self.query = db.session.query(VulnerabilityGeneric).join(CommandObject, and_(VulnerabilityWeb.id == CommandObject.object_id, CommandObject.object_type=='vulnerability')) + + query = super(VulnerabilityFilterSet, self).filter() + + if command_id: + query = query.filter(CommandObject.command_id==int(command_id)) + return query + class VulnerabilityView(PaginatedMixin, FilterAlchemyMixin, diff --git a/server/models.py b/server/models.py index ee880f42c1d..df74ab71176 100644 --- a/server/models.py +++ b/server/models.py @@ -16,8 +16,9 @@ String, Text, UniqueConstraint, - event -) + event, + and_) +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import backref, relationship, undefer from sqlalchemy.sql import select, text, table from sqlalchemy import func @@ -110,6 +111,19 @@ def _make_generic_count_property(parent_table, children_table, where=None): return column_property(query, deferred=True) +def _make_command_created_related_object(): + query = select([BooleanToIntColumn("(count(*) = 0)")]) + query = query.select_from(text('command_object as command_object_inner')) + where_expr = " command_object_inner.create_date < command_object.create_date and " \ + " (command_object_inner.object_id = command_object.object_id and " \ + " command_object_inner.object_type = command_object.object_type) and " \ + " command_object_inner.workspace_id = command_object.workspace_id " + query = query.where(text(where_expr)) + return column_property( + query, + ) + + class DatabaseMetadata(db.Model): __tablename__ = 'db_metadata' id = Column(Integer, primary_key=True) @@ -427,6 +441,110 @@ class VulnerabilityTemplate(VulnerabilityABC): ) +class CommandObject(db.Model): + __tablename__ = 'command_object' + id = Column(Integer, primary_key=True) + + object_id = Column(Integer, nullable=False) + object_type = Column(Text, nullable=False) + + command = relationship('Command', backref='command_objects') + command_id = Column(Integer, ForeignKey('command.id'), index=True) + + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) + workspace = relationship('Workspace', foreign_keys=[workspace_id]) + + create_date = Column(DateTime, default=datetime.utcnow) + + # the following properties are used to know if the command created the specified objects_type + # remeber that this table has a row instances per relationship. + # this created integer can be used to obtain the total object_type objects created. + created = _make_command_created_related_object() + + # We are currently using the column property created. however to avoid losing information + # we also store the a boolean to know if at the moment of created the object related to the + # Command was created. + created_persistent = Column(Boolean, default=False) + + __table_args__ = ( + UniqueConstraint('object_id', 'object_type', 'command_id', 'workspace_id', + name='uix_command_object_object_id_object_type_command_id_workspace_id'), + ) + + @property + def parent(self): + return self.command + + +def _make_created_objects_sum(object_type_filter): + where_conditions = ["command_object.object_type= '%s'" % object_type_filter] + where_conditions.append("command_object.command_id = command.id") + where_conditions.append("command_object.workspace_id = command.workspace_id") + return column_property( + select([func.sum(CommandObject.created)]).\ + select_from(table('command_object')). \ + where(text(' and '.join(where_conditions))) + ) + + +def _make_created_objects_sum_joined(object_type_filter, join_filters): + """ + + :param object_type_filter: can be any host, service, vulnerability, credential or any object created from commands. + :param join_filters: Filter for vulnerability fields. + :return: column property with sum of created objects. + """ + where_conditions = ["command_object.object_type= '%s'" % object_type_filter] + where_conditions.append("command_object.command_id = command.id") + where_conditions.append("vulnerability.id = command_object.object_id ") + where_conditions.append("command_object.workspace_id = vulnerability.workspace_id") + for attr, filter_value in join_filters.items(): + where_conditions.append("vulnerability.{0} = {1}".format(attr, filter_value)) + return column_property( + select([func.sum(CommandObject.created)]). \ + select_from(table('command_object')). \ + select_from(table('vulnerability')). \ + where(text(' and '.join(where_conditions))) + ) + + +class Command(Metadata): + + IMPORT_SOURCE = [ + 'report', # all the files the tools export and faraday imports it from the resports directory, gtk manual import or web import. + 'shell', # command executed on the shell or webshell with hooks connected to faraday. + ] + + __tablename__ = 'command' + id = Column(Integer, primary_key=True) + command = Column(Text(), nullable=False) + start_date = Column(DateTime, nullable=False) + end_date = Column(DateTime, nullable=True) + ip = Column(String(250), nullable=False) # where the command was executed + hostname = Column(String(250), nullable=False) # where the command was executed + params = Column(Text(), nullable=True) + user = Column(String(250), nullable=True) # os username where the command was executed + import_source = Column(Enum(*IMPORT_SOURCE, name='import_source_enum')) + + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) + workspace = relationship('Workspace', foreign_keys=[workspace_id]) + # TODO: add Tool relationship and report_attachment + + sum_created_vulnerabilities = _make_created_objects_sum('vulnerability') + + sum_created_vulnerabilities_web = _make_created_objects_sum_joined('vulnerability', {'type': '\'vulnerability_web\''}) + + sum_created_hosts = _make_created_objects_sum('host') + + sum_created_services = _make_created_objects_sum('service') + + sum_created_vulnerability_critical = _make_created_objects_sum_joined('vulnerability', {'severity': '\'critical\''}) + + @property + def parent(self): + return + + class VulnerabilityGeneric(VulnerabilityABC): STATUSES = [ 'open', @@ -777,107 +895,6 @@ def parent(self): return self.host or self.service -def _make_command_created_related_object(): - query = select([BooleanToIntColumn("(count(*) = 0)")]) - query = query.select_from(text('command_object as command_object_inner')) - where_expr = " command_object_inner.create_date < command_object.create_date and " \ - " (command_object_inner.object_id = command_object.object_id and " \ - " command_object_inner.object_type = command_object.object_type) " - query = query.where(text(where_expr)) - return column_property( - query, - ) - - -class CommandObject(db.Model): - __tablename__ = 'command_object' - id = Column(Integer, primary_key=True) - - object_id = Column(Integer, nullable=False) - object_type = Column(Text, nullable=False) - - command = relationship('Command', backref='command_objects') - command_id = Column(Integer, ForeignKey('command.id'), index=True) - - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) - workspace = relationship('Workspace', foreign_keys=[workspace_id]) - - create_date = Column(DateTime, default=datetime.utcnow) - - # the following properties are used to know if the command created the specified objects_type - # remeber that this table has a row instances per relationship. - # this created integer can be used to obtain the total object_type objects created. - created = _make_command_created_related_object() - - # We are currently using the column property created. however to avoid losing information - # we also store the a boolean to know if at the moment of created the object related to the - # Command was created. - created_persistent = Column(Boolean, default=False) - - __table_args__ = ( - UniqueConstraint('object_id', 'object_type', 'command_id', 'workspace_id', - name='uix_command_object_object_id_object_type_command_id_workspace_id'), - ) - - -def _make_created_objects_sum(object_type_filter): - where_condition = "command_object.command_id = command.id and command_object.object_type= '%s'" % object_type_filter - return column_property( - select([func.sum(CommandObject.created)]).\ - select_from(table('command_object')). \ - where(text(where_condition)) - ) - - -def _make_created_objects_sum_joined(object_type_filter, join_filters): - where_conditions = ["vulnerability.id = command_object.object_id and command_object.command_id = command.id and command_object.object_type= '%s'" % object_type_filter] - for attr, filter_value in join_filters.items(): - where_conditions.append("vulnerability.{0} = {1}".format(attr, filter_value)) - return column_property( - select([func.sum(CommandObject.created)]). \ - select_from(table('command_object')). \ - select_from(table('vulnerability')). \ - where(text(' and '.join(where_conditions))) - ) - - -class Command(Metadata): - - IMPORT_SOURCE = [ - 'report', # all the files the tools export and faraday imports it from the resports directory, gtk manual import or web import. - 'shell', # command executed on the shell or webshell with hooks connected to faraday. - ] - - __tablename__ = 'command' - id = Column(Integer, primary_key=True) - command = Column(Text(), nullable=False) - start_date = Column(DateTime, nullable=False) - end_date = Column(DateTime, nullable=True) - ip = Column(String(250), nullable=False) # where the command was executed - hostname = Column(String(250), nullable=False) # where the command was executed - params = Column(Text(), nullable=True) - user = Column(String(250), nullable=True) # os username where the command was executed - import_source = Column(Enum(*IMPORT_SOURCE, name='import_source_enum')) - - workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) - workspace = relationship('Workspace', foreign_keys=[workspace_id]) - # TODO: add Tool relationship and report_attachment - - sum_created_vulnerabilities = _make_created_objects_sum('vulnerability') - - sum_created_vulnerabilities_web = _make_created_objects_sum_joined('vulnerability', {'type': '\'vulnerability_web\''}) - - sum_created_hosts = _make_created_objects_sum('host') - - sum_created_services = _make_created_objects_sum('service') - - sum_created_vulnerability_critical = _make_created_objects_sum_joined('vulnerability', {'severity': '\'critical\''}) - - @property - def parent(self): - return - - def _make_vuln_count_property(type_=None, only_confirmed=False, use_column_property=True): query = (select([func.count(text('vulnerability.id'))]). diff --git a/test_cases/factories.py b/test_cases/factories.py index 1d0bf34c1e7..225b64ea676 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -260,6 +260,7 @@ class Meta: class CommandObjectFactory(FaradayFactory): + workspace = factory.SubFactory(WorkspaceFactory) class Meta: model = CommandObject @@ -289,14 +290,19 @@ def attach_vuln_object(self, create, extracted, **kwargs): object_type='vulnerability', object_id=vuln.id, command=self, + workspace=self.workspace ) CommandObjectFactory.create( object_type='host', object_id=host.id, command=self, + workspace=self.workspace ) -class EmptyCommandFactory(FaradayFactory): +class EmptyCommandFactory(WorkspaceObjectFactory): + """ + A command without command objects. + """ command = FuzzyText() end_date = FuzzyDateTime(datetime.datetime.utcnow().replace(tzinfo=pytz.utc) + datetime.timedelta(20), datetime.datetime.utcnow().replace(tzinfo=pytz.utc) + datetime.timedelta(30)) start_date = FuzzyDateTime(datetime.datetime.utcnow().replace(tzinfo=pytz.utc) - datetime.timedelta(30), datetime.datetime.utcnow().replace(tzinfo=pytz.utc) - datetime.timedelta(20)) diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index 648f7e6e0f5..cebe290528b 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -59,12 +59,14 @@ def test_activity_feed(self, session, test_client): CommandObjectFactory.create( command=another_command, object_type='vulnerability', - object_id=vuln.id + object_id=vuln.id, + workspace=command.workspace ) CommandObjectFactory.create( command=another_command, object_type='host', - object_id=vuln.host.id + object_id=vuln.host.id, + workspace=command.workspace ) session.commit() res = test_client.get(self.url(workspace=command.workspace) + 'activity_feed/') @@ -103,17 +105,20 @@ def test_verify_created_critical_vulns_is_correctly_showing_sum_values(self, ses CommandObjectFactory.create( command=command, object_type='host', - object_id=host.id + object_id=host.id, + workspace=workspace ) CommandObjectFactory.create( command=command, object_type='vulnerability', - object_id=vuln.id + object_id=vuln.id, + workspace=workspace ) CommandObjectFactory.create( command=command, object_type='vulnerability', - object_id=vuln_med.id + object_id=vuln_med.id, + workspace=workspace ) session.commit() res = test_client.get(self.url(workspace=command.workspace) + 'activity_feed/') @@ -142,22 +147,26 @@ def test_verify_created_vulns_with_host_and_service_verification(self, session, CommandObjectFactory.create( command=command, object_type='host', - object_id=host.id + object_id=host.id, + workspace=workspace ) CommandObjectFactory.create( command=command, object_type='vulnerability', - object_id=vuln.id + object_id=vuln.id, + workspace=workspace ) CommandObjectFactory.create( command=command, object_type='service', - object_id=service.id + object_id=service.id, + workspace=workspace ) CommandObjectFactory.create( command=command, object_type='vulnerability', - object_id=vuln_med.id + object_id=vuln_med.id, + workspace=workspace ) session.commit() res = test_client.get(self.url(workspace=command.workspace) + 'activity_feed/') @@ -189,12 +198,14 @@ def test_multiple_commands_executed_with_same_objects_found(self, session, test_ CommandObjectFactory.create( command=command, object_type='host', - object_id=host.id + object_id=host.id, + workspace=workspace ) CommandObjectFactory.create( command=command, object_type='vulnerability', - object_id=vuln.id + object_id=vuln.id, + workspace=workspace ) vuln_med = VulnerabilityFactory.create(severity='medium', workspace=workspace, service=service, host=None) session.flush() @@ -202,12 +213,14 @@ def test_multiple_commands_executed_with_same_objects_found(self, session, test_ CommandObjectFactory.create( command=command, object_type='service', - object_id=service.id + object_id=service.id, + workspace=workspace ) CommandObjectFactory.create( command=command, object_type='vulnerability', - object_id=vuln_med.id + object_id=vuln_med.id, + workspace=workspace ) session.commit() res = test_client.get(self.url(workspace=command.workspace) + 'activity_feed/') From a27ffb61af3e0cd4939c4bc43b4165b5f994a62e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 8 Nov 2017 15:02:09 -0300 Subject: [PATCH 0510/1506] Add test for command filter --- test_cases/test_api_vulnerability.py | 87 +++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 2 deletions(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index c7d9abb4fe9..14b8c347693 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -12,8 +12,8 @@ from server.models import ( Vulnerability, VulnerabilityWeb, - Reference, PolicyViolation) -from test_cases.factories import ServiceFactory + Reference, PolicyViolation, CommandObject) +from test_cases.factories import ServiceFactory, CommandFactory, CommandObjectFactory, HostFactory, EmptyCommandFactory CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) @@ -27,6 +27,7 @@ class TestListVulnerabilityView(ReadOnlyAPITests): #update_fields = ['ip', 'description', 'os'] view_class = VulnerabilityView + @pytest.mark.usefixtures('ignore_nplusone') def test_backward_json_compatibility(self, test_client, second_workspace, session): self.factory.create(workspace=second_workspace) session.commit() @@ -192,6 +193,7 @@ def _create_post_data_vulnerability(self, name, vuln_type, parent_id, return data + @pytest.mark.usefixtures('ignore_nplusone') def test_create_vuln(self, host_with_hostnames, test_client, session): """ This one should only check basic vuln properties @@ -430,6 +432,7 @@ def test_create_vuln_web(self, host_with_hostnames, test_client, session): assert res.json['path'] == '/pepep' @pytest.mark.usefixtures('mock_envelope_list') + @pytest.mark.usefixtures('ignore_nplusone') def test_filter_by_severity(self, test_client, session, second_workspace, vulnerability_factory, @@ -464,6 +467,7 @@ def test_filter_by_invalid_severity(self, test_client): assert res.status_code == 400 @pytest.mark.usefixtures('mock_envelope_list') + @pytest.mark.usefixtures('ignore_nplusone') def test_filter_by_method(self, test_client, session, second_workspace, vulnerability_factory, vulnerability_web_factory): @@ -494,6 +498,7 @@ def test_filter_by_method(self, test_client, session, second_workspace, assert len(res.json['data']) == 0 @pytest.mark.usefixtures('mock_envelope_list') + @pytest.mark.usefixtures('ignore_nplusone') def test_filter_by_website(self, test_client, session, second_workspace, vulnerability_factory, @@ -723,3 +728,81 @@ def test_target(self, test_client, session, second_workspace, assert res.status_code == 200 for v in res.json['data']: assert v['target'] == host.ip + + @pytest.mark.usefixtures('mock_envelope_list') + @pytest.mark.usefixtures('ignore_nplusone') + def test_filter_by_command_id(self, test_client, session, + second_workspace, + vulnerability_factory, + vulnerability_web_factory, + ): + expected_ids = set() + web_expected_ids = set() + host = HostFactory.create(workspace=second_workspace) + service = ServiceFactory.create(workspace=second_workspace) + + command = EmptyCommandFactory.create(workspace=second_workspace) + web_command = EmptyCommandFactory.create(workspace=second_workspace) + high_vulns = vulnerability_factory.create_batch( + 5, workspace=second_workspace, severity='high', host=host, service=None) + high_vulns_web = vulnerability_web_factory.create_batch( + 5, workspace=second_workspace, severity='high', service=service) + session.commit() + CommandObjectFactory.create( + command=command, + object_type='host', + object_id=host.id, + workspace=second_workspace + ) + CommandObjectFactory.create( + command=web_command, + object_type='service', + object_id=service.id, + workspace=second_workspace + ) + for high_vuln in high_vulns: + + CommandObjectFactory.create( + command=command, + object_type='vulnerability', + object_id=high_vuln.id, + workspace=second_workspace + ) + for high_vuln_web in high_vulns_web: + + CommandObjectFactory.create( + command=web_command, + object_type='vulnerability', + object_id=high_vuln_web.id, + workspace=second_workspace + ) + + session.commit() + + expected_ids.update(vuln.id for vuln in high_vulns) + web_expected_ids.update(vuln.id for vuln in high_vulns_web) + + res = test_client.get(self.url( + workspace=second_workspace) + '?command_id={0}'.format(command.id)) + assert res.status_code == 200 + for vuln in res.json['data']: + command_object = CommandObject.query.filter_by( + object_id=vuln['_id'], + object_type='vulnerability', + workspace=second_workspace, + ).first() + vuln['metadata']['command_id'] == command_object.command.id + assert set(vuln['_id'] for vuln in res.json['data']) == expected_ids + + # Check for web vulns + res = test_client.get(self.url( + workspace=second_workspace) + '?command_id={0}'.format(web_command.id)) + assert res.status_code == 200 + for vuln in res.json['data']: + command_object = CommandObject.query.filter_by( + object_id=vuln['_id'], + object_type='vulnerability', + workspace=second_workspace, + ).first() + vuln['metadata']['command_id'] == command_object.command.id + assert set(vuln['_id'] for vuln in res.json['data']) == web_expected_ids \ No newline at end of file From bcd588cae42d8cf0b56ac64130fb0b8687c674c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 8 Nov 2017 16:34:32 -0300 Subject: [PATCH 0511/1506] Fix vuln count in status report --- server/api/modules/vulns.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 703d9f6e7a0..414a9bd79ea 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -400,6 +400,8 @@ def _envelope_list(self, objects, pagination_metadata=None): }) return { 'vulnerabilities': vulns, + 'count': (pagination_metadata.total + if pagination_metadata is not None else len(vulns)) } def count(self, **kwargs): From a77746394ec0d0d9818b27a4a701c09235162119 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 8 Nov 2017 16:35:34 -0300 Subject: [PATCH 0512/1506] Improve reset_db command Add more conflicting tables --- server/commands/reset_db.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/commands/reset_db.py b/server/commands/reset_db.py index 96f4714a3bf..1aed2ce90f0 100644 --- a/server/commands/reset_db.py +++ b/server/commands/reset_db.py @@ -5,10 +5,11 @@ class ResetDB(Command): def run(self): # It might be required to do a cascade delete to correctly the # vulnerability table - try: - db.engine.execute('DROP TABLE vulnerability CASCADE') - except: - pass + for table in ('vulnerability', 'vulnerability_template', 'comment'): + try: + db.engine.execute('DROP TABLE {} CASCADE'.format(table)) + except: + pass db.drop_all() db.create_all() From 3df9535c21f00c60e5df441a48a58c76ece486f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 8 Nov 2017 17:30:22 -0300 Subject: [PATCH 0513/1506] Fix vuln and credentials count un service list (in host details) --- server/api/modules/hosts.py | 4 +++- server/models.py | 4 +++- server/www/scripts/hosts/controllers/host.js | 5 ----- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index d961d74e7bc..248f8617d4d 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -77,10 +77,12 @@ class Meta(FilterSetMeta): class ServiceSchema(AutoSchema): vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) + credentials = fields.Integer(attribute='credentials_count', dump_only=True) class Meta: model = Service - fields = ('id', 'name', 'description', 'port', 'protocol', 'status', 'vulns') + fields = ('id', 'name', 'description', 'port', 'protocol', 'status', + 'vulns', 'credentials') class HostsView(PaginatedMixin, diff --git a/server/models.py b/server/models.py index df74ab71176..25d39e1f6ca 100644 --- a/server/models.py +++ b/server/models.py @@ -276,7 +276,9 @@ class Service(Metadata): foreign_keys=[workspace_id] ) - vulnerability_count = _make_generic_count_property('service', 'vulnerability') + vulnerability_count = _make_generic_count_property('service', + 'vulnerability') + credentials_count = _make_generic_count_property('service', 'credential') __table_args__ = ( UniqueConstraint(port, protocol, host_id, workspace_id, name='uix_service_port_protocol_host_workspace'), diff --git a/server/www/scripts/hosts/controllers/host.js b/server/www/scripts/hosts/controllers/host.js index 230d0eeadc3..2dd823da79b 100644 --- a/server/www/scripts/hosts/controllers/host.js +++ b/server/www/scripts/hosts/controllers/host.js @@ -59,11 +59,6 @@ angular.module('faradayApp') return services; }) - .then(function(vulns) { - $scope.services.forEach(function(service) { - service.vulns = vulns[service.id] || 0; - }); - }) .catch(function(e) { console.log(e); }); From c50494089628772c7587781b06fad1a989433116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 8 Nov 2017 17:42:08 -0300 Subject: [PATCH 0514/1506] Fix ports and version in service list To avoid modifying the javascript, I added a field "ports" in the service schema that behaves the same as the "port" field --- server/api/modules/hosts.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 248f8617d4d..d73198169c9 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -78,11 +78,12 @@ class Meta(FilterSetMeta): class ServiceSchema(AutoSchema): vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) credentials = fields.Integer(attribute='credentials_count', dump_only=True) + ports = fields.Integer(attribute='port') class Meta: model = Service - fields = ('id', 'name', 'description', 'port', 'protocol', 'status', - 'vulns', 'credentials') + fields = ('id', 'name', 'description', 'port', 'ports', 'protocol', + 'status', 'vulns', 'credentials', 'version') class HostsView(PaginatedMixin, From 1b0cdd34e461ae761b9dcedea790db8e37ea4672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 9 Nov 2017 13:03:23 -0300 Subject: [PATCH 0515/1506] Fix workspace checker It wasn't imported anywhere. The tests weren't working properly. I added a message to the asssertion error --- server/events.py | 6 +++++- server/models.py | 5 ++++- test_cases/test_model_events.py | 13 ++++++------- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/server/events.py b/server/events.py index cfd855a0860..5d2c73f99ce 100644 --- a/server/events.py +++ b/server/events.py @@ -6,10 +6,14 @@ def after_insert_check_child_has_same_workspace(mapper, connection, inserted_instance): if inserted_instance.parent: - assert inserted_instance.workspace == inserted_instance.parent.workspace + assert (inserted_instance.workspace == + inserted_instance.parent.workspace), \ + "Conflicting workspace assignation for objects. " \ + "This should never happen!!!" # register the workspace verification for all objs that has workspace_id for name, obj in inspect.getmembers(sys.modules['server.models']): if inspect.isclass(obj) and getattr(obj, 'workspace_id', None): event.listen(obj, 'after_insert', after_insert_check_child_has_same_workspace) + event.listen(obj, 'after_update', after_insert_check_child_has_same_workspace) diff --git a/server/models.py b/server/models.py index 25d39e1f6ca..539d686c8eb 100644 --- a/server/models.py +++ b/server/models.py @@ -1312,4 +1312,7 @@ def log_command_object_found(command, object, created): object_type=object_type, workspace=object.workspace, ) - log.created_persistent=created + log.created_persistent = created + +# We have to import this after all models are defined +import server.events diff --git a/test_cases/test_model_events.py b/test_cases/test_model_events.py index baebce85a26..0074437d5a7 100644 --- a/test_cases/test_model_events.py +++ b/test_cases/test_model_events.py @@ -1,21 +1,20 @@ +import pytest from server import events from test_cases.factories import HostFactory, ServiceFactory -def test_child_parent_verification_event(session, workspace, second_workspace): +def test_child_parent_verification_event_fails(session, workspace, + second_workspace): host = HostFactory.build(workspace=workspace) ServiceFactory.build(host=host, workspace=second_workspace) - try: + with pytest.raises(AssertionError): session.commit() - except AssertionError: - return True - return False -def test_child_parent_verification_event(session, workspace): + +def test_child_parent_verification_event_succeeds(session, workspace): """ Asserts that no exception will be raised when workspace are the same. """ host = HostFactory.build(workspace=workspace) ServiceFactory.build(host=host, workspace=workspace) session.commit() - return True \ No newline at end of file From 9399571cdeca33377ead3a56637bcf9fd6628f25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 9 Nov 2017 14:23:10 -0300 Subject: [PATCH 0516/1506] Fix service creation --- server/api/modules/hosts.py | 1 + server/api/modules/services.py | 22 +++++++++++++------ server/www/scripts/app.js | 10 ++++++++- .../services/controllers/serviceModalNew.js | 7 +++--- .../scripts/services/partials/modalNew.html | 4 +++- test_cases/test_api_services.py | 18 +++++++++++++++ 6 files changed, 50 insertions(+), 12 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index d73198169c9..8fd60160ff3 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -76,6 +76,7 @@ class Meta(FilterSetMeta): class ServiceSchema(AutoSchema): + # TODO migration: use the schema in ./services.py vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) credentials = fields.Integer(attribute='credentials_count', dump_only=True) ports = fields.Integer(attribute='port') diff --git a/server/api/modules/services.py b/server/api/modules/services.py index 239fe402298..bc8803814b8 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -10,8 +10,7 @@ from server.api.base import AutoSchema, ReadWriteWorkspacedView from server.models import Service -from server.utils.logger import get_logger -from server.utils.web import gzipped, validate_workspace, get_integer_parameter +from server.schemas import MutableField, PrimaryKeyRelatedField services_api = Blueprint('services_api', __name__) @@ -22,14 +21,22 @@ class ServiceSchema(AutoSchema): _rev = fields.String(default='', dump_only=True) metadata = fields.Method('get_metadata') owned = fields.Boolean(default=False) - owner = fields.String(dump_only=True, attribute='creator.username') - ports = fields.Method(attribute='port', deserialize='load_port') + owner = PrimaryKeyRelatedField('username', dump_only=True, + attribute='creator') + port = fields.Integer(dump_only=True) # Port is loaded via ports + ports = MutableField(fields.String(), + fields.Method(deserialize='load_port'), + required=True, + attribute='port') status = fields.String(default='open') - parent = fields.Integer(attribute='host_id', load_only=True) + parent = fields.Integer(attribute='host_id', load_only=True, required=True) host_id = fields.Integer(attribute='host_id', dump_only=True) summary = fields.Method('get_summary') + vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) + credentials = fields.Integer(attribute='credentials_count', dump_only=True) def load_port(self, value): + # TODO migration: handle empty list and not numeric value return str(value.pop()) def get_metadata(self, obj): @@ -50,8 +57,8 @@ class Meta: model = Service fields = ('id', '_id', 'status', 'parent', 'protocol', 'description', '_rev', - 'owned', 'owner', 'credentials', - 'name', 'version', '_id', 'ports', + 'owned', 'owner', 'credentials', 'vulns', + 'name', 'version', '_id', 'port', 'ports', 'metadata', 'summary', 'host_id') @@ -60,6 +67,7 @@ class ServiceView(ReadWriteWorkspacedView): model_class = Service schema_class = ServiceSchema count_extra_filters = [Service.status == 'open'] + get_undefer = [Service.credentials_count, Service.vulnerability_count] def _envelope_list(self, objects, pagination_metadata=None): services = [] diff --git a/server/www/scripts/app.js b/server/www/scripts/app.js index 3435bd800b2..5a64bfdfe17 100644 --- a/server/www/scripts/app.js +++ b/server/www/scripts/app.js @@ -48,9 +48,17 @@ var faradayApp = angular.module('faradayApp', ['ngRoute', 'selectionModel', 'ui. ]; return exploitations; })()) + .constant("SERVICE_STATUSES", (function() { + var statuses = [ + "open", + "closed", + "filtered" + ]; + return statuses; + })()) .constant("STATUSES", (function() { var statuses = [ - "opened", + "opened", // TODO migration: should we change this to "open"? "closed", "re-opened", "risk-accepted" diff --git a/server/www/scripts/services/controllers/serviceModalNew.js b/server/www/scripts/services/controllers/serviceModalNew.js index f407a2a8527..913d41d40ed 100644 --- a/server/www/scripts/services/controllers/serviceModalNew.js +++ b/server/www/scripts/services/controllers/serviceModalNew.js @@ -4,8 +4,8 @@ angular.module('faradayApp') .controller('serviceModalNew', - ['$scope', '$modalInstance', '$routeParams', 'host', 'servicesManager', 'hostsManager', - function($scope, $modalInstance, $routeParams, host, servicesManager, hostsManager) { + ['$scope', '$modalInstance', '$routeParams', 'SERVICE_STATUSES', 'host', 'servicesManager', 'hostsManager', + function($scope, $modalInstance, $routeParams, SERVICE_STATUSES, host, servicesManager, hostsManager) { init = function() { $scope.service = { @@ -22,6 +22,7 @@ angular.module('faradayApp') // current Workspace var ws = $routeParams.wsId; $scope.service.parent = host.id; + $scope.statuses = SERVICE_STATUSES; }; @@ -47,4 +48,4 @@ angular.module('faradayApp') }; init(); - }]); \ No newline at end of file + }]); diff --git a/server/www/scripts/services/partials/modalNew.html b/server/www/scripts/services/partials/modalNew.html index 4d966decc87..8141a8f8f40 100644 --- a/server/www/scripts/services/partials/modalNew.html +++ b/server/www/scripts/services/partials/modalNew.html @@ -55,7 +55,9 @@
    Version
    Status
    - +
    diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index ddd2d789e07..725065b2c43 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -46,3 +46,21 @@ def test_service_list_backwards_compatibility(self, test_client, second_workspac expected = set(object_properties) result = set(service['value'].keys()) assert expected <= result + + def test_create_service(self, test_client, host, session): + session.commit() + data = { + "name": "ftp", + "description": "test. test", + "owned": False, + "ports": [21], + "protocol": "tcp", + "status": "open", + "parent": host.id + } + res = test_client.post(self.url(), data=data) + assert res.status_code == 201 + service = Service.query.get(res.json['_id']) + assert service.name == "ftp" + assert service.port == 21 + assert service.host is host From c3f4ee057e78240fd8577fe287d3464227d91414 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 9 Nov 2017 14:56:22 -0300 Subject: [PATCH 0517/1506] Add MetadaSchema for metadara serialization --- server/api/modules/credentials.py | 18 ++---------- server/api/modules/hosts.py | 21 ++------------ server/api/modules/services.py | 17 ++--------- server/api/modules/vulns.py | 22 ++------------- server/schemas.py | 27 +++++++++++++++++- test_cases/test_api_vulnerability.py | 42 ++++++++++++++++++++++++++-- 6 files changed, 75 insertions(+), 72 deletions(-) diff --git a/server/api/modules/credentials.py b/server/api/modules/credentials.py index d1962eb3fea..aab779cc080 100644 --- a/server/api/modules/credentials.py +++ b/server/api/modules/credentials.py @@ -1,9 +1,6 @@ # Faraday Penetration Test IDE # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information -import time - -import flask from flask import Blueprint from marshmallow import fields, post_load, ValidationError from filteralchemy import FilterSet, operators @@ -14,7 +11,7 @@ FilterSetMeta, FilterAlchemyMixin, InvalidUsage) from server.models import Credential, Host, Service, db -from server.schemas import MutableField +from server.schemas import MutableField, SelfNestedField, MetadataSchema credentials_api = Blueprint('credentials_api', __name__) @@ -22,7 +19,6 @@ class CredentialSchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') _rev = fields.String(default='', dump_only=True) - metadata = fields.Method('get_metadata') owned = fields.Boolean(default=False) owner = fields.String(dump_only=True, attribute='creator.username', default='') username = fields.String(default='') @@ -39,6 +35,7 @@ class CredentialSchema(AutoSchema): # for filtering host_id = fields.Integer(load_only=True) service_id = fields.Integer(load_only=True) + metadata = SelfNestedField(MetadataSchema()) def get_parent(self, obj): return obj.host_id or obj.service_id @@ -47,17 +44,6 @@ def get_parent_type(self, obj): assert obj.host_id is not None or obj.service_id is not None return 'Service' if obj.service_id is not None else 'Host' - def get_metadata(self, obj): - return { - "command_id": "e1a042dd0e054c1495e1c01ced856438", - "create_time": time.mktime(obj.create_date.utctimetuple()), - "creator": "Metasploit", - "owner": "", "update_action": 0, - "update_controller_action": "No model controller call", - "update_time": time.mktime(obj.update_date.utctimetuple()), - "update_user": "" - } - class Meta: model = Credential diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index d73198169c9..28318b09a1b 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -7,12 +7,9 @@ from flask_classful import route from marshmallow import fields from filteralchemy import FilterSet, operators -from sqlalchemy.orm import undefer, joinedload from server.utils.database import get_or_create -from server.utils.logger import get_logger -from server.utils.web import gzipped, validate_workspace,\ - get_integer_parameter, filter_request_args + from server.api.base import ( ReadWriteWorkspacedView, PaginatedMixin, @@ -20,7 +17,7 @@ FilterAlchemyMixin, FilterSetMeta, ) -from server.schemas import PrimaryKeyRelatedField +from server.schemas import PrimaryKeyRelatedField, MetadataSchema, SelfNestedField from server.models import Host, Service, db, Hostname host_api = Blueprint('host_api', __name__) @@ -33,7 +30,6 @@ class HostSchema(AutoSchema): ip = fields.String(default='') description = fields.String(required=True) # Explicitly set required=True default_gateway = fields.List(fields.String, attribute="default_gateway_ip") - metadata = fields.Method('get_metadata') name = fields.String(dump_only=True, attribute='ip', default='') os = fields.String(default='') owned = fields.Boolean(default=False) @@ -46,18 +42,7 @@ class HostSchema(AutoSchema): # TODO migration: make it writable dump_only=True, # Only for now default=[]) - - def get_metadata(self, obj): - return { - "command_id": None, - "create_time": None, - "creator": None, - "owner": None, - "update_action": None, - "update_controller_action": None, - "update_time": 1504796508.21, - "update_user": None - } + metadata = SelfNestedField(MetadataSchema()) class Meta: model = Host diff --git a/server/api/modules/services.py b/server/api/modules/services.py index 239fe402298..87974f38d93 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -10,9 +10,7 @@ from server.api.base import AutoSchema, ReadWriteWorkspacedView from server.models import Service -from server.utils.logger import get_logger -from server.utils.web import gzipped, validate_workspace, get_integer_parameter - +from server.schemas import MetadataSchema, SelfNestedField services_api = Blueprint('services_api', __name__) @@ -20,7 +18,6 @@ class ServiceSchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') _rev = fields.String(default='', dump_only=True) - metadata = fields.Method('get_metadata') owned = fields.Boolean(default=False) owner = fields.String(dump_only=True, attribute='creator.username') ports = fields.Method(attribute='port', deserialize='load_port') @@ -28,21 +25,11 @@ class ServiceSchema(AutoSchema): parent = fields.Integer(attribute='host_id', load_only=True) host_id = fields.Integer(attribute='host_id', dump_only=True) summary = fields.Method('get_summary') + metadata = SelfNestedField(MetadataSchema()) def load_port(self, value): return str(value.pop()) - def get_metadata(self, obj): - return { - "command_id": "e1a042dd0e054c1495e1c01ced856438", - "create_time": time.mktime(obj.create_date.utctimetuple()), - "creator": "Metasploit", - "owner": "", "update_action": 0, - "update_controller_action": "No model controller call", - "update_time": time.mktime(obj.update_date.utctimetuple()), - "update_user": "" - } - def get_summary(self, obj): return "(%s/%s) %s" % (obj.port, obj.protocol, obj.name) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 414a9bd79ea..f0758fbddc4 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -25,8 +25,6 @@ from server.fields import FaradayUploadedFile from server.models import ( db, - Tag, - TagObject, Vulnerability, VulnerabilityWeb, VulnerabilityGeneric, @@ -38,7 +36,7 @@ MutableField, PrimaryKeyRelatedField, SelfNestedField, -) + MetadataSchema) vulns_api = Blueprint('vulns_api', __name__) logger = logging.getLogger(__name__) @@ -91,7 +89,6 @@ class VulnerabilitySchema(AutoSchema): tags = PrimaryKeyRelatedField('name', dump_only=True, many=True) easeofresolution = fields.String(attribute='ease_of_resolution', validate=OneOf(Vulnerability.EASE_OF_RESOLUTIONS),) hostnames = PrimaryKeyRelatedField('name', many=True, dump_only=True) - metadata = fields.Method(serialize='get_metadata') service = fields.Nested(ServiceSchema(only=[ '_id', 'ports', 'status', 'protocol', 'name', 'version', 'summary' ]), dump_only=True) @@ -101,6 +98,7 @@ class VulnerabilitySchema(AutoSchema): type = fields.Method(serialize='get_type', deserialize='load_type') obj_id = fields.String(dump_only=True, attribute='id') target = fields.Method('get_target') + metadata = SelfNestedField(MetadataSchema()) class Meta: model = Vulnerability @@ -118,22 +116,6 @@ class Meta: def get_type(self, obj): return obj.__class__.__name__ - def get_metadata(self, obj): - command_id = None - command_obj = CommandObject.query.filter_by(object_type='vulnerability', object_id=obj.id, workspace=obj.workspace).first() - if command_obj: - command_id = command_obj.command_id - return { - "command_id": command_id, - "create_time": time.mktime(obj.create_date.utctimetuple()), - "creator": "", - "owner": obj.creator and obj.creator.username or '', - "update_action": 0, - "update_controller_action": "", - "update_time": time.mktime(obj.update_date.utctimetuple()), - "update_user": "" - } - def get_attachments(self, obj): res = [] diff --git a/server/schemas.py b/server/schemas.py index 940756e0e4a..a985a0d59a4 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -1,7 +1,10 @@ import time -from marshmallow import fields +from marshmallow import fields, Schema from marshmallow.exceptions import ValidationError +from server.api.base import AutoSchema +from server.models import CommandObject + class JSTimestampField(fields.Field): """A field to serialize datetime objects into javascript @@ -97,3 +100,25 @@ def _add_to_schema(self, field_name, schema): super(MutableField, self)._add_to_schema(field_name, schema) self.read_field._add_to_schema(field_name, schema) self.write_field._add_to_schema(field_name, schema) + + +class MetadataSchema(Schema): + command_id = fields.Method('get_command_id', dump_only=True) + + creator = fields.Function(lambda x: '') + owner = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator') + + create_time = JSTimestampField(attribute='create_date', dump_only=True) + update_time = JSTimestampField(attribute='update_date', dump_only=True) + + update_user = fields.String(default='', dump_only=True) + update_action = fields.Integer(default=0, dump_only=True) + update_controller_action = fields.String(default='', dump_only=True) + + def get_command_id(self, obj): + command_id = None + command_obj = CommandObject.query.filter_by(object_type='vulnerability', object_id=obj.id, workspace=obj.workspace).first() + if command_obj: + command_id = command_obj.command_id + + return command_id \ No newline at end of file diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 14b8c347693..977587850b1 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -3,6 +3,7 @@ from base64 import b64encode import pytest +import time from server.api.modules.vulns import VulnerabilityView from test_cases import factories @@ -13,7 +14,8 @@ Vulnerability, VulnerabilityWeb, Reference, PolicyViolation, CommandObject) -from test_cases.factories import ServiceFactory, CommandFactory, CommandObjectFactory, HostFactory, EmptyCommandFactory +from test_cases.factories import ServiceFactory, CommandFactory, CommandObjectFactory, HostFactory, EmptyCommandFactory, \ + UserFactory, VulnerabilityWebFactory CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) @@ -805,4 +807,40 @@ def test_filter_by_command_id(self, test_client, session, workspace=second_workspace, ).first() vuln['metadata']['command_id'] == command_object.command.id - assert set(vuln['_id'] for vuln in res.json['data']) == web_expected_ids \ No newline at end of file + assert set(vuln['_id'] for vuln in res.json['data']) == web_expected_ids + + @pytest.mark.usefixtures('ignore_nplusone') + def test_vulnerability_metadata(self, session, test_client, workspace): + owner = UserFactory.create() + service = ServiceFactory.create(workspace=workspace) + command = EmptyCommandFactory.create(id=5555, workspace=workspace) + + vuln = VulnerabilityWebFactory.create( + service=service, + creator = owner, + workspace=workspace + ) + session.flush() + CommandObjectFactory.create( + command=command, + object_type='vulnerability', + object_id=vuln.id, + workspace=workspace + ) + + res = test_client.get(self.url()) + assert res.status_code == 200 + from_json_vuln = filter(lambda raw_vuln: raw_vuln['id'] == 6, res.json['vulnerabilities']) + assert 'metadata' in from_json_vuln[0]['value'] + expected_metadata = { + u'command_id': 5555, + u'create_time': int(time.mktime(vuln.create_date.timetuple()) * 1000), + u'creator': u'', + u'owner': owner.username, + u'update_action': 0, + u'update_controller_action': u'', + u'update_time': int(time.mktime(vuln.update_date.timetuple()) * 1000), + u'update_user': u'' + } + + assert expected_metadata == from_json_vuln[0]['value']['metadata'] \ No newline at end of file From 8e1f6a733850a181d3d4266e0097b190eac2c585 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 9 Nov 2017 15:16:40 -0300 Subject: [PATCH 0518/1506] Set metadata in couchdb importer --- server/importer.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/server/importer.py b/server/importer.py index 41e8fd83858..452fbdadde0 100644 --- a/server/importer.py +++ b/server/importer.py @@ -180,6 +180,24 @@ def create_tags(raw_tags, parent_id, parent_type): session.commit() +def set_metadata(document, obj): + if 'metadata' in document: + for key, value in document['metadata'].iteritems(): + if not value: + continue + try: + if key == 'create_time': + obj.create_date = datetime.datetime.fromtimestamp(document['metadata']['create_time']) + if key == 'owner': + creator = User.query.filter_by(username=value) + obj.creator = creator + except ValueError: + if key == 'create_time': + obj.create_date = datetime.datetime.fromtimestamp(document['metadata']['create_time'] / 1000) + except TypeError: + print('') + + class EntityNotFound(Exception): def __init__(self, entity_id): super(EntityNotFound, self).__init__("Entity (%s) wasn't found" % entity_id) @@ -716,6 +734,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation task.description = document.get('description') task.assigned_to = session.query(User).filter_by(username=document.get('username')).first() + mapped_status = { 'New': 'new', 'In Progress': 'in progress', @@ -1144,6 +1163,7 @@ def import_workspace_into_database(self, workspace_name): for new_obj in obj_importer.update_from_document(raw_obj, workspace, level, couchdb_relational_map): if not new_obj: continue + set_metadata(raw_obj, new_obj) session.commit() couchdb_relational_map_by_type[couchdb_id].append({'type': obj_type, 'id': new_obj.id}) couchdb_relational_map[couchdb_id].append(new_obj.id) From 18f31664c8677f748a0f0d203d655326e6f7cc89 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 9 Nov 2017 15:17:01 -0300 Subject: [PATCH 0519/1506] Move attachment importer to a method --- server/importer.py | 53 ++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/server/importer.py b/server/importer.py index 452fbdadde0..45470b1eb8f 100644 --- a/server/importer.py +++ b/server/importer.py @@ -528,35 +528,38 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation if len(tags): create_tags(tags, vulnerability.id, document['type']) - attachments_data = document.get('_attachments') or {} - for attachment_name, attachment_data in attachments_data.items(): - #http://localhost:5984/evidence/334389048b872a533002b34d73f8c29fd09efc50.c7b0f6cba2fae8e446b7ffedfdb18026bb9ba41d/forbidden.png - attachment_url = "http://{username}:{password}@{hostname}:{port}/{path}".format( - username=server.config.couchdb.user, - password=server.config.couchdb.password, - hostname=server.config.couchdb.host, - port=server.config.couchdb.port, - path='{0}/{1}/{2}'.format(workspace.name, document.get('_id'), attachment_name) - ) - response = requests.get(attachment_url) - response.raw.decode_content = True - attachment_file = NamedTemporaryFile() - attachment_file.write(response.content) - attachment_file.seek(0) - session.commit() - file, created = get_or_create( - session, - File, - filename=attachment_name, - object_id=vulnerability.id, - object_type=vulnerability.__class__.__name__) - file.content = attachment_file.read() - - attachment_file.close() + self.add_attachments(document, vulnerability, workspace) yield vulnerability + def add_attachments(self, document, vulnerability, workspace): + attachments_data = document.get('_attachments') or {} + for attachment_name, attachment_data in attachments_data.items(): + # http://localhost:5984/evidence/334389048b872a533002b34d73f8c29fd09efc50.c7b0f6cba2fae8e446b7ffedfdb18026bb9ba41d/forbidden.png + attachment_url = "http://{username}:{password}@{hostname}:{port}/{path}".format( + username=server.config.couchdb.user, + password=server.config.couchdb.password, + hostname=server.config.couchdb.host, + port=server.config.couchdb.port, + path='{0}/{1}/{2}'.format(workspace.name, document.get('_id'), attachment_name) + ) + response = requests.get(attachment_url) + response.raw.decode_content = True + attachment_file = NamedTemporaryFile() + attachment_file.write(response.content) + attachment_file.seek(0) + session.commit() + file, created = get_or_create( + session, + File, + filename=attachment_name, + object_id=vulnerability.id, + object_type=vulnerability.__class__.__name__) + file.content = attachment_file.read() + + attachment_file.close() + def add_policy_violations(self, document, vulnerability, workspace): policy_violations = set() for policy_violation in document.get('policyviolations', []): From dda1d9efa410264b6e8996dc8883ce0c253b9076 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 9 Nov 2017 15:17:29 -0300 Subject: [PATCH 0520/1506] Add ignore nplusone. We need to optimize command objects --- test_cases/test_api_credentials.py | 3 +++ test_cases/test_api_hosts.py | 3 +++ test_cases/test_api_pagination.py | 2 ++ test_cases/test_api_services.py | 1 + 4 files changed, 9 insertions(+) diff --git a/test_cases/test_api_credentials.py b/test_cases/test_api_credentials.py index f9b473a8dec..9865effe252 100644 --- a/test_cases/test_api_credentials.py +++ b/test_cases/test_api_credentials.py @@ -1,3 +1,5 @@ +import pytest + from test_cases import factories from test_api_workspaced_base import ( ReadOnlyAPITests, @@ -14,6 +16,7 @@ class TestCredentialsAPIGeneric(ReadOnlyAPITests): api_endpoint = 'credential' update_fields = ['username', 'password'] + @pytest.mark.usefixtures('ignore_nplusone') def test_get_list_backwards_compatibility(self, test_client, session, second_workspace): cred = self.factory.create(workspace=second_workspace) session.commit() diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 6062a10e666..c8bc34885f8 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -59,6 +59,7 @@ def compare_results(self, hosts, response): hosts_in_response = set(host['id'] for host in response.json['rows']) assert hosts_in_list == hosts_in_response + @pytest.mark.usefixtures('ignore_nplusone') def test_list_retrieves_all_items_from_workspace(self, test_client, second_workspace, session, @@ -211,6 +212,7 @@ def test_retrieve_shows_service_count(self, test_client, host_services, res = test_client.get(self.url(host)) assert res.json['services'] == len(services) + @pytest.mark.usefixtures('ignore_nplusone') def test_index_shows_service_count(self, test_client, host_services, service_factory): ids_map = {host.id: services @@ -270,6 +272,7 @@ def test_filter_by_os_like_ilike(self, test_client, session, workspace, assert res.status_code == 200 self.compare_results(hosts + [case_insensitive_host], res) + @pytest.mark.usefixtures('ignore_nplusone') def test_host_with_open_vuln_count_verification(self, test_client, session, workspace, host_factory, vulnerability_factory, service_factory): diff --git a/test_cases/test_api_pagination.py b/test_cases/test_api_pagination.py index cd60aa2e387..79fd7ea83a3 100644 --- a/test_cases/test_api_pagination.py +++ b/test_cases/test_api_pagination.py @@ -38,6 +38,7 @@ def page_url(self, page_number=None, per_page=None): @pytest.mark.parametrize("page_number", [None, 1, 2]) @pytest.mark.usefixtures('pagination_test_logic') + @pytest.mark.usefixtures('ignore_nplusone') @pytest.mark.pagination def test_returns_all_with_no_per_page(self, test_client, session, page_number): @@ -69,6 +70,7 @@ def test_does_not_allow_negative_page_number(self, session, test_client, assert res.json == {u'data': []} @pytest.mark.usefixtures('pagination_test_logic') + @pytest.mark.usefixtures('ignore_nplusone') @pytest.mark.pagination def test_pages_have_different_elements(self, session, test_client): """Test correct page size, correct IDs and that there are diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index ddd2d789e07..764027118f6 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -22,6 +22,7 @@ class TestListServiceView(ReadOnlyAPITests): #update_fields = ['ip', 'description', 'os'] view_class = ServiceView + @pytest.mark.usefixtures('ignore_nplusone') def test_service_list_backwards_compatibility(self, test_client, second_workspace, session): self.factory.create(workspace=second_workspace) session.commit() From 6ffc72ca9171214d2332a1ba1cb5d202fac36183 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 9 Nov 2017 15:38:49 -0300 Subject: [PATCH 0521/1506] Fix metadata importer set creator --- server/importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index 45470b1eb8f..9e5072bb4a9 100644 --- a/server/importer.py +++ b/server/importer.py @@ -189,7 +189,7 @@ def set_metadata(document, obj): if key == 'create_time': obj.create_date = datetime.datetime.fromtimestamp(document['metadata']['create_time']) if key == 'owner': - creator = User.query.filter_by(username=value) + creator = User.query.filter_by(username=value).first() obj.creator = creator except ValueError: if key == 'create_time': From 1c68727859cda047f19b90447ef28c6b45ab7a3f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 9 Nov 2017 15:39:14 -0300 Subject: [PATCH 0522/1506] Fix a problem with workspaces and tasks/methodology in the importer --- server/importer.py | 11 +++++------ server/models.py | 8 ++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/server/importer.py b/server/importer.py index 9e5072bb4a9..a3de4374c4c 100644 --- a/server/importer.py +++ b/server/importer.py @@ -699,12 +699,10 @@ class MethodologyImporter(object): def update_from_document(self, document, workspace, level=None, couchdb_relational_map=None): if document.get('group_type') == 'template': methodology, created = get_or_create(session, MethodologyTemplate, name=document.get('name')) - methodology.workspace = workspace yield methodology if document.get('group_type') == 'instance': - methodology, created = get_or_create(session, Methodology, name=document.get('name')) - methodology.workspace = workspace + methodology, created = get_or_create(session, Methodology, name=document.get('name'), workspace=workspace) yield methodology @@ -723,17 +721,18 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation if len(couchdb_relational_map[document.get('group_id')]) > 1: logger.error('It was expected only one parent in methodology {0}'.format(document.get('_id'))) - methodology = session.query(Methodology).filter_by(id=methodology_id).first() + methodology = session.query(Methodology).filter_by(id=methodology_id, workspace=workspace).first() task_class = Task if not methodology: methodology = session.query(MethodologyTemplate).filter_by(id=methodology_id).first() task_class = TaskTemplate - task, task_created = get_or_create(session, task_class, name=document.get('name')) + if task_class == TaskTemplate: + task, task_created = get_or_create(session, task_class, name=document.get('name')) task.template = methodology else: + task, task_created = get_or_create(session, task_class, name=document.get('name'), workspace=workspace) task.methodology = methodology - task.workspace = workspace task.description = document.get('description') task.assigned_to = session.query(User).filter_by(username=document.get('username')).first() diff --git a/server/models.py b/server/models.py index 539d686c8eb..d3e26aa7eeb 100644 --- a/server/models.py +++ b/server/models.py @@ -1120,6 +1120,10 @@ class Methodology(Metadata): workspace = relationship('Workspace', backref='methodologies') workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) + @property + def parent(self): + return + class TaskABC(Metadata): __abstract__ = True @@ -1194,6 +1198,10 @@ class Task(TaskABC): # UniqueConstraint(TaskABC.name, methodology_id, workspace_id, name='uix_task_name_desc_methodology_workspace'), # ) + @property + def parent(self): + return self.methodology + class License(Metadata): __tablename__ = 'license' From cce2cf0daafb6e89a04f0a2b50e00d0acee00fb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 9 Nov 2017 15:48:04 -0300 Subject: [PATCH 0523/1506] Add workspace check in host_id when creating services. I had to _get_schema_instance and _set_schema_context methods on GenericView and to modify the code to use them. --- server/api/base.py | 43 +++++++++++++++++++++++++++------ server/api/modules/services.py | 23 ++++++++++++++++-- test_cases/test_api_services.py | 28 +++++++++++++++++---- 3 files changed, 79 insertions(+), 15 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index d2d8e61ba89..2994ccad478 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -74,6 +74,28 @@ def _get_schema_class(self): assert self.schema_class is not None, "You must define schema_class" return self.schema_class + def _get_schema_instance(self, route_kwargs, **kwargs): + """Instances a model schema. + + By default it uses sets strict to True + but this can be overriden as well as any other parameters in + the function's kwargs. + + It also uses _set_schema_context to set the context of the + schema. + """ + if 'strict' not in kwargs: + kwargs['strict'] = True + kwargs['context'] = self._set_schema_context( + kwargs.get('context', {}), **route_kwargs) + return self._get_schema_class()(**kwargs) + + def _set_schema_context(self, context, **kwargs): + """This function can be overriden to update the context passed + to the schema. + """ + return context + def _get_lookup_field(self): return getattr(self.model_class, self.lookup_field) @@ -121,8 +143,8 @@ def _get_object(self, object_id, eagerload=False, **kwargs): flask.abort(404, 'Object with id "%s" not found' % object_id) return obj - def _dump(self, obj, **kwargs): - return self._get_schema_class()(**kwargs).dump(obj).data + def _dump(self, obj, route_kwargs, **kwargs): + return self._get_schema_instance(route_kwargs, **kwargs).dump(obj).data def _parse_data(self, schema, request, *args, **kwargs): return FlaskParser().parse(schema, request, locations=('json',), @@ -190,6 +212,11 @@ def _get_object(self, object_id, workspace_name, eagerload=False): flask.abort(404, 'Object with id "%s" not found' % object_id) return obj + def _set_schema_context(self, context, **kwargs): + """Overriden to pass the workspace name to the schema""" + context.update(kwargs) + return context + def _validate_uniqueness(self, obj, object_id=None): # TODO: Use implementation of GenericView assert obj.workspace is not None, "Object must have a " \ @@ -231,7 +258,7 @@ def index(self, **kwargs): if self.order_field is not None: query = query.order_by(self.order_field) objects, pagination_metadata = self._paginate(query) - return self._envelope_list(self._dump(objects, many=True), + return self._envelope_list(self._dump(objects, kwargs, many=True), pagination_metadata) @@ -286,7 +313,7 @@ class RetrieveMixin(object): def get(self, object_id, **kwargs): return self._dump(self._get_object(object_id, eagerload=True, - **kwargs)) + **kwargs), kwargs) class RetrieveWorkspacedMixin(RetrieveMixin): @@ -314,12 +341,12 @@ class CreateMixin(object): """Add POST / route""" def post(self, **kwargs): - data = self._parse_data(self._get_schema_class()(strict=True), + data = self._parse_data(self._get_schema_instance(kwargs), flask.request) created = self._perform_create(data, **kwargs) created.creator = g.user db.session.commit() - return self._dump(created), 201 + return self._dump(created, kwargs), 201 def _perform_create(self, data, **kwargs): obj = self.model_class(**data) @@ -355,12 +382,12 @@ class UpdateMixin(object): """Add PUT /// route""" def put(self, object_id, **kwargs): - data = self._parse_data(self._get_schema_class()(strict=True), + data = self._parse_data(self._get_schema_instance(kwargs), flask.request) obj = self._get_object(object_id, **kwargs) self._update_object(obj, data) updated = self._perform_update(object_id, obj, **kwargs) - return self._dump(obj), 200 + return self._dump(obj, kwargs), 200 def _update_object(self, obj, data): for (key, value) in data.items(): diff --git a/server/api/modules/services.py b/server/api/modules/services.py index bc8803814b8..2006c5af272 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -5,11 +5,12 @@ import flask from flask import Blueprint -from marshmallow import fields +from marshmallow import fields, post_load, ValidationError from sqlalchemy.orm import joinedload +from sqlalchemy.orm.exc import NoResultFound from server.api.base import AutoSchema, ReadWriteWorkspacedView -from server.models import Service +from server.models import Host, Service, Workspace from server.schemas import MutableField, PrimaryKeyRelatedField @@ -53,6 +54,24 @@ def get_metadata(self, obj): def get_summary(self, obj): return "(%s/%s) %s" % (obj.port, obj.protocol, obj.name) + @post_load + def post_load_parent(self, data): + """Gets the host_id from parent attribute. Pops it and tries to + get a Host with that id in the corresponding workspace. + """ + host_id = data.pop('host_id', None) + print data + if host_id is None: + # Partial update? + return + try: + data['host'] = Host.query.join(Workspace).filter( + Workspace.name == self.context['workspace_name'], + Host.id == host_id + ).one() + except NoResultFound: + raise ValidationError('Host with id {} not found'.format(host_id)) + class Meta: model = Service fields = ('id', '_id', 'status', 'parent', diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index 725065b2c43..0458019d844 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -1,16 +1,14 @@ -#-*- coding: utf8 -*- +# -*- coding: utf8 -*- """Tests for many API endpoints that do not depend on workspace_name""" import pytest from server.api.modules.services import ServiceView from test_cases import factories -from test_api_workspaced_base import API_PREFIX, ReadOnlyAPITests +from test_api_workspaced_base import ReadOnlyAPITests from server.models import ( Service ) -from server.api.modules.commandsrun import CommandView -from server.api.modules.workspaces import WorkspaceView @pytest.mark.usefixtures('logged_user') @@ -22,7 +20,8 @@ class TestListServiceView(ReadOnlyAPITests): #update_fields = ['ip', 'description', 'os'] view_class = ServiceView - def test_service_list_backwards_compatibility(self, test_client, second_workspace, session): + def test_service_list_backwards_compatibility(self, test_client, + second_workspace, session): self.factory.create(workspace=second_workspace) session.commit() res = test_client.get(self.url()) @@ -64,3 +63,22 @@ def test_create_service(self, test_client, host, session): assert service.name == "ftp" assert service.port == 21 assert service.host is host + + def test_create_fails_with_host_of_other_workspace(self, test_client, + host, session, + second_workspace): + # TODO migration: check why workspace checker isn't triggering an error + # here + session.commit() + assert host.workspace_id != second_workspace.id + data = { + "name": "ftp", + "description": "test. test", + "owned": False, + "ports": [21], + "protocol": "tcp", + "status": "open", + "parent": host.id + } + res = test_client.post(self.url(workspace=second_workspace), data=data) + assert res.status_code == 400 From 5783cb6482d74dfd8ba39bcdefb42b2668624df3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 9 Nov 2017 15:55:14 -0300 Subject: [PATCH 0524/1506] Add tests for Host model I forgot to add them a days ago --- test_cases/models/test_host.py | 41 ++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 test_cases/models/test_host.py diff --git a/test_cases/models/test_host.py b/test_cases/models/test_host.py new file mode 100644 index 00000000000..d008484948d --- /dev/null +++ b/test_cases/models/test_host.py @@ -0,0 +1,41 @@ +import random +import pytest +from functools import partial + + +@pytest.mark.parametrize( + "with_host_vulns,with_service_vulns", [[True, False], + [False, True], + [True, True]], + ids=["with host vulnerabilities", + "with service vulnerabilities", + "with host and service vulnerabilities"] +) +def test_vulnerability_count(with_host_vulns, with_service_vulns, host, + session, service_factory, + vulnerability_factory, vulnerability_web_factory): + expected_count = 0 + + if with_host_vulns: + vulnerability_factory.create_batch(8, host=host, service=None, + workspace=host.workspace) + expected_count += 8 + + if with_service_vulns: + services = service_factory.create_batch(10, workspace=host.workspace, + host=host) + for service in services: + for _ in range(5): + # Randomly pick between a standard or a web vuln + create = random.choice([ + vulnerability_web_factory.create, + partial(vulnerability_factory.create, host=None) + ]) + create( + service=service, + workspace=host.workspace + ) + expected_count += 1 + + session.commit() + assert host.vulnerability_count == expected_count From 95e353323cfd9f46ca8c2d3c59abfd8702b2f7a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 9 Nov 2017 16:13:31 -0300 Subject: [PATCH 0525/1506] Add message to faraday_schema_display command So the user knows where the PNG was saved. Also add commented line explained how to show model names only instead of model and field names --- server/commands/faraday_schema_display.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/commands/faraday_schema_display.py b/server/commands/faraday_schema_display.py index 5bebf7ef529..a402f19764e 100644 --- a/server/commands/faraday_schema_display.py +++ b/server/commands/faraday_schema_display.py @@ -1,3 +1,4 @@ +from __future__ import print_function import sys from flask_script import Command from sqlalchemy import MetaData @@ -50,6 +51,8 @@ def _draw_uml_class_diagram(self): graph = create_uml_graph( mappers, show_operations=False, # not necessary in this case - show_multiplicity_one=False # some people like to see the ones, some don't + show_multiplicity_one=False, # some people like to see the ones, some don't + # show_attributes=False, # Uncomment to don't show fields, only model names ) graph.write_png('uml_schema.png') # write out the file + print("Graph written to fle uml_schema.png") From 8062c83261d9d7e39bae301c1f03e566afcccc73 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 9 Nov 2017 16:42:52 -0300 Subject: [PATCH 0526/1506] Add parent properties --- server/models.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/models.py b/server/models.py index d3e26aa7eeb..ab28374b05d 100644 --- a/server/models.py +++ b/server/models.py @@ -1006,6 +1006,10 @@ class Scope(Metadata): name='uix_scope_name_workspace'), ) + @property + def parent(self): + return + def is_valid_workspace(workspace_name): return db.session.query(server.models.Workspace).filter_by(name=workspace_name).first() is not None @@ -1256,6 +1260,10 @@ class Comment(Metadata): object_id = Column(Integer, nullable=False) object_type = Column(Text, nullable=False) + @property + def parent(self): + return + class ExecutiveReport(Metadata): STATUSES = [ @@ -1282,6 +1290,10 @@ class ExecutiveReport(Metadata): workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship('Workspace', foreign_keys=[workspace_id]) + @property + def parent(self): + return + # This constraint uses Columns from different classes # Since it applies to the table vulnerability it should be adVulnerability.ded to the Vulnerability class From 9faa314cd12c7dc5414bd086dbbb0f9cf2644780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 9 Nov 2017 17:49:35 -0300 Subject: [PATCH 0527/1506] Add workspace checking on credential and vuln parents --- server/api/modules/credentials.py | 13 +++--- server/api/modules/vulns.py | 20 +++++++--- test_cases/test_api_credentials.py | 54 ++++++++++++++++++++++++- test_cases/test_api_services.py | 23 ++++++++++- test_cases/test_api_vulnerability.py | 60 +++++++++++++++++++++++++++- 5 files changed, 156 insertions(+), 14 deletions(-) diff --git a/server/api/modules/credentials.py b/server/api/modules/credentials.py index aab779cc080..c9fba8a5c74 100644 --- a/server/api/modules/credentials.py +++ b/server/api/modules/credentials.py @@ -4,13 +4,14 @@ from flask import Blueprint from marshmallow import fields, post_load, ValidationError from filteralchemy import FilterSet, operators +from sqlalchemy.orm.exc import NoResultFound from server.api.base import ( AutoSchema, ReadWriteWorkspacedView, FilterSetMeta, FilterAlchemyMixin, InvalidUsage) -from server.models import Credential, Host, Service, db +from server.models import Credential, Host, Service, Workspace, db from server.schemas import MutableField, SelfNestedField, MetadataSchema credentials_api = Blueprint('credentials_api', __name__) @@ -44,7 +45,6 @@ def get_parent_type(self, obj): assert obj.host_id is not None or obj.service_id is not None return 'Service' if obj.service_id is not None else 'Host' - class Meta: model = Credential fields = ('id', '_id', "_rev", 'parent', @@ -67,9 +67,12 @@ def set_parent(self, data): else: raise ValidationError( 'Unknown parent type: {}'.format(parent_type)) - parent = db.session.query(parent_class).filter_by(id=parent_id).first() - if not parent: - raise InvalidUsage('Parent not found.') + try: + parent = db.session.query(parent_class).join(Workspace).filter( + Workspace.name == self.context['workspace_name'], + parent_class.id == parent_id).one() + except NoResultFound: + raise InvalidUsage('Parent id not found: {}'.format(parent_id)) data[parent_field] = parent.id return data diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index f0758fbddc4..9536cf75bbe 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -13,6 +13,7 @@ from marshmallow.validate import OneOf from sqlalchemy import and_ from sqlalchemy.orm import joinedload, selectin_polymorphic +from sqlalchemy.orm.exc import NoResultFound from depot.manager import DepotManager from server.api.base import ( @@ -25,10 +26,15 @@ from server.fields import FaradayUploadedFile from server.models import ( db, + CommandObject, + File, + Host, + Service, Vulnerability, VulnerabilityWeb, VulnerabilityGeneric, - Host, Service, File, CommandObject) + Workspace +) from server.utils.database import get_or_create from server.api.modules.services import ServiceSchema @@ -206,9 +212,13 @@ def post_load_parent(self, data): print('parent_type', parent_type) raise ValidationError('Unknown parent type') - parent = db.session.query(parent_class).filter_by(id=parent_id).first() - if not parent: - raise ValidationError('Parent not found') + try: + parent = db.session.query(parent_class).join(Workspace).filter( + Workspace.name == self.context['workspace_name'], + parent_class.id == parent_id + ).one() + except NoResultFound: + raise ValidationError('Parent id not found: {}'.format(parent_id)) data[parent_field] = parent.id return data @@ -298,7 +308,7 @@ class VulnerabilityView(PaginatedMixin, } def _perform_create(self, data, **kwargs): - data = self._parse_data(self._get_schema_class()(strict=True), + data = self._parse_data(self._get_schema_instance(kwargs), request) # TODO migration: use default values when popping and validate the # popped object has the expected type. diff --git a/test_cases/test_api_credentials.py b/test_cases/test_api_credentials.py index 9865effe252..009e75be663 100644 --- a/test_cases/test_api_credentials.py +++ b/test_cases/test_api_credentials.py @@ -6,7 +6,7 @@ ) from server.api.modules.credentials import CredentialView from server.models import Credential -from test_cases.factories import ServiceFactory +from test_cases.factories import HostFactory, ServiceFactory class TestCredentialsAPIGeneric(ReadOnlyAPITests): @@ -116,3 +116,55 @@ def test_update_credentials(self, test_client, session): assert res.json['username'] == u'Username2' assert res.json['password'] == u'Password3' assert res.json['name'] == u'Name1' + + @pytest.mark.parametrize("parent_type, parent_factory", [ + ("Host", HostFactory), + ("Service", ServiceFactory), + ], ids=["with host parent", "with service parent"]) + def test_create_with_parent_of_other_workspace( + self, parent_type, parent_factory, test_client, session, + second_workspace): + parent = parent_factory.create(workspace=second_workspace) + session.commit() + assert parent.workspace_id != self.workspace.id + data = { + "username": "admin", + "password": "admin", + "name": "test", + "parent_type": parent_type, + "parent": parent.id + } + res = test_client.post(self.url(), data=data) + assert res.status_code == 400 + assert 'Parent id not found' in res.data + + @pytest.mark.parametrize("parent_type, parent_factory", [ + ("Host", HostFactory), + ("Service", ServiceFactory), + ], ids=["with host parent", "with service parent"]) + def test_update_with_parent_of_other_workspace( + self, parent_type, parent_factory, test_client, session, + second_workspace, credential_factory): + parent = parent_factory.create(workspace=second_workspace) + if parent_type == 'Host': + credential = credential_factory.create( + host=HostFactory.create(workspace=self.workspace), + service=None, + workspace=self.workspace) + else: + credential = credential_factory.create( + host=None, + service=ServiceFactory.create(workspace=self.workspace), + workspace=self.workspace) + session.commit() + assert parent.workspace_id != self.workspace.id + data = { + "username": "admin", + "password": "admin", + "name": "test", + "parent_type": parent_type, + "parent": parent.id + } + res = test_client.put(self.url(credential), data=data) + assert res.status_code == 400 + assert 'Parent id not found' in res.data diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index 3a98bc4359b..91ced88bb47 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -68,8 +68,6 @@ def test_create_service(self, test_client, host, session): def test_create_fails_with_host_of_other_workspace(self, test_client, host, session, second_workspace): - # TODO migration: check why workspace checker isn't triggering an error - # here session.commit() assert host.workspace_id != second_workspace.id data = { @@ -83,3 +81,24 @@ def test_create_fails_with_host_of_other_workspace(self, test_client, } res = test_client.post(self.url(workspace=second_workspace), data=data) assert res.status_code == 400 + assert 'Host with id' in res.data + + def test_update_fails_with_host_of_other_workspace(self, test_client, + second_workspace, + host_factory, + session): + host = host_factory.create(workspace=second_workspace) + session.commit() + assert host.workspace_id != self.first_object.workspace_id + data = { + "name": "ftp", + "description": "test. test", + "owned": False, + "ports": [21], + "protocol": "tcp", + "status": "open", + "parent": host.id + } + res = test_client.put(self.url(self.first_object), data=data) + assert res.status_code == 400 + assert 'Host with id' in res.data diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 977587850b1..0e87ba97aac 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -843,4 +843,62 @@ def test_vulnerability_metadata(self, session, test_client, workspace): u'update_user': u'' } - assert expected_metadata == from_json_vuln[0]['value']['metadata'] \ No newline at end of file + assert expected_metadata == from_json_vuln[0]['value']['metadata'] + + @pytest.mark.parametrize("parent_type, parent_factory", [ + ("Host", HostFactory), + ("Service", ServiceFactory), + ], ids=["with host parent", "with service parent"]) + def test_create_with_parent_of_other_workspace( + self, parent_type, parent_factory, test_client, session, + second_workspace): + parent = parent_factory.create(workspace=second_workspace) + session.commit() + assert parent.workspace_id != self.workspace.id + data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='Vulnerability', + parent_id=parent.id, + parent_type=parent_type, + refs=[], + policyviolations=[], + description='helloworld', + severity='low', + ) + res = test_client.post(self.url(), data=data) + assert res.status_code == 400 + assert 'Parent id not found' in res.data + + @pytest.mark.skip(reason="Update not implemented yet. Also it won't work " + "instantly because it is a copy paste of other test. " + "It should be changed") + @pytest.mark.parametrize("parent_type, parent_factory", [ + ("Host", HostFactory), + ("Service", ServiceFactory), + ], ids=["with host parent", "with service parent"]) + def test_update_with_parent_of_other_workspace( + self, parent_type, parent_factory, test_client, session, + second_workspace, credential_factory): + parent = parent_factory.create(workspace=second_workspace) + if parent_type == 'Host': + credential = credential_factory.create( + host=HostFactory.create(workspace=self.workspace), + service=None, + workspace=self.workspace) + else: + credential = credential_factory.create( + host=None, + service=ServiceFactory.create(workspace=self.workspace), + workspace=self.workspace) + session.commit() + assert parent.workspace_id != self.workspace.id + data = { + "username": "admin", + "password": "admin", + "name": "test", + "parent_type": parent_type, + "parent": parent.id + } + res = test_client.put(self.url(credential), data=data) + assert res.status_code == 400 + assert 'Parent id not found' in res.data From 0ae8bf23314fe36cfcfbf4d13dc4a73bd2aece21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 9 Nov 2017 17:54:54 -0300 Subject: [PATCH 0528/1506] Always disable SQLALCHEMY_ECHO when testing --- server/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/app.py b/server/app.py index f37c701b5ae..30a97cb8f8d 100644 --- a/server/app.py +++ b/server/app.py @@ -82,6 +82,7 @@ def create_app(db_connection_string=None, testing=None): 'depot.storage_path': storage_path }) if testing: + app.config['SQLALCHEMY_ECHO'] = False app.config['TESTING'] = testing app.config['NPLUSONE_LOGGER'] = logging.getLogger('faraday.nplusone') app.config['NPLUSONE_LOG_LEVEL'] = logging.ERROR From c3bb03e255a1aa6feacabd8d6bd17a8c469b0bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 10 Nov 2017 15:07:01 -0300 Subject: [PATCH 0529/1506] Fix a test to work with PostgreSQL ID was hardcoded --- test_cases/test_api_vulnerability.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 0e87ba97aac..e4c333375a3 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -817,7 +817,7 @@ def test_vulnerability_metadata(self, session, test_client, workspace): vuln = VulnerabilityWebFactory.create( service=service, - creator = owner, + creator=owner, workspace=workspace ) session.flush() @@ -830,7 +830,8 @@ def test_vulnerability_metadata(self, session, test_client, workspace): res = test_client.get(self.url()) assert res.status_code == 200 - from_json_vuln = filter(lambda raw_vuln: raw_vuln['id'] == 6, res.json['vulnerabilities']) + from_json_vuln = filter(lambda raw_vuln: raw_vuln['id'] == vuln.id, + res.json['vulnerabilities']) assert 'metadata' in from_json_vuln[0]['value'] expected_metadata = { u'command_id': 5555, From 8bfe7936625fa8f7fa0939ff1c535324446a45f4 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 10 Nov 2017 15:41:58 -0300 Subject: [PATCH 0530/1506] Fix postgresql tests --- server/commands/reports.py | 0 test_cases/test_api_commands.py | 68 ++++++++++++++++++++------------- 2 files changed, 42 insertions(+), 26 deletions(-) create mode 100644 server/commands/reports.py diff --git a/server/commands/reports.py b/server/commands/reports.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index cebe290528b..4a0acbf1b4d 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -184,16 +184,25 @@ def test_verify_created_vulns_with_host_and_service_verification(self, session, ] def test_multiple_commands_executed_with_same_objects_found(self, session, test_client): - + """ + This text verifies that multiple command does not affect activity feed counters. + """ workspace = WorkspaceFactory.create() host = HostFactory.create(workspace=workspace) vuln = VulnerabilityFactory.create(severity='low', workspace=workspace, host=host, service=None) service = ServiceFactory.create(workspace=workspace) commands = [] + in_the_middle_commands = [] + first_command = None for index in range(0, 10): + command = EmptyCommandFactory.create(workspace=workspace) commands.append(command) - + if index > 0: + # in the middle commands should not affect counters (should be at 0) + in_the_middle_commands.append(command) + else: + first_command = command session.flush() CommandObjectFactory.create( command=command, @@ -207,17 +216,18 @@ def test_multiple_commands_executed_with_same_objects_found(self, session, test_ object_id=vuln.id, workspace=workspace ) + # This command will change activity feed counters vuln_med = VulnerabilityFactory.create(severity='medium', workspace=workspace, service=service, host=None) session.flush() - command = EmptyCommandFactory.create(workspace=workspace) + last_command = EmptyCommandFactory.create(workspace=workspace) CommandObjectFactory.create( - command=command, + command=last_command, object_type='service', object_id=service.id, workspace=workspace ) CommandObjectFactory.create( - command=command, + command=last_command, object_type='vulnerability', object_id=vuln_med.id, workspace=workspace @@ -225,36 +235,42 @@ def test_multiple_commands_executed_with_same_objects_found(self, session, test_ session.commit() res = test_client.get(self.url(workspace=command.workspace) + 'activity_feed/') assert res.status_code == 200 - assert res.json[0] == {u'_id': commands[0].id, - u'command': commands[0].command, - u'import_source': u'shell', - u'user': commands[0].user, - u'date': time.mktime(commands[0].start_date.timetuple()) * 1000, - u'params': commands[0].params, - u'hosts_count': 1, - u'services_count': 0, - u'vulnerabilities_count': 1, - u'criticalIssue': 0} + raw_first_command = filter(lambda comm: comm['_id'] == commands[0].id, res.json) + + assert raw_first_command.pop() == { + u'_id': first_command.id, + u'command': first_command.command, + u'import_source': u'shell', + u'user': first_command.user, + u'date': time.mktime(first_command.start_date.timetuple()) * 1000, + u'params': first_command.params, + u'hosts_count': 1, + u'services_count': 0, + u'vulnerabilities_count': 1, + u'criticalIssue': 0 + } - for index in range(1, 10): - assert res.json[index] == {u'_id': commands[index].id, - u'command': commands[index].command, + for in_the_middle_command in in_the_middle_commands: + raw_in_the_middle_command = filter(lambda comm: comm['_id'] == in_the_middle_command.id, res.json) + assert raw_in_the_middle_command.pop() == {u'_id': in_the_middle_command.id, + u'command': in_the_middle_command.command, u'import_source': u'shell', - u'user': commands[index].user, - u'date': time.mktime(commands[index].start_date.timetuple()) * 1000, - u'params': commands[index].params, + u'user': in_the_middle_command.user, + u'date': time.mktime(in_the_middle_command.start_date.timetuple()) * 1000, + u'params': in_the_middle_command.params, u'hosts_count': 0, u'services_count': 0, u'vulnerabilities_count': 0, u'criticalIssue': 0} # new command must create new service and vuln - assert res.json[10] == {u'_id': command.id, - u'command': command.command, + raw_last_command = filter(lambda comm: comm['_id'] == last_command.id, res.json) + assert raw_last_command.pop() == {u'_id': last_command.id, + u'command': last_command.command, u'import_source': u'shell', - u'user': command.user, - u'date': time.mktime(command.start_date.timetuple()) * 1000, - u'params': command.params, + u'user': last_command.user, + u'date': time.mktime(last_command.start_date.timetuple()) * 1000, + u'params': last_command.params, u'hosts_count': 0, u'services_count': 1, u'vulnerabilities_count': 1, From 07b9b5da60f84264d8c2b73de84addcdfa344dba Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 10 Nov 2017 19:08:24 -0300 Subject: [PATCH 0531/1506] Add initial code for command process_reports --- manage.py | 2 ++ managers/reports_managers.py | 8 +++++--- plugins/controller.py | 8 +++++--- server/commands/initdb.py | 3 +-- server/commands/reports.py | 38 ++++++++++++++++++++++++++++++++++++ server/importer.py | 2 ++ 6 files changed, 53 insertions(+), 8 deletions(-) diff --git a/manage.py b/manage.py index 8f01b04d1bc..119b8070724 100755 --- a/manage.py +++ b/manage.py @@ -11,6 +11,7 @@ from server.commands.faraday_schema_display import DatabaseSchema from server.commands.app_urls import AppUrls from server.commands.reset_db import ResetDB +from server.commands.reports import ImporExternalReports manager = Manager(app) @@ -22,4 +23,5 @@ manager.add_command('faraday_schema_display', DatabaseSchema()) manager.add_command('show_urls', AppUrls()) manager.add_command('reset_db', ResetDB()) + manager.add_command('process_reports', ImporExternalReports()) manager.run() diff --git a/managers/reports_managers.py b/managers/reports_managers.py index 4c5bddca493..12b9f5b3021 100755 --- a/managers/reports_managers.py +++ b/managers/reports_managers.py @@ -27,8 +27,9 @@ class ReportProcessor(): - def __init__(self, plugin_controller): + def __init__(self, plugin_controller, ws_name=None): self.plugin_controller = plugin_controller + self.ws_name = ws_name def processReport(self, filename): """ @@ -49,7 +50,7 @@ def sendReport(self, plugin_id, filename): """Sends a report to the appropiate plugin specified by plugin_id""" getLogger(self).info( 'The file is %s, %s' % (filename, plugin_id)) - if not self.plugin_controller.processReport(plugin_id, filename): + if not self.plugin_controller.processReport(plugin_id, filename, self.ws_name): getLogger(self).error( "Faraday doesn't have a plugin for this tool..." " Processing: ABORT") @@ -65,13 +66,14 @@ def onlinePlugin(self, cmd): class ReportManager(threading.Thread): def __init__(self, timer, ws_name, plugin_controller): threading.Thread.__init__(self) + self.ws_name = ws_name self.setDaemon(True) self.timer = timer self._stop = False self._report_path = os.path.join(CONF.getReportPath(), ws_name) self._report_ppath = os.path.join(self._report_path, "process") self._report_upath = os.path.join(self._report_path, "unprocessed") - self.processor = ReportProcessor(plugin_controller) + self.processor = ReportProcessor(plugin_controller, ws_name) if not os.path.exists(self._report_path): os.mkdir(self._report_path) diff --git a/plugins/controller.py b/plugins/controller.py index c32b72d9c7b..3970281451b 100644 --- a/plugins/controller.py +++ b/plugins/controller.py @@ -237,13 +237,15 @@ def onCommandFinished(self, pid, exit_code, term_output): del self._active_plugins[pid] return True - def processReport(self, plugin, filepath): - + def processReport(self, plugin, filepath, ws_name=None): + if not ws_name: + ws_name = model.api.getActiveWorkspace().name cmd_info = CommandRunInformation( - **{'workspace': model.api.getActiveWorkspace().name, + **{'workspace': ws_name, 'itime': time.time(), 'command': 'Import %s:' % plugin, 'params': filepath}) + self._mapper_manager.createMappers(ws_name) self._mapper_manager.save(cmd_info) if plugin in self._plugins: diff --git a/server/commands/initdb.py b/server/commands/initdb.py index b9ec8152e24..fd888e8ccb1 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -2,7 +2,6 @@ import os import sys -import getpass from random import SystemRandom from tempfile import TemporaryFile from subprocess import Popen, PIPE @@ -17,7 +16,7 @@ from flask import current_app from flask_script import Command from colorama import init -from colorama import Fore, Back, Style +from colorama import Fore from sqlalchemy.exc import OperationalError from config.globals import CONST_FARADAY_HOME_PATH diff --git a/server/commands/reports.py b/server/commands/reports.py index e69de29bb2d..8d706ddac1d 100644 --- a/server/commands/reports.py +++ b/server/commands/reports.py @@ -0,0 +1,38 @@ +import os +import logging +from flask_script import Command + +from managers.mapper_manager import MapperManager +from managers.reports_managers import ReportManager, CONF +from managers.workspace_manager import WorkspaceManager +from plugins.controller import PluginController +from plugins.manager import PluginManager +from server.models import Workspace + +logger = logging.getLogger(__name__) + + +class ImporExternalReports(Command): + + def run(self): + threads = [] + plugin_manager = PluginManager( + os.path.join(CONF.getConfigPath(), "plugins")) + mappers_manager = MapperManager() + + plugin_controller = PluginController( + 'PluginController', + plugin_manager, + mappers_manager + ) + for workspace in Workspace.query.all(): + report_manager = ReportManager( + 10, + workspace.name, + plugin_controller + ) + threads.append(report_manager) + report_manager.start() + + for thread in threads: + thread.join() \ No newline at end of file diff --git a/server/importer.py b/server/importer.py index a3de4374c4c..31666adec84 100644 --- a/server/importer.py +++ b/server/importer.py @@ -188,6 +188,8 @@ def set_metadata(document, obj): try: if key == 'create_time': obj.create_date = datetime.datetime.fromtimestamp(document['metadata']['create_time']) + if obj.create_date > datetime.datetime.now(): + raise Exception('Invalid date!') if key == 'owner': creator = User.query.filter_by(username=value).first() obj.creator = creator From f821a564682093c43c475029e1f0b5d8a093cc1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 13 Nov 2017 17:17:07 -0300 Subject: [PATCH 0532/1506] Add SortableMixin and use it by default in all views --- server/api/base.py | 56 +++++++++++++++++++++++++++++++++--- server/api/modules/hosts.py | 3 +- test_cases/test_api_hosts.py | 32 +++++++++++++++++++++ 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 2994ccad478..fb035f999a1 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -253,15 +253,59 @@ def _envelope_list(self, objects, pagination_metadata=None): def _paginate(self, query): return query, None + def _get_order_field(self, **kwargs): + """Override this to enable custom sorting""" + return self.order_field + def index(self, **kwargs): query = self._filter_query(self._get_eagerloaded_query(**kwargs)) - if self.order_field is not None: - query = query.order_by(self.order_field) + order_field = self._get_order_field(**kwargs) + if order_field is not None: + query = query.order_by(order_field) objects, pagination_metadata = self._paginate(query) return self._envelope_list(self._dump(objects, kwargs, many=True), pagination_metadata) +class SortableMixin(object): + """Enables custom sorting by a field specified by te user""" + sort_field_paremeter_name = "sort" + sort_direction_paremeter_name = "sort_dir" + default_sort_direction = "asc" + + def _get_order_field(self, **kwargs): + try: + order_field = flask.request.args[self.sort_field_paremeter_name] + except KeyError: + # Sort field not specified, return the default + return self.order_field + # Check that the field is in the schema to prevent unwanted fields + # value leaking + schema = self._get_schema_instance(kwargs) + try: + field_instance = schema.fields[order_field] + except KeyError: + raise InvalidUsage("Unknown field: %s" % order_field) + + # Translate from the field name in the schema to the database field + # name + order_field = field_instance.attribute or order_field + + # TODO migration: improve this checking or use a whitelist. + # Handle PrimaryKeyRelatedField + if order_field not in inspect(self.model_class).attrs: + # It could be something like fields.Method + raise InvalidUsage("Field not in the DB: %s" % order_field) + + field = getattr(self.model_class, order_field) + sort_dir = flask.request.args.get(self.sort_direction_paremeter_name, + self.default_sort_direction) + if sort_dir not in ('asc', 'desc'): + raise InvalidUsage("Invalid value for sorting direction: %s" % + sort_dir) + return getattr(field, sort_dir)() + + class PaginatedMixin(object): """Add pagination for list route""" per_page_parameter_name = 'page_size' @@ -323,14 +367,16 @@ class RetrieveWorkspacedMixin(RetrieveMixin): pass -class ReadOnlyView(ListMixin, +class ReadOnlyView(SortableMixin, + ListMixin, RetrieveMixin, GenericView): """A generic view with list and retrieve endpoints""" pass -class ReadOnlyWorkspacedView(ListWorkspacedMixin, +class ReadOnlyWorkspacedView(SortableMixin, + ListWorkspacedMixin, RetrieveWorkspacedMixin, GenericWorkspacedView): """A workspaced generic view with list and retrieve endpoints""" @@ -442,6 +488,8 @@ def count(self, **kwargs): # TODO migration: whitelist fields to avoid leaking a confidential # field's value. # Example: /users/count/?group_by=password + # Also we should check that the field exists in the db and isn't, for + # example, a relationship if not group_by or group_by not in inspect(self.model_class).attrs: abort(404) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index a2cae3a5a26..93a3dbdb4e1 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -83,7 +83,7 @@ class HostsView(PaginatedMixin, filterset_class = HostFilterSet get_undefer = [Host.open_service_count, Host.vulnerability_count] - get_joinedloads = [Host.hostnames] + get_joinedloads = [Host.hostnames, Host.workspace] @route('//services/') def service_list(self, workspace_name, host_id): @@ -115,4 +115,5 @@ def _envelope_list(self, objects, pagination_metadata=None): or len(hosts)), } + HostsView.register(host_api) diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index c8bc34885f8..040f43378a8 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -1,3 +1,4 @@ +import operator import pytest from sqlalchemy.orm.util import was_deleted @@ -334,3 +335,34 @@ class TestHostAPIGeneric(ReadWriteAPITests, PaginationTestsMixin): unique_fields = ['ip'] update_fields = ['ip', 'description', 'os'] view_class = HostsView + + @pytest.mark.usefixtures("mock_envelope_list") + def test_sort_by_description(self, test_client, session): + session.commit() + expected_ids = [host.id for host in + sorted(Host.query.all(), + key=operator.attrgetter('description'))] + res = test_client.get(self.url() + '?sort=description&sort_dir=asc') + assert res.status_code == 200 + assert [host['_id'] for host in res.json['data']] == expected_ids + + expected_ids.reverse() # In place list reverse + res = test_client.get(self.url() + '?sort=description&sort_dir=desc') + assert res.status_code == 200 + assert [host['_id'] for host in res.json['data']] == expected_ids + + @pytest.mark.usefixtures("mock_envelope_list") + def test_sort_by_services(self, test_client, session, second_workspace, + host_factory, service_factory): + expected_ids = [] + for i in range(10): + host = host_factory.create(workspace=second_workspace) + service_factory.create(host=host, workspace=second_workspace, + status='open') + session.flush() + expected_ids.append(host.id) + session.commit() + res = test_client.get(self.url(workspace=second_workspace) + + '?sort=services&sort_dir=asc') + assert res.status_code == 200 + assert [h['_id'] for h in res.json['data']] == expected_ids From b1dfcd4393984b09eac8d2f147415dd48bcacbfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 13 Nov 2017 17:25:56 -0300 Subject: [PATCH 0533/1506] Sort vulns by date in the dashboard --- server/api/modules/vulns.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 9536cf75bbe..362bc984899 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -105,6 +105,8 @@ class VulnerabilitySchema(AutoSchema): obj_id = fields.String(dump_only=True, attribute='id') target = fields.Method('get_target') metadata = SelfNestedField(MetadataSchema()) + date = fields.DateTime(attribute='create_date', + dump_only=True) # This is only used for sorting class Meta: model = Vulnerability @@ -113,7 +115,7 @@ class Meta: 'issuetracker', 'description', 'parent', 'parent_type', 'tags', 'severity', '_rev', 'easeofresolution', 'owned', 'hostnames', 'owner', - 'data', 'refs', + 'date', 'data', 'refs', 'desc', 'impact', 'confirmed', 'name', 'service', 'obj_id', 'type', 'policyviolations', '_attachments', @@ -241,7 +243,7 @@ class Meta: 'website', 'issuetracker', 'description', 'parent', 'tags', 'severity', '_rev', 'easeofresolution', 'owned', 'hostnames', 'pname', 'query', 'owner', - 'path', 'data', 'response', 'refs', + 'path', 'date', 'data', 'response', 'refs', 'desc', 'impact', 'confirmed', 'name', 'service', 'obj_id', 'type', 'policyviolations', 'request', '_attachments', 'params', From f66a649cd5671d41050f87b5f7b068aa9b3934d2 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 14 Nov 2017 12:43:32 -0300 Subject: [PATCH 0534/1506] Fix minor issues on the importer * When running multiple times the importer the created_pesistent was always False * tags sometimes were None --- server/importer.py | 3 ++- server/models.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/server/importer.py b/server/importer.py index a3de4374c4c..7e3f3ee491e 100644 --- a/server/importer.py +++ b/server/importer.py @@ -407,6 +407,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation 'open|filtered': 'filtered', 'unknown': 'closed', '-': 'closed', + 'running': 'open', } couchdb_status = document.get('status', 'open') if couchdb_status.lower() not in status_mapper: @@ -525,7 +526,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation # need the vuln ID before creating Tags for it session.flush() tags = document.get('tags', []) - if len(tags): + if tags and len(tags): create_tags(tags, vulnerability.id, document['type']) self.add_attachments(document, vulnerability, workspace) diff --git a/server/models.py b/server/models.py index ab28374b05d..5e26ff0b35e 100644 --- a/server/models.py +++ b/server/models.py @@ -1324,7 +1324,7 @@ def log_command_object_found(command, object, created): object_type = 'vulnerability' db.session.flush() - log, _ = get_or_create( + log, log_created = get_or_create( db.session, CommandObject, command=command, @@ -1332,7 +1332,9 @@ def log_command_object_found(command, object, created): object_type=object_type, workspace=object.workspace, ) - log.created_persistent = created + if not log_created: + # without this if, multiple executions of the importer will write this attribute with False + log.created_persistent = created # We have to import this after all models are defined import server.events From ab935680b77a1a7158ae9de580a6742827740e3e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 14 Nov 2017 15:40:47 -0300 Subject: [PATCH 0535/1506] Add great or equal on pyasn1 --- requirements_server.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_server.txt b/requirements_server.txt index a6fce372dc0..21925639d47 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -7,7 +7,7 @@ Flask-Security==3.0.0 flask==0.12.2 marshmallow Pillow==4.2.1 -pyasn1-modules==0.0.11 +pyasn1-modules>=0.0.11 pyopenssl==17.2.0 python-slugify==1.2.4 requests==2.18.4 From 46c5db068fa1d2c41da826721c722dda24433795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 14 Nov 2017 15:43:57 -0300 Subject: [PATCH 0536/1506] Order workspaces by name --- server/api/modules/workspaces.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index f3e2299f603..7659a57e521 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -71,6 +71,7 @@ class WorkspaceView(ReadWriteView): lookup_field_type = unicode model_class = Workspace schema_class = WorkspaceSchema + order_field = Workspace.name.asc() def _get_base_query(self): try: From 93e42d42753897955cc15eecc17d09e701556c64 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 14 Nov 2017 15:44:55 -0300 Subject: [PATCH 0537/1506] Fix create_time last vulns on dashboard --- server/www/scripts/dashboard/partials/last-vulns.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/dashboard/partials/last-vulns.html b/server/www/scripts/dashboard/partials/last-vulns.html index c169810b681..86a7728a2d0 100644 --- a/server/www/scripts/dashboard/partials/last-vulns.html +++ b/server/www/scripts/dashboard/partials/last-vulns.html @@ -32,7 +32,7 @@

    Last Vulnerabilities

    From 77e7308e6acd58f9b3daa0c161b6dbe149a2503e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 14 Nov 2017 16:10:21 -0300 Subject: [PATCH 0538/1506] Fix duration on commands api --- server/api/modules/commandsrun.py | 6 +++- .../scripts/dashboard/providers/dashboard.js | 6 ++-- test_cases/test_api_commands.py | 29 ++++++++++++++++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/server/api/modules/commandsrun.py b/server/api/modules/commandsrun.py index ab555235024..7f7eee1de69 100644 --- a/server/api/modules/commandsrun.py +++ b/server/api/modules/commandsrun.py @@ -34,7 +34,11 @@ def get_itime(self, obj): def get_duration(self, obj): if obj.end_date and obj.start_date: - return (obj.end_date - obj.start_date).seconds + return (obj.end_date - obj.start_date).seconds + ((obj.end_date - obj.start_date).microseconds / 1000000.0) + if obj.start_date and not obj.end_date: + return 'In progress' + if not obj.start_date and not obj.end_date: + return 'Not started' class Meta: model = Command diff --git a/server/www/scripts/dashboard/providers/dashboard.js b/server/www/scripts/dashboard/providers/dashboard.js index 84e65dc1d4f..05cfb4380c3 100644 --- a/server/www/scripts/dashboard/providers/dashboard.js +++ b/server/www/scripts/dashboard/providers/dashboard.js @@ -251,12 +251,14 @@ angular.module('faradayApp') _cmd.user = _cmd.user || "unknown"; _cmd.hostname = _cmd.hostname || "unknown"; _cmd.ip = _cmd.ip || "0.0.0.0"; - if(_cmd.duration == "0" || _cmd.duration == "") { + if(_cmd.duration == "In progres") { _cmd.duration = "In progress"; + } else if (_cmd.duration == "Not started") { + _cmd.duration = "Not started"; } else if(_cmd.duration != undefined) { _cmd.duration = _cmd.duration.toFixed(2) + "s"; } - _cmd.date = _cmd.itime * 1000; + _cmd.date = _cmd.itime; tmp.push(_cmd); }); diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index 4a0acbf1b4d..17fd2ef6c61 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -1,6 +1,6 @@ #-*- coding: utf8 -*- """Tests for many API endpoints that do not depend on workspace_name""" - +import datetime import pytest import time @@ -275,3 +275,30 @@ def test_multiple_commands_executed_with_same_objects_found(self, session, test_ u'services_count': 1, u'vulnerabilities_count': 1, u'criticalIssue': 0} + + def test_sub_second_command_returns_correct_duration_value(self, test_client): + command = self.factory( + start_date=datetime.datetime(2017, 11, 14, 12, 29, 21, 248433), + end_date=datetime.datetime(2017, 11, 14, 12, 29, 21, 690839) + ) + res = test_client.get(self.url(workspace=command.workspace)) + assert res.status_code == 200 + assert res.json['commands'][0]['value']['duration'] == 0.442406 + + def test_more_than_one_second_command_returns_correct_duration_value(self, test_client): + command = self.factory( + start_date=datetime.datetime(2017, 11, 14, 12, 29, 20, 248433), + end_date=datetime.datetime(2017, 11, 14, 12, 29, 21, 690839) + ) + res = test_client.get(self.url(workspace=command.workspace)) + assert res.status_code == 200 + assert res.json['commands'][0]['value']['duration'] == 1.442406 + + def test_more_than_one_hour_command_returns_correct_duration_value(self, test_client): + command = self.factory( + start_date=datetime.datetime(2017, 11, 14, 12, 28, 20, 248433), + end_date=datetime.datetime(2017, 11, 14, 12, 29, 21, 690839) + ) + res = test_client.get(self.url(workspace=command.workspace)) + assert res.status_code == 200 + assert res.json['commands'][0]['value']['duration'] == 61.442406 \ No newline at end of file From a34848ddfdcedc6ab956b5687c260e75d834ea03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 14 Nov 2017 18:41:21 -0300 Subject: [PATCH 0539/1506] Fix detected n+1 issues in many views and enable nplusone again By stopping using the ignore_nplusone fixture in many tests --- server/api/modules/commandsrun.py | 7 +++---- server/api/modules/hosts.py | 2 +- server/schemas.py | 4 ++-- test_cases/test_api_commands.py | 2 +- test_cases/test_api_credentials.py | 1 - test_cases/test_api_hosts.py | 3 --- test_cases/test_api_pagination.py | 2 -- test_cases/test_api_services.py | 1 - test_cases/test_api_vulnerability.py | 6 ------ test_cases/test_api_workspaced_base.py | 1 - 10 files changed, 7 insertions(+), 22 deletions(-) diff --git a/server/api/modules/commandsrun.py b/server/api/modules/commandsrun.py index ab555235024..3a37907f8ab 100644 --- a/server/api/modules/commandsrun.py +++ b/server/api/modules/commandsrun.py @@ -16,6 +16,7 @@ filter_request_args, get_integer_parameter ) from server.models import Command, Workspace +from server.schemas import PrimaryKeyRelatedField commandsrun_api = Blueprint('commandsrun_api', __name__) @@ -24,10 +25,7 @@ class CommandSchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') itime = fields.Method('get_itime') duration = fields.Method('get_duration') - workspace = fields.Method('get_workspace_name') - - def get_workspace_name(self, obj): - return obj.workspace.name + workspace = PrimaryKeyRelatedField('name') def get_itime(self, obj): return time.mktime(obj.start_date.utctimetuple()) @@ -46,6 +44,7 @@ class CommandView(ReadWriteWorkspacedView): route_base = 'commands' model_class = Command schema_class = CommandSchema + get_joinedloads = [Command.workspace] def _envelope_list(self, objects, pagination_metadata=None): commands = [] diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 93a3dbdb4e1..9c57b9569e0 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -83,7 +83,7 @@ class HostsView(PaginatedMixin, filterset_class = HostFilterSet get_undefer = [Host.open_service_count, Host.vulnerability_count] - get_joinedloads = [Host.hostnames, Host.workspace] + get_joinedloads = [Host.hostnames] @route('//services/') def service_list(self, workspace_name, host_id): diff --git a/server/schemas.py b/server/schemas.py index a985a0d59a4..002d1f1417d 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -117,8 +117,8 @@ class MetadataSchema(Schema): def get_command_id(self, obj): command_id = None - command_obj = CommandObject.query.filter_by(object_type='vulnerability', object_id=obj.id, workspace=obj.workspace).first() + command_obj = CommandObject.query.filter_by(object_type='vulnerability', object_id=obj.id, workspace_id=obj.workspace_id).first() if command_obj: command_id = command_obj.command_id - return command_id \ No newline at end of file + return command_id diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index 4a0acbf1b4d..709c8e10ee2 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -29,7 +29,6 @@ class TestListCommandView(ReadOnlyAPITests): api_endpoint = 'commands' view_class = CommandView - @pytest.mark.usefixtures('ignore_nplusone') def test_backwards_compatibility_list(self, test_client, second_workspace, session): self.factory.create(workspace=second_workspace) session.commit() @@ -49,6 +48,7 @@ def test_backwards_compatibility_list(self, test_client, second_workspace, sessi u'user', u'workspace' ] + assert command['value']['workspace'] == self.workspace.name assert set(object_properties) == set(command['value'].keys()) def test_activity_feed(self, session, test_client): diff --git a/test_cases/test_api_credentials.py b/test_cases/test_api_credentials.py index 009e75be663..7452cd80289 100644 --- a/test_cases/test_api_credentials.py +++ b/test_cases/test_api_credentials.py @@ -16,7 +16,6 @@ class TestCredentialsAPIGeneric(ReadOnlyAPITests): api_endpoint = 'credential' update_fields = ['username', 'password'] - @pytest.mark.usefixtures('ignore_nplusone') def test_get_list_backwards_compatibility(self, test_client, session, second_workspace): cred = self.factory.create(workspace=second_workspace) session.commit() diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 040f43378a8..a102e365c1d 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -60,7 +60,6 @@ def compare_results(self, hosts, response): hosts_in_response = set(host['id'] for host in response.json['rows']) assert hosts_in_list == hosts_in_response - @pytest.mark.usefixtures('ignore_nplusone') def test_list_retrieves_all_items_from_workspace(self, test_client, second_workspace, session, @@ -213,7 +212,6 @@ def test_retrieve_shows_service_count(self, test_client, host_services, res = test_client.get(self.url(host)) assert res.json['services'] == len(services) - @pytest.mark.usefixtures('ignore_nplusone') def test_index_shows_service_count(self, test_client, host_services, service_factory): ids_map = {host.id: services @@ -273,7 +271,6 @@ def test_filter_by_os_like_ilike(self, test_client, session, workspace, assert res.status_code == 200 self.compare_results(hosts + [case_insensitive_host], res) - @pytest.mark.usefixtures('ignore_nplusone') def test_host_with_open_vuln_count_verification(self, test_client, session, workspace, host_factory, vulnerability_factory, service_factory): diff --git a/test_cases/test_api_pagination.py b/test_cases/test_api_pagination.py index 79fd7ea83a3..cd60aa2e387 100644 --- a/test_cases/test_api_pagination.py +++ b/test_cases/test_api_pagination.py @@ -38,7 +38,6 @@ def page_url(self, page_number=None, per_page=None): @pytest.mark.parametrize("page_number", [None, 1, 2]) @pytest.mark.usefixtures('pagination_test_logic') - @pytest.mark.usefixtures('ignore_nplusone') @pytest.mark.pagination def test_returns_all_with_no_per_page(self, test_client, session, page_number): @@ -70,7 +69,6 @@ def test_does_not_allow_negative_page_number(self, session, test_client, assert res.json == {u'data': []} @pytest.mark.usefixtures('pagination_test_logic') - @pytest.mark.usefixtures('ignore_nplusone') @pytest.mark.pagination def test_pages_have_different_elements(self, session, test_client): """Test correct page size, correct IDs and that there are diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index 91ced88bb47..c63a0eb1b97 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -20,7 +20,6 @@ class TestListServiceView(ReadOnlyAPITests): #update_fields = ['ip', 'description', 'os'] view_class = ServiceView - @pytest.mark.usefixtures('ignore_nplusone') def test_service_list_backwards_compatibility(self, test_client, second_workspace, session): self.factory.create(workspace=second_workspace) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index e4c333375a3..020a4e39c3c 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -29,7 +29,6 @@ class TestListVulnerabilityView(ReadOnlyAPITests): #update_fields = ['ip', 'description', 'os'] view_class = VulnerabilityView - @pytest.mark.usefixtures('ignore_nplusone') def test_backward_json_compatibility(self, test_client, second_workspace, session): self.factory.create(workspace=second_workspace) session.commit() @@ -195,7 +194,6 @@ def _create_post_data_vulnerability(self, name, vuln_type, parent_id, return data - @pytest.mark.usefixtures('ignore_nplusone') def test_create_vuln(self, host_with_hostnames, test_client, session): """ This one should only check basic vuln properties @@ -434,7 +432,6 @@ def test_create_vuln_web(self, host_with_hostnames, test_client, session): assert res.json['path'] == '/pepep' @pytest.mark.usefixtures('mock_envelope_list') - @pytest.mark.usefixtures('ignore_nplusone') def test_filter_by_severity(self, test_client, session, second_workspace, vulnerability_factory, @@ -469,7 +466,6 @@ def test_filter_by_invalid_severity(self, test_client): assert res.status_code == 400 @pytest.mark.usefixtures('mock_envelope_list') - @pytest.mark.usefixtures('ignore_nplusone') def test_filter_by_method(self, test_client, session, second_workspace, vulnerability_factory, vulnerability_web_factory): @@ -500,7 +496,6 @@ def test_filter_by_method(self, test_client, session, second_workspace, assert len(res.json['data']) == 0 @pytest.mark.usefixtures('mock_envelope_list') - @pytest.mark.usefixtures('ignore_nplusone') def test_filter_by_website(self, test_client, session, second_workspace, vulnerability_factory, @@ -809,7 +804,6 @@ def test_filter_by_command_id(self, test_client, session, vuln['metadata']['command_id'] == command_object.command.id assert set(vuln['_id'] for vuln in res.json['data']) == web_expected_ids - @pytest.mark.usefixtures('ignore_nplusone') def test_vulnerability_metadata(self, session, test_client, workspace): owner = UserFactory.create() service = ServiceFactory.create(workspace=workspace) diff --git a/test_cases/test_api_workspaced_base.py b/test_cases/test_api_workspaced_base.py index 048528a1f9d..a244ec227d7 100644 --- a/test_cases/test_api_workspaced_base.py +++ b/test_cases/test_api_workspaced_base.py @@ -62,7 +62,6 @@ def _envelope_list(self, objects, pagination_metadata=None): monkeypatch.setattr(self.view_class, '_envelope_list', _envelope_list) @pytest.mark.usefixtures('mock_envelope_list') - @pytest.mark.usefixtures('ignore_nplusone') def test_list_retrieves_all_items_from_workspace(self, test_client, second_workspace, session): From 7585d2a6476294fb135c8ccc05991143b29fb68d Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 15 Nov 2017 12:29:59 -0300 Subject: [PATCH 0540/1506] Avoid saving the id on updates. Fix hosts update --- server/api/base.py | 3 + server/api/modules/hosts.py | 2 - server/www/scripts/hosts/providers/hosts.js | 2 +- test_cases/test_api_hosts.py | 61 ++++++++++++++++++++- test_cases/test_api_workspaced_base.py | 10 ++++ 5 files changed, 74 insertions(+), 4 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 2994ccad478..a09ef60edf4 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -384,6 +384,9 @@ class UpdateMixin(object): def put(self, object_id, **kwargs): data = self._parse_data(self._get_schema_instance(kwargs), flask.request) + if 'id' in data: + # just in case an schema allows id as writable. + data.pop('id') obj = self._get_object(object_id, **kwargs) self._update_object(obj, data) updated = self._perform_update(object_id, obj, **kwargs) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index a2cae3a5a26..c7f7bc4556e 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -92,9 +92,7 @@ def service_list(self, workspace_name, host_id): def _perform_create(self, data, **kwargs): hostnames = data.pop('hostnames', []) - default_gateway_ip = data.pop('default_gateway_ip', [None]) host = super(HostsView, self)._perform_create(data, **kwargs) - host.default_gateway_ip = default_gateway_ip[0] for name in hostnames: get_or_create(db.session, Hostname, name=name['key'], host=host, workspace=host.workspace) diff --git a/server/www/scripts/hosts/providers/hosts.js b/server/www/scripts/hosts/providers/hosts.js index a8740bd9b87..9310264b224 100644 --- a/server/www/scripts/hosts/providers/hosts.js +++ b/server/www/scripts/hosts/providers/hosts.js @@ -116,7 +116,7 @@ angular.module('faradayApp') var deferred = $q.defer(), self = this; - resp.update(hostData, ws) + host.update(hostData, ws) .then(function(host) { deferred.resolve(host); }) diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index c8bc34885f8..538a539f27f 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -1,4 +1,5 @@ import pytest +import time from sqlalchemy.orm.util import was_deleted from test_cases import factories @@ -9,6 +10,7 @@ ) from server.models import db, Host from server.api.modules.hosts import HostsView +from test_cases.factories import HostFactory HOSTS_COUNT = 5 SERVICE_COUNT = [10, 5] # 10 services to the first host, 5 to the second @@ -324,7 +326,64 @@ def test_create_host_with_default_gateway(self, test_client): } res = test_client.post(self.url(), data=raw_data) assert res.status_code == 201 - assert res.json['default_gateway'] == [u'192.168.0.1'] + assert res.json['default_gateway'] == '192.168.0.1' + + def test_update_host(self, test_client, session): + host = HostFactory.create() + session.commit() + raw_data = { + "metadata": + { + "update_time":1510688312.431, + "update_user":"UI Web", + "update_action":0, + "creator":"", + "create_time":1510673388000, + "update_controller_action":"", + "owner":"leonardo", + "command_id": None}, + "name":"10.31.112.21", + "ip":"10.31.112.21", + "_rev":"", + "description":"", + "owned": False, + "services":12, + "hostnames":[], + "vulns":43, + "owner":"leonardo", + "credentials":0, + "_id": 4000, + "os":"Microsoft Windows Server 2008 R2 Standard Service Pack 1", + "id": 4000, + "icon":"windows"} + + res = test_client.put(self.url(host, workspace=host.workspace), data=raw_data) + assert res.status_code == 200 + updated_host = Host.query.filter_by(id=host.id).first() + assert res.json == { + u'_id': 6, + u'_rev': u'', + u'credentials': 0, + u'default_gateway': None, + u'description': u'', + u'hostnames': [], + u'id': 6, + u'ip': u'10.31.112.21', + u'metadata': {u'command_id': None, + u'create_time': int(time.mktime(updated_host.create_date.timetuple())) * 1000, + u'creator': u'', + u'owner': host.creator.username, + u'update_action': 0, + u'update_controller_action': u'', + u'update_time': int(time.mktime(updated_host.update_date.timetuple())) * 1000, + u'update_user': u''}, + u'name': u'10.31.112.21', + u'os': u'Microsoft Windows Server 2008 R2 Standard Service Pack 1', + u'owned': False, + u'owner': host.creator.username, + u'services': 0, + u'vulns': 0} + class TestHostAPIGeneric(ReadWriteAPITests, PaginationTestsMixin): diff --git a/test_cases/test_api_workspaced_base.py b/test_cases/test_api_workspaced_base.py index 048528a1f9d..c3f1da5d1b8 100644 --- a/test_cases/test_api_workspaced_base.py +++ b/test_cases/test_api_workspaced_base.py @@ -157,6 +157,16 @@ def test_update_an_object_fails_with_empty_dict(self, test_client): res = test_client.put(self.url(self.first_object), data={}) assert res.status_code == 400 + def test_update_cant_change_id(self, test_client): + raw_json = self.factory.build_dict() + expected_id = self.first_object.id + raw_json['id'] = 100000 + res = test_client.put(self.url(self.first_object), + data=raw_json) + assert res.status_code == 200 + assert res.json['id'] == expected_id + + class DeleteTestsMixin: From d309e60226907b7645bc7cf7855b85aa4aec7e3d Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 15 Nov 2017 12:30:50 -0300 Subject: [PATCH 0541/1506] Fix css problem. Button was above edit and delete buttons --- server/www/estilos.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/estilos.css b/server/www/estilos.css index afda0a58629..043eb882008 100644 --- a/server/www/estilos.css +++ b/server/www/estilos.css @@ -1189,7 +1189,7 @@ a.button-disable{cursor: not-allowed;pointer-events: none;opacity: 0.5} display: none !important; } -#single-button { +.header_right { /* TODO: this is a fast hask, maybe there is a better way to do this */ - top: -50px; + margin-top: -3%; } From 3b4a14870ad36d52c9f563e6c9703f63d8880c32 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 17 Nov 2017 12:56:54 -0300 Subject: [PATCH 0542/1506] Add error 500 handlers. Clean up create_app --- server/api/base.py | 2 +- server/api/modules/handlers.py | 15 +++++ server/app.py | 107 +++++++++++++++++++-------------- server/web.py | 2 - 4 files changed, 77 insertions(+), 49 deletions(-) create mode 100644 server/api/modules/handlers.py diff --git a/server/api/base.py b/server/api/base.py index a09ef60edf4..34607d03327 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -13,7 +13,7 @@ from marshmallow_sqlalchemy.schema import ModelSchemaMeta, ModelSchemaOpts from webargs.flaskparser import FlaskParser, parser, abort from webargs.core import ValidationError -from server.models import Workspace, db, Metadata +from server.models import Workspace, db import server.utils.logger logger = server.utils.logger.get_logger(__name__) diff --git a/server/api/modules/handlers.py b/server/api/modules/handlers.py new file mode 100644 index 00000000000..ee86d5a6d8a --- /dev/null +++ b/server/api/modules/handlers.py @@ -0,0 +1,15 @@ +from flask import jsonify, Blueprint + +handlers_api = Blueprint('handlers_api', __name__) + + +@handlers_api.errorhandler(500) +def error_response(e): + return jsonify({ + 'messages': 'error', + }), 500 + + + + +#.register(commandsrun_api) \ No newline at end of file diff --git a/server/app.py b/server/app.py index 30a97cb8f8d..7cb6e3d70b9 100644 --- a/server/app.py +++ b/server/app.py @@ -46,6 +46,62 @@ def setup_storage_path(): return default_path +def register_blueprints(app): + from server.modules.info import info_api + from server.api.modules.commandsrun import commandsrun_api + from server.api.modules.credentials import credentials_api + from server.api.modules.doc import doc_api + from server.api.modules.hosts import host_api + from server.api.modules.licenses import license_api + from server.api.modules.services import services_api + from server.api.modules.session import session_api + from server.api.modules.vulns import vulns_api + from server.api.modules.vulnerability_template import vulnerability_template_api + from server.api.modules.workspaces import workspace_api + from server.api.modules.handlers import handlers_api + app.register_blueprint(commandsrun_api) + app.register_blueprint(credentials_api) + app.register_blueprint(doc_api) + app.register_blueprint(host_api) + app.register_blueprint(info_api) + app.register_blueprint(license_api) + app.register_blueprint(services_api) + app.register_blueprint(session_api) + app.register_blueprint(vulns_api) + app.register_blueprint(vulnerability_template_api) + app.register_blueprint(workspace_api) + app.register_blueprint(handlers_api) + + +def check_testing_configuration(testing, app): + if testing: + app.config['SQLALCHEMY_ECHO'] = False + app.config['TESTING'] = testing + app.config['NPLUSONE_LOGGER'] = logging.getLogger('faraday.nplusone') + app.config['NPLUSONE_LOG_LEVEL'] = logging.ERROR + app.config['NPLUSONE_RAISE'] = True + NPlusOne(app) + + +def register_handlers(app): + # We are exposing a RESTful API, so don't redirect a user to a login page in + # case of being unauthorized, raise a 403 error instead + @app.login_manager.unauthorized_handler + def unauthorized(): + flask.abort(403) + + @app.before_request + def default_login_required(): + view = app.view_functions.get(flask.request.endpoint) + logged_in = 'user_id' in flask.session + if (not logged_in and not getattr(view, 'is_public', False)): + flask.abort(403) + + g.user = None + if logged_in: + user = User.query.filter_by(id=session["user_id"]).first() + g.user = user + def create_app(db_connection_string=None, testing=None): app = Flask(__name__) @@ -81,13 +137,9 @@ def create_app(db_connection_string=None, testing=None): DepotManager.configure('default', { 'depot.storage_path': storage_path }) - if testing: - app.config['SQLALCHEMY_ECHO'] = False - app.config['TESTING'] = testing - app.config['NPLUSONE_LOGGER'] = logging.getLogger('faraday.nplusone') - app.config['NPLUSONE_LOG_LEVEL'] = logging.ERROR - app.config['NPLUSONE_RAISE'] = True - NPlusOne(app) + + check_testing_configuration(testing, app) + try: app.config['SQLALCHEMY_DATABASE_URI'] = db_connection_string or server.config.database.connection_string.strip("'") except AttributeError: @@ -114,46 +166,9 @@ def create_app(db_connection_string=None, testing=None): for handler in LOGGING_HANDLERS: app.logger.addHandler(handler) - from server.modules.info import info_api - from server.api.modules.commandsrun import commandsrun_api - from server.api.modules.credentials import credentials_api - from server.api.modules.doc import doc_api - from server.api.modules.hosts import host_api - from server.api.modules.licenses import license_api - from server.api.modules.services import services_api - from server.api.modules.session import session_api - from server.api.modules.vulns import vulns_api - from server.api.modules.vulnerability_template import vulnerability_template_api - from server.api.modules.workspaces import workspace_api - app.register_blueprint(commandsrun_api) - app.register_blueprint(credentials_api) - app.register_blueprint(doc_api) - app.register_blueprint(host_api) - app.register_blueprint(info_api) - app.register_blueprint(license_api) - app.register_blueprint(services_api) - app.register_blueprint(session_api) - app.register_blueprint(vulns_api) - app.register_blueprint(vulnerability_template_api) - app.register_blueprint(workspace_api) - - # We are exposing a RESTful API, so don't redirect a user to a login page in - # case of being unauthorized, raise a 403 error instead - @app.login_manager.unauthorized_handler - def unauthorized(): - flask.abort(403) - - @app.before_request - def default_login_required(): - view = app.view_functions.get(flask.request.endpoint) - logged_in = 'user_id' in flask.session - if (not logged_in and not getattr(view, 'is_public', False)): - flask.abort(403) + register_blueprints(app) + register_handlers(app) - g.user = None - if logged_in: - user = User.query.filter_by(id=session["user_id"]).first() - g.user = user return app diff --git a/server/web.py b/server/web.py index a20dd8581bb..fb6bc9e36f0 100644 --- a/server/web.py +++ b/server/web.py @@ -10,9 +10,7 @@ import server.config -from twisted.web import proxy from twisted.internet import ssl, reactor, error -from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.web.static import File from twisted.web.wsgi import WSGIResource from server.utils import logger From 4062e8949942aaf191ef86b41d8d8bd11770f015 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 17 Nov 2017 13:04:39 -0300 Subject: [PATCH 0543/1506] Fix broken test --- server/api/modules/hosts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index c7f7bc4556e..b9beef6f965 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -29,7 +29,7 @@ class HostSchema(AutoSchema): _rev = fields.String(default='') ip = fields.String(default='') description = fields.String(required=True) # Explicitly set required=True - default_gateway = fields.List(fields.String, attribute="default_gateway_ip") + default_gateway = fields.String(attribute="default_gateway_ip") name = fields.String(dump_only=True, attribute='ip', default='') os = fields.String(default='') owned = fields.Boolean(default=False) From 33aaa94a128f19ddccd5435bacaed8c2a86d9258 Mon Sep 17 00:00:00 2001 From: Winna_Z Date: Fri, 17 Nov 2017 16:26:05 -0300 Subject: [PATCH 0544/1506] New issue template for GitHub --- .github/issue_template.md | 60 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/issue_template.md diff --git a/.github/issue_template.md b/.github/issue_template.md new file mode 100644 index 00000000000..d859a46215c --- /dev/null +++ b/.github/issue_template.md @@ -0,0 +1,60 @@ +Please search the [Wiki](https://github.com/infobyte/faraday/wiki) for a solution before posting a ticket. Use the “New ” button to the right of the screen to submit a ticket for technical support. + +## Issue Type + - Bug Report + - Feature Idea + - Documentation Report + + +## Faraday version + +Paste the output of the *./faraday.py --version* command + +## Component Name + +If you know where the problem lays indicate it: +WebGui/GTKGui/Plugin/Console/Continuous Scanning/Etc. + +## Steps to reproduce + +Provide detailed steps on how the issue happened so we can try to reproduce it. If the issue is random, please provide as much information as possible. + +## Expected results + +What did you expect to happen when following the steps above? + +### Debugging tracebacks (current results) + +Try to reproduce the bug with the server and/or gtk client in debug mode and check the logs for the ERROR string. +Add here any errors you find while running in debug mode or, if possible, Faraday’s log files (located at *$HOME/.faraday/logs/*). + +If you need help on how to execute in debug mode [click here for more information](https://github.com/infobyte/faraday/wiki/troubleshooting). + +Please attach the result of: + +pip freeze > requirements_freeze.txt + +### Screenshots + +If you don't find anything on the logs, please provide screenshots of the error. + +## Environment information + +### Configuration files + +Mention any settings you have changed/added/removed. + +### Reports/Extra data + +If you are having issues with plugins, please attach relevant files if possible. +(strip your reports of all sensitive information beforehand). + +### OS + +Provide information on your operating system. Example: + +$ cat /etc/lsb-release +DISTRIB_ID=Ubuntu +DISTRIB_RELEASE=16.10 +DISTRIB_CODENAME=yakkety +DISTRIB_DESCRIPTION="Ubuntu 16.10" From 2850a3fc264455c38e14d47899d93f509858ffa0 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 17 Nov 2017 17:43:50 -0300 Subject: [PATCH 0545/1506] Update services api * Test success story: we found that it was possible to change parents of services. * Added object instance on marshmallow context * When the obj is being updated check the parent id and workspace id. --- server/api/base.py | 11 +++-- server/api/modules/services.py | 34 +++++++++------ test_cases/test_api_services.py | 73 ++++++++++++++++++++++++++++++++- 3 files changed, 101 insertions(+), 17 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index fb035f999a1..aa76534df7d 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -387,8 +387,10 @@ class CreateMixin(object): """Add POST / route""" def post(self, **kwargs): - data = self._parse_data(self._get_schema_instance(kwargs), + context = {'updating': False} + data = self._parse_data(self._get_schema_instance(kwargs, context=context), flask.request) + data.pop('id', None) created = self._perform_create(data, **kwargs) created.creator = g.user db.session.commit() @@ -428,9 +430,12 @@ class UpdateMixin(object): """Add PUT /// route""" def put(self, object_id, **kwargs): - data = self._parse_data(self._get_schema_instance(kwargs), - flask.request) obj = self._get_object(object_id, **kwargs) + context = {'updating': True, 'object': obj} + data = self._parse_data(self._get_schema_instance(kwargs, context=context), + flask.request) + # just in case an schema allows id as writable. + data.pop('id', None) self._update_object(obj, data) updated = self._perform_update(object_id, obj, **kwargs) return self._dump(obj, kwargs), 200 diff --git a/server/api/modules/services.py b/server/api/modules/services.py index d5daa2f4209..fdb34461a13 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -23,7 +23,7 @@ class ServiceSchema(AutoSchema): - _id = fields.Integer(dump_only=True, attribute='id') + _id = fields.Integer(attribute='id', dump_only=True) _rev = fields.String(default='', dump_only=True) owned = fields.Boolean(default=False) owner = PrimaryKeyRelatedField('username', dump_only=True, @@ -34,7 +34,7 @@ class ServiceSchema(AutoSchema): required=True, attribute='port') status = fields.String(default='open') - parent = fields.Integer(attribute='host_id', load_only=True, required=True) + parent = fields.Integer(attribute='host_id', load_only=True) # parent is not required for updates host_id = fields.Integer(attribute='host_id', dump_only=True) summary = fields.Method('get_summary') vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) @@ -54,17 +54,25 @@ def post_load_parent(self, data): get a Host with that id in the corresponding workspace. """ host_id = data.pop('host_id', None) - print data - if host_id is None: - # Partial update? - return - try: - data['host'] = Host.query.join(Workspace).filter( - Workspace.name == self.context['workspace_name'], - Host.id == host_id - ).one() - except NoResultFound: - raise ValidationError('Host with id {} not found'.format(host_id)) + if self.context['updating']: + if host_id is None: + # Partial update? + return + + if host_id != self.context['object'].parent.id: + raise ValidationError('Can\'t change service parent.') + + else: + if not host_id: + raise ValidationError('Parent id is required when creating a service.') + + try: + data['host'] = Host.query.join(Workspace).filter( + Workspace.name == self.context['workspace_name'], + Host.id == host_id + ).one() + except NoResultFound: + raise ValidationError('Host with id {} not found'.format(host_id)) class Meta: model = Service diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index c63a0eb1b97..8822dfcea49 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -9,6 +9,7 @@ from server.models import ( Service ) +from test_cases.factories import HostFactory @pytest.mark.usefixtures('logged_user') @@ -100,4 +101,74 @@ def test_update_fails_with_host_of_other_workspace(self, test_client, } res = test_client.put(self.url(self.first_object), data=data) assert res.status_code == 400 - assert 'Host with id' in res.data + assert 'Can\'t change service parent.' in res.data + + def _raw_put_data(self, id, parent=None, status='open', protocol='tcp', ports=None): + if not ports: + ports = [22] + raw_data = {"status": status, + "protocol": protocol, + "description": "", + "_rev": "", + "metadata": {"update_time": 1510945708000, "update_user": "", "update_action": 0, "creator": "", + "create_time": 1510945708000, "update_controller_action": "", "owner": "leonardo", + "command_id": None}, + "owned": False, + "owner": "", + "version": "", + "_id": id, + "ports": ports, + "name": "ssh2", + "type": "Service"} + if parent: + raw_data['parent'] = parent + return raw_data + + def test_update_with_json_from_webui(self, test_client, session): + service = self.factory() + session.commit() + raw_data = self._raw_put_data(service.id) + + res = test_client.put(self.url(service, workspace=service.workspace), data=raw_data) + assert res.status_code == 200 + updated_service = Service.query.filter_by(id=service.id).first() + assert updated_service.status == 'open' + assert updated_service.name == 'ssh2' + + def test_update_cant_change_parent(self, test_client, session): + service = self.factory() + host = HostFactory.create() + session.commit() + raw_data = self._raw_put_data(service.id, parent=host.id) + res = test_client.put(self.url(service, workspace=service.workspace), data=raw_data) + assert res.status_code == 400 + assert 'Can\'t change service parent.' in res.data + updated_service = Service.query.filter_by(id=service.id).first() + assert updated_service.name == service.name + + def test_update_status(self, test_client, session): + service = self.factory(status='open') + session.commit() + raw_data = self._raw_put_data(service.id, parent=service.host.id, status='closed') + res = test_client.put(self.url(service, workspace=service.workspace), data=raw_data) + assert res.status_code == 200 + updated_service = Service.query.filter_by(id=service.id).first() + assert updated_service.status == 'closed' + + def test_update_ports(self, test_client, session): + service = self.factory(port=22) + session.commit() + raw_data = self._raw_put_data(service.id, parent=service.host.id, ports=[221]) + res = test_client.put(self.url(service, workspace=service.workspace), data=raw_data) + assert res.status_code == 200 + updated_service = Service.query.filter_by(id=service.id).first() + assert updated_service.port == 221 + + def test_update_cant_change_id(self, test_client, session): + service = self.factory() + host = HostFactory.create() + session.commit() + raw_data = self._raw_put_data(service.id) + res = test_client.put(self.url(service, workspace=service.workspace), data=raw_data) + assert res.status_code == 200 + assert res.json['id'] == service.id From 542d50fd751a9e4b806ca540f7f3d724dfab6bfd Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 17 Nov 2017 18:42:54 -0300 Subject: [PATCH 0546/1506] Fix Host edit --- server/api/modules/hosts.py | 2 +- test_cases/test_api_hosts.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 9fd73d2114d..a86a921e178 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -29,7 +29,7 @@ class HostSchema(AutoSchema): _rev = fields.String(default='') ip = fields.String(default='') description = fields.String(required=True) # Explicitly set required=True - default_gateway = fields.String(attribute="default_gateway_ip") + default_gateway = fields.String(attribute="default_gateway_ip", allow_none=True) name = fields.String(dump_only=True, attribute='ip', default='') os = fields.String(default='') owned = fields.Boolean(default=False) diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 6037202c779..c5a9d6e8dbc 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -344,6 +344,7 @@ def test_update_host(self, test_client, session): "ip":"10.31.112.21", "_rev":"", "description":"", + "default_gateway": None, "owned": False, "services":12, "hostnames":[], From 8caa9a4e09c996caeca78c635a52ffd76fd3348f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 17 Nov 2017 18:43:24 -0300 Subject: [PATCH 0547/1506] Goodbye ok, undefined, you win message! --- server/www/scripts/commons/providers/server.js | 8 -------- 1 file changed, 8 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index 3b8b6c210b4..1bebfb0498e 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -87,14 +87,6 @@ angular.module("faradayApp") var send_data = function(url, data, is_update, method) { // undefined is just evil... if (typeof is_update === "undefined") {var is_update = false;} - if (is_update && !data._rev) { - // ok, undefined, you win - console.log('ok, undefined, you win'); - return get(url).then(function s(r) { - data._rev = r.data._rev; - return serverComm(method, url, data); - }).catch(function e(r) {$q.reject(r)}); - } return serverComm(method, url, data); }; From ffa56c28b016d5a62c18a64d45e8479e64044339 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 17 Nov 2017 19:38:54 -0300 Subject: [PATCH 0548/1506] Fix broken tests in postgresql --- test_cases/test_api_hosts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index c5a9d6e8dbc..b8b08c259a2 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -360,13 +360,13 @@ def test_update_host(self, test_client, session): assert res.status_code == 200 updated_host = Host.query.filter_by(id=host.id).first() assert res.json == { - u'_id': 6, + u'_id': host.id, u'_rev': u'', u'credentials': 0, u'default_gateway': None, u'description': u'', u'hostnames': [], - u'id': 6, + u'id': host.id, u'ip': u'10.31.112.21', u'metadata': {u'command_id': None, u'create_time': int(time.mktime(updated_host.create_date.timetuple())) * 1000, From 8744fb9fc2f5b006be5f407d401545a380f89cdb Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 21 Nov 2017 19:10:25 -0300 Subject: [PATCH 0549/1506] Remove flask script since it was deprecated. --- manage.py | 70 +++++++++++++++++------ requirements_server.txt | 1 - server/commands/app_urls.py | 7 +-- server/commands/faraday_schema_display.py | 3 +- server/commands/initdb.py | 3 +- server/commands/reset_db.py | 22 ++++--- server/importer.py | 9 ++- 7 files changed, 71 insertions(+), 44 deletions(-) diff --git a/manage.py b/manage.py index 8f01b04d1bc..437d1ba3ea8 100755 --- a/manage.py +++ b/manage.py @@ -1,25 +1,61 @@ #!/usr/bin/env python -try: - from flask_script import Manager -except ImportError: - print('Flask-Script missing. Install requirements with "pip install -r requirements_server.txt') - -from server.web import app +import click from server.importer import ImportCouchDB from server.commands.initdb import InitDB from server.commands.faraday_schema_display import DatabaseSchema -from server.commands.app_urls import AppUrls -from server.commands.reset_db import ResetDB +from server.commands.app_urls import show_all_urls +from server.commands.reset_db import reset_db_all +from server.commands.reports import import_external_reports + +from server.web import app + + + +@click.group() +def cli(): + pass + +@click.command() +def process_reports(): + import_external_reports() + +@click.command() +def reset_db(): + with app.app_context(): + reset_db_all() + +@click.command() +def show_urls(): + show_all_urls() + +@click.command() +def faraday_schema_display(): + DatabaseSchema().run() + +@click.command() +def initdb(): + with app.app_context(): + InitDB().run() + +@click.command() +def import_from_couchdb(): + with app.app_context(): + ImportCouchDB().run() + +@click.command() +def database_schema(): + DatabaseSchema().run() + -manager = Manager(app) +cli.add_command(process_reports) +cli.add_command(reset_db) +cli.add_command(show_urls) +cli.add_command(faraday_schema_display) +cli.add_command(initdb) +cli.add_command(import_from_couchdb) +cli.add_command(database_schema) -if __name__ == "__main__": - manager.add_command('import_from_couchdb', ImportCouchDB()) - manager.add_command('generate_database_schemas', DatabaseSchema()) - manager.add_command('initdb', InitDB()) - manager.add_command('faraday_schema_display', DatabaseSchema()) - manager.add_command('show_urls', AppUrls()) - manager.add_command('reset_db', ResetDB()) - manager.run() +if __name__ == '__main__': + cli() diff --git a/requirements_server.txt b/requirements_server.txt index 21925639d47..fd02d837bd0 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -1,7 +1,6 @@ bcrypt couchdbkit==0.6.5 Flask-SQLAlchemy==2.3.1 -Flask-Script==2.0.5 flask-classful==0.14 Flask-Security==3.0.0 flask==0.12.2 diff --git a/server/commands/app_urls.py b/server/commands/app_urls.py index 48bba9cc029..4e1066ac05a 100644 --- a/server/commands/app_urls.py +++ b/server/commands/app_urls.py @@ -1,8 +1,5 @@ -from flask_script import Command - from server.web import app -class AppUrls(Command): - def run(self): - print(app.url_map) \ No newline at end of file +def show_all_urls(): + print(app.url_map) \ No newline at end of file diff --git a/server/commands/faraday_schema_display.py b/server/commands/faraday_schema_display.py index a402f19764e..de224b767d2 100644 --- a/server/commands/faraday_schema_display.py +++ b/server/commands/faraday_schema_display.py @@ -1,6 +1,5 @@ from __future__ import print_function import sys -from flask_script import Command from sqlalchemy import MetaData try: from sqlalchemy_schemadisplay import create_schema_graph @@ -14,7 +13,7 @@ import server.config -class DatabaseSchema(Command): +class DatabaseSchema(): def run(self): self._draw_entity_diagrama() diff --git a/server/commands/initdb.py b/server/commands/initdb.py index b9ec8152e24..db23a535d80 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -15,7 +15,6 @@ from ConfigParser import ConfigParser, NoSectionError, NoOptionError from flask import current_app -from flask_script import Command from colorama import init from colorama import Fore, Back, Style from sqlalchemy.exc import OperationalError @@ -25,7 +24,7 @@ init() -class InitDB(Command): +class InitDB(): def _check_current_config(self, config): try: diff --git a/server/commands/reset_db.py b/server/commands/reset_db.py index 1aed2ce90f0..cc9cc3c56b4 100644 --- a/server/commands/reset_db.py +++ b/server/commands/reset_db.py @@ -1,15 +1,13 @@ -from flask_script import Command from server.models import db -class ResetDB(Command): - def run(self): - # It might be required to do a cascade delete to correctly the - # vulnerability table - for table in ('vulnerability', 'vulnerability_template', 'comment'): - try: - db.engine.execute('DROP TABLE {} CASCADE'.format(table)) - except: - pass - db.drop_all() - db.create_all() +def reset_db_all(): + # It might be required to do a cascade delete to correctly the + # vulnerability table + for table in ('vulnerability', 'vulnerability_template', 'comment'): + try: + db.engine.execute('DROP TABLE {} CASCADE'.format(table)) + except: + pass + db.drop_all() + db.create_all() diff --git a/server/importer.py b/server/importer.py index 7e3f3ee491e..ff46cb36c8c 100644 --- a/server/importer.py +++ b/server/importer.py @@ -19,7 +19,6 @@ from binascii import unhexlify from IPy import IP -from flask_script import Command as FlaskScriptCommand from passlib.utils.binary import ab64_encode from restkit.errors import RequestError, Unauthorized from tqdm import tqdm @@ -828,7 +827,7 @@ def get_importer_from_document(self, doc_type): return importer_self -class ImportCouchDBUsers(FlaskScriptCommand): +class ImportCouchDBUsers(): def modular_crypt_pbkdf2_sha1(self, checksum, salt, iterations=1000): return '$pbkdf2${iterations}${salt}${checksum}'.format( @@ -917,7 +916,7 @@ def run(self): -class ImportVulnerabilityTemplates(FlaskScriptCommand): +class ImportVulnerabilityTemplates(): def __init__(self): self.names = Counter() @@ -992,7 +991,7 @@ def get_severity(self, document): return default -class ImportLicense(FlaskScriptCommand): +class ImportLicense(): def run(self): licenses_url = "http://{username}:{password}@{hostname}:{port}/{path}".format( @@ -1026,7 +1025,7 @@ def run(self): ) -class ImportCouchDB(FlaskScriptCommand): +class ImportCouchDB(): def _open_couchdb_conn(self): try: couchdb_server_conn = server.couchdb.CouchDBServer() From e0613e6e936a11bcadce085a0792104add49fa68 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 21 Nov 2017 19:14:33 -0300 Subject: [PATCH 0550/1506] add generate reports command --- server/commands/reports.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/server/commands/reports.py b/server/commands/reports.py index e69de29bb2d..3ac9abb0f04 100644 --- a/server/commands/reports.py +++ b/server/commands/reports.py @@ -0,0 +1,35 @@ +import os +import logging + +from managers.mapper_manager import MapperManager +from managers.reports_managers import ReportManager, CONF + +from plugins.controller import PluginController +from plugins.manager import PluginManager +from server.models import Workspace + +logger = logging.getLogger(__name__) + + +def import_external_reports(): + threads = [] + plugin_manager = PluginManager( + os.path.join(CONF.getConfigPath(), "plugins")) + mappers_manager = MapperManager() + + plugin_controller = PluginController( + 'PluginController', + plugin_manager, + mappers_manager + ) + for workspace in Workspace.query.all(): + report_manager = ReportManager( + 10, + workspace.name, + plugin_controller + ) + threads.append(report_manager) + report_manager.start() + + for thread in threads: + thread.join() From 0debdd1132867b0dd5f6f15db00320256f6ca56f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 22 Nov 2017 12:04:31 -0300 Subject: [PATCH 0551/1506] Print a message when database is down --- manage.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/manage.py b/manage.py index 35fad6a111b..24ccddfd0a6 100755 --- a/manage.py +++ b/manage.py @@ -1,6 +1,8 @@ #!/usr/bin/env python import click +from sqlalchemy.exc import OperationalError + from server.importer import ImportCouchDB from server.commands.initdb import InitDB from server.commands.faraday_schema_display import DatabaseSchema @@ -17,7 +19,12 @@ def cli(): @click.command() def process_reports(): - import_external_reports() + with app.app_context(): + try: + import_external_reports() + except OperationalError as ex: + print('{0}'.format(ex)) + print('Please verify database is running or configuration on server.ini!') @click.command() def reset_db(): From e31c6de4077f3ace2f97ae6bb9f95a8e72112510 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 22 Nov 2017 18:34:05 -0300 Subject: [PATCH 0552/1506] Print response text when an error occurred --- persistence/server/server.py | 2 +- persistence/server/server_io_exceptions.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/persistence/server/server.py b/persistence/server/server.py index 46b5d2b2cf1..0cac24d0ebd 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -125,7 +125,7 @@ def _unsafe_io_with_server(server_io_function, server_expected_response, if answer.status_code != server_expected_response: raise requests.exceptions.RequestException(response=answer) except requests.exceptions.RequestException: - raise CantCommunicateWithServerError(server_io_function, server_url, payload) + raise CantCommunicateWithServerError(server_io_function, server_url, payload, answer) return answer diff --git a/persistence/server/server_io_exceptions.py b/persistence/server/server_io_exceptions.py index fc3ee50d4a8..e88a8b337c0 100644 --- a/persistence/server/server_io_exceptions.py +++ b/persistence/server/server_io_exceptions.py @@ -21,15 +21,16 @@ def __str__(self): class CantCommunicateWithServerError(ServerRequestException): - def __init__(self, function, server_url, payload): + def __init__(self, function, server_url, payload, response): self.function = function self.server_url = server_url self.payload = payload + self.response = response def __str__(self): return ("Couldn't get a valid response from the server when requesting " - "to URL {0} and function {1}.".format(self.server_url, - self.function)) + "to URL {0} and function {1}. Response was {2}".format(self.server_url, + self.function, self.response.text)) class ConflictInDatabase(ServerRequestException): def __init__(self, answer): From 546d68b62ee2d40c701878de3f0c17c5f9483ae2 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 23 Nov 2017 11:53:16 -0300 Subject: [PATCH 0553/1506] Add initial changes to support plugins. * Added requests session (auth is missing in this commit). * Add support for commands creation in POST. * Add unit test to verify command creation with duration in none. --- managers/mapper_manager.py | 11 +++---- managers/reports_managers.py | 20 ++++++------- persistence/server/models.py | 8 +++--- persistence/server/server.py | 48 +++++++++++++++++-------------- persistence/server/utils.py | 17 +++++------ plugins/controller.py | 2 +- server/api/modules/commandsrun.py | 19 ++++++++---- server/commands/reports.py | 8 +++--- server/schemas.py | 3 +- test_cases/test_api_commands.py | 16 ++++++++++- 10 files changed, 91 insertions(+), 61 deletions(-) diff --git a/managers/mapper_manager.py b/managers/mapper_manager.py index 7b6af550c2a..e9a51bf5e01 100644 --- a/managers/mapper_manager.py +++ b/managers/mapper_manager.py @@ -3,7 +3,7 @@ Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) See the file 'doc/LICENSE' for the license information ''' - +from requests import Session from persistence.server.models import create_object, get_object, update_object, delete_object # NOTE: This class is intended to be instantiated by the @@ -17,22 +17,23 @@ class MapperManager(object): def __init__(self): # create and store the datamappers self.workspace_name = None + self.session = None def createMappers(self, workpace_name): self.workspace_name = workpace_name def save(self, obj): - if create_object(self.workspace_name, obj.class_signature, obj): + if create_object(self.workspace_name, obj.class_signature, obj, self.session): return True return False def update(self, obj): - if update_object(self.workspace_name, obj.class_signature, obj): + if update_object(self.workspace_name, obj.class_signature, obj, self.session): return True return False def find(self, class_signature, obj_id): - return get_object(self.workspace_name, class_signature, obj_id) + return get_object(self.workspace_name, class_signature, obj_id, self.session) def remove(self, obj_id, class_signature): - return delete_object(self.workspace_name, class_signature, obj_id) \ No newline at end of file + return delete_object(self.workspace_name, class_signature, obj_id, self.session) \ No newline at end of file diff --git a/managers/reports_managers.py b/managers/reports_managers.py index 12b9f5b3021..9c80b462f30 100755 --- a/managers/reports_managers.py +++ b/managers/reports_managers.py @@ -7,11 +7,11 @@ ''' import os -#import model.api -import threading +import re import time import traceback -import re +from multiprocessing import Process + from utils.logs import getLogger @@ -63,11 +63,11 @@ def onlinePlugin(self, cmd): self.plugin_controller.onCommandFinished('0', 0, cmd) -class ReportManager(threading.Thread): +class ReportManager(Process): def __init__(self, timer, ws_name, plugin_controller): - threading.Thread.__init__(self) + Process.__init__(self) self.ws_name = ws_name - self.setDaemon(True) + self.daemon = True self.timer = timer self._stop = False self._report_path = os.path.join(CONF.getReportPath(), ws_name) @@ -85,19 +85,19 @@ def __init__(self, timer, ws_name, plugin_controller): os.mkdir(self._report_upath) def run(self): - tmp_timer = 0 + tmp_timer = .0 tmp_timer_sentinel = 0 while not self._stop: - time.sleep(1) - tmp_timer += 1 + time.sleep(.1) + tmp_timer += .1 tmp_timer_sentinel += 1 if tmp_timer_sentinel == 1800: tmp_timer_sentinel = 0 self.launchSentinel() - if tmp_timer == self.timer: + if tmp_timer >= self.timer: try: self.syncReports() except Exception: diff --git a/persistence/server/models.py b/persistence/server/models.py index 3033b8114ae..91d8cd0bb80 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -408,9 +408,9 @@ def update_credential(workspace_name, credential): return server.update_credential(workspace_name, **credential_properties) @_ignore_in_changes -def create_command(workspace_name, command): +def create_command(workspace_name, command, session): command_properties = get_command_properties(command) - return server.create_command(workspace_name, **command_properties) + return server.create_command(workspace_name, session, **command_properties) @_ignore_in_changes def update_command(workspace_name, command): @@ -420,7 +420,7 @@ def update_command(workspace_name, command): command_properties = get_command_properties(command) return server.update_command(workspace_name, **command_properties) -def create_object(workspace_name, object_signature, obj): +def create_object(workspace_name, object_signature, obj, session): """Given a workspace name, an object_signature as string and obj, a Faraday object, save that object on the server. @@ -442,7 +442,7 @@ def create_object(workspace_name, object_signature, obj): except KeyError: raise WrongObjectSignature(object_signature) - return appropiate_function(workspace_name, obj) + return appropiate_function(workspace_name, obj, session) def update_object(workspace_name, object_signature, obj): """Given a workspace name, an object_signature as string and obj, a Faraday diff --git a/persistence/server/server.py b/persistence/server/server.py index 0cac24d0ebd..2c039765c08 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -56,7 +56,7 @@ def _get_base_server_url(): def _create_server_api_url(): """Return the server's api url.""" - return "{0}/_api".format(_get_base_server_url()) + return "{0}/_api/v2".format(_get_base_server_url()) def _create_server_get_url(workspace_name, object_name=None): """Creates a url to get from the server. Takes the workspace name @@ -74,9 +74,10 @@ def _create_server_get_url(workspace_name, object_name=None): return get_url -def _create_server_post_url(workspace_name, object_id): +def _create_server_post_url(workspace_name, obj_type): server_api_url = _create_server_api_url() - post_url = '{0}/ws/{1}/doc/{2}'.format(server_api_url, workspace_name, object_id) + object_end_point_name = OBJECT_END_POINT_MAPPER[obj_type] + post_url = '{0}/ws/{1}/{2}/'.format(server_api_url, workspace_name, object_end_point_name) return post_url @@ -137,7 +138,7 @@ def _parse_json(response_object): return {} -def _get(request_url, **params): +def _get(request_url, session, **params): """Get from the request_url. Takes an arbitrary number of parameters to customize the request_url if necessary. @@ -146,12 +147,12 @@ def _get(request_url, **params): Return a dictionary with the information in the json. """ - return _parse_json(_unsafe_io_with_server(requests.get, + return _parse_json(_unsafe_io_with_server(session.get, 200, request_url, params=params)) -def _put(post_url, update=False, expected_response=201, **params): +def _put(post_url, session, update=False, expected_response=201, **params): """Put to the post_url. If update is True, try to get the object revision first so as to update the object in Couch. You can customize the expected response (it should be 201, but Couchdbkit returns @@ -167,20 +168,27 @@ def _put(post_url, update=False, expected_response=201, **params): if update: last_rev = _get(post_url)['_rev'] params['_rev'] = last_rev - return _parse_json(_unsafe_io_with_server(requests.put, + return _parse_json(_unsafe_io_with_server(session.put, expected_response, post_url, json=params)) -def _delete(delete_url, database=False): +def _post(post_url, session, update=False, expected_response=201, **params): + return _parse_json(_unsafe_io_with_server(session.post, + expected_response, + post_url, + json=params)) + + +def _delete(delete_url, session, database=False): """Deletes the object on delete_url. If you're deleting a database, specify the database parameter to True""" params = {} if not database: last_rev = _get(delete_url)['_rev'] params = {'rev': last_rev} - return _parse_json(_unsafe_io_with_server(requests.delete, + return _parse_json(_unsafe_io_with_server(session.delete, 200, delete_url, params=params)) @@ -237,23 +245,21 @@ def _get_raw_workspace_summary(workspace_name): request_url = _create_server_get_url(workspace_name, 'summary') return _get(request_url) -# XXX: COUCH IT! def _save_to_couch(workspace_name, faraday_object_id, **params): post_url = _create_couch_post_url(workspace_name, faraday_object_id) - return _put(post_url, update=False, **params) + return _post(post_url, update=False, **params) -# XXX: COUCH IT! def _update_in_couch(workspace_name, faraday_object_id, **params): post_url = _create_server_post_url(workspace_name, faraday_object_id) return _put(post_url, update=True, **params) -def _save_to_server(workspace_name, faraday_object_id, **params): - post_url = _create_server_post_url(workspace_name, faraday_object_id) - return _put(post_url, update=False, expected_response=200, **params) +def _save_to_server(workspace_name, session, **params): + post_url = _create_server_post_url(workspace_name, params['type']) + return _post(post_url, session, update=False, expected_response=200, **params) -def _update_in_server(workspace_name, faraday_object_id, **params): +def _update_in_server(workspace_name, faraday_object_id, session, **params): post_url = _create_server_post_url(workspace_name, faraday_object_id) - return _put(post_url, update=True, expected_response=200, **params) + return _put(post_url, session, update=True, expected_response=200, **params) def _save_db_to_server(db_name, **params): post_url = _create_server_db_url(db_name) @@ -1341,14 +1347,14 @@ def update_credential(workspace_name, id, name, username, password, password=password, type="Cred") -def create_command(workspace_name, id, command, duration=None, hostname=None, +def create_command(workspace_name, session, command, duration=None, hostname=None, ip=None, itime=None, params=None, user=None): """Creates a command. Args: workspace_name (str): the name of the workspace where the vuln web will be saved. - id (str): the id of the vuln web. Must be unique. command (str): the command to be created + session (obj): request authenticated Session instance duration (str). the command's duration hostname (str): the hostname where the command was executed ip (str): the ip of the host where the command was executed @@ -1360,7 +1366,7 @@ def create_command(workspace_name, id, command, duration=None, hostname=None, A dictionary with the server's response. """ return _save_to_server(workspace_name, - id, + session, command=command, duration=duration, hostname=hostname, @@ -1371,7 +1377,7 @@ def create_command(workspace_name, id, command, duration=None, hostname=None, workspace=workspace_name, type="CommandRunInformation") -def update_command(workspace_name, id, command, duration=None, hostname=None, +def update_command(workspace_name, command, duration=None, hostname=None, ip=None, itime=None, params=None, user=None): """Updates a command. diff --git a/persistence/server/utils.py b/persistence/server/utils.py index d91b2d99088..8221f87b535 100644 --- a/persistence/server/utils.py +++ b/persistence/server/utils.py @@ -103,11 +103,12 @@ def get_credential_properties(credential): return cred_dict def get_command_properties(command): - return {'id': command.getID(), - 'command': command.command, - 'user': command.user, - 'ip': command.ip, - 'hostname': command.hostname, - 'itime': command.itime, - 'duration': command.duration, - 'params': command.params} + return { + 'command': command.command, + 'user': command.user, + 'ip': command.ip, + 'hostname': command.hostname, + 'itime': command.itime, + 'duration': command.duration, + 'params': command.params + } diff --git a/plugins/controller.py b/plugins/controller.py index 3970281451b..97a4499c716 100644 --- a/plugins/controller.py +++ b/plugins/controller.py @@ -249,7 +249,7 @@ def processReport(self, plugin, filepath, ws_name=None): self._mapper_manager.save(cmd_info) if plugin in self._plugins: - self.processOutput(self._plugins[plugin], filepath, cmd_info.getID(), True ) + self.processOutput(self._plugins[plugin], filepath, cmd_info.getID(), True) cmd_info.duration = time.time() - cmd_info.itime self._mapper_manager.update(cmd_info) return True diff --git a/server/api/modules/commandsrun.py b/server/api/modules/commandsrun.py index 087539e03ec..850665698eb 100644 --- a/server/api/modules/commandsrun.py +++ b/server/api/modules/commandsrun.py @@ -3,10 +3,11 @@ # See the file 'doc/LICENSE' for the license information import time +import datetime import flask from flask import Blueprint from flask_classful import route -from marshmallow import fields +from marshmallow import fields, post_load from server.api.base import AutoSchema, ReadWriteWorkspacedView from server.utils.logger import get_logger @@ -23,9 +24,12 @@ class CommandSchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') - itime = fields.Method('get_itime') - duration = fields.Method('get_duration') - workspace = PrimaryKeyRelatedField('name') + itime = fields.Method(serialize='get_itime', deserialize='load_itime', required=True, attribute='start_date') + duration = fields.Method(serialize='get_duration', allow_none=True) + workspace = PrimaryKeyRelatedField('name', dump_only=True) + + def load_itime(self, value): + return datetime.datetime.fromtimestamp(value) def get_itime(self, obj): return time.mktime(obj.start_date.utctimetuple()) @@ -38,6 +42,12 @@ def get_duration(self, obj): if not obj.start_date and not obj.end_date: return 'Not started' + @post_load + def post_load_set_end_date_with_duration(self, data): + duration = data.pop('duration', None) + if duration: + pass + class Meta: model = Command fields = ('_id', 'command', 'duration', 'itime', 'ip', 'hostname', @@ -78,7 +88,6 @@ def activity_feed(self, workspace_name): 'services_count': command.sum_created_services or 0, 'criticalIssue': command.sum_created_vulnerability_critical or 0, 'date': time.mktime(command.start_date.timetuple()) * 1000, - }) return res diff --git a/server/commands/reports.py b/server/commands/reports.py index 621f6e30192..bc297849d4c 100644 --- a/server/commands/reports.py +++ b/server/commands/reports.py @@ -13,7 +13,7 @@ def import_external_reports(): - threads = [] + processes = [] plugin_manager = PluginManager( os.path.join(CONF.getConfigPath(), "plugins")) mappers_manager = MapperManager() @@ -25,12 +25,12 @@ def import_external_reports(): ) for workspace in Workspace.query.all(): report_manager = ReportManager( - 10, + 0.1, workspace.name, plugin_controller ) - threads.append(report_manager) + processes.append(report_manager) report_manager.start() - for thread in threads: + for thread in processes: thread.join() diff --git a/server/schemas.py b/server/schemas.py index 002d1f1417d..2a8c52dfa30 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -2,8 +2,7 @@ from marshmallow import fields, Schema from marshmallow.exceptions import ValidationError -from server.api.base import AutoSchema -from server.models import CommandObject +from server.models import db, CommandObject class JSTimestampField(fields.Field): diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index ec3f2db2c91..7d6956b4331 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -301,4 +301,18 @@ def test_more_than_one_hour_command_returns_correct_duration_value(self, test_cl ) res = test_client.get(self.url(workspace=command.workspace)) assert res.status_code == 200 - assert res.json['commands'][0]['value']['duration'] == 61.442406 \ No newline at end of file + assert res.json['commands'][0]['value']['duration'] == 61.442406 + + def test_create_command(self, test_client): + raw_data ={ + 'command': 'Import Nessus:', + 'duration': None, + 'hostname': 'mandarina', + 'ip': '192.168.20.53', + 'itime': 1511387720.048548, + 'params': u'/home/lcubo/.faraday/report/airbnb/nessus_report_Remote.nessus', + 'user': 'lcubo' + } + + res = test_client.post(self.url(), data=raw_data) + assert res.status_code == 201 \ No newline at end of file From 7ed6385f59be34c9e0f870d8f164782c187221d5 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 23 Nov 2017 17:03:59 -0300 Subject: [PATCH 0554/1506] Initial recon-ng plugin --- managers/reports_managers.py | 12 +++++++- plugins/repo/reconng/__init__.py | 0 plugins/repo/reconng/plugin.py | 48 ++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 plugins/repo/reconng/__init__.py create mode 100644 plugins/repo/reconng/plugin.py diff --git a/managers/reports_managers.py b/managers/reports_managers.py index 4c5bddca493..e641230e479 100755 --- a/managers/reports_managers.py +++ b/managers/reports_managers.py @@ -5,6 +5,7 @@ See the file 'doc/LICENSE' for the license information ''' +import json import os #import model.api @@ -240,14 +241,23 @@ def open_file(self, file_path): if file_signature.find(key) == 0: result = signatures[key] - getLogger(self).debug("Report type detected: %s" % result) break + if not result: + # try json loads to detect a json file. + try: + json.loads(f.read()) + f.seek(0) + result = 'json' + except ValueError: + pass + except IOError, err: self.report_type = None getLogger(self).error( "Error while opening file.\n%s. %s" % (err, file_path)) + getLogger(self).debug("Report type detected: %s" % result) return f, result def getRootTag(self, file_path): diff --git a/plugins/repo/reconng/__init__.py b/plugins/repo/reconng/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/repo/reconng/plugin.py b/plugins/repo/reconng/plugin.py new file mode 100644 index 00000000000..0478df0c225 --- /dev/null +++ b/plugins/repo/reconng/plugin.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +''' +Faraday Penetration Test IDE +Copyright (C) 2017 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information +''' +import re + +from plugins.core import PluginBase + +__author__ = 'Leonardo Lazzaro' +__copyright__ = 'Copyright (c) 2017, Infobyte LLC' +__credits__ = ['Leonardo Lazzaro'] +__license__ = '' +__version__ = '0.1.0' +__maintainer__ = 'Leonardo Lazzaro' +__email__ = 'leonardol@infobytesec.com' +__status__ = 'Development' + + +class ReconngPlugin(PluginBase): + """ + Example plugin to parse qualysguard output. + """ + + def __init__(self): + + PluginBase.__init__(self) + self.id = 'Reconng' + self.name = 'Reconng XML Output Plugin' + self.plugin_version = '0.0.2' + self.version = '' + self.framework_version = '' + self.options = None + self._current_output = None + self._command_regex = re.compile( + r'records added to') + self.importing_report = False + + def parseOutputString(self, output): + pass + + def parseCommandString(self, username, current_path, command_string): + self.importing_report = False + +def createPlugin(): + return ReconngPlugin() \ No newline at end of file From 5d4b43bde7b4ec2bc192c9699c60e95fb928f437 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 23 Nov 2017 17:06:10 -0300 Subject: [PATCH 0555/1506] Add dirsearch plugin It allows running from command line and importing JSON reports. By default it doesn't create vulns with status code 403 but it can be changed in the settings --- plugins/repo/dirsearch/__init__.py | 6 + plugins/repo/dirsearch/plugin.py | 180 +++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 plugins/repo/dirsearch/__init__.py create mode 100644 plugins/repo/dirsearch/plugin.py diff --git a/plugins/repo/dirsearch/__init__.py b/plugins/repo/dirsearch/__init__.py new file mode 100644 index 00000000000..004c49be6c6 --- /dev/null +++ b/plugins/repo/dirsearch/__init__.py @@ -0,0 +1,6 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' diff --git a/plugins/repo/dirsearch/plugin.py b/plugins/repo/dirsearch/plugin.py new file mode 100644 index 00000000000..c92bb308a2f --- /dev/null +++ b/plugins/repo/dirsearch/plugin.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information +''' + +import re +import os +import json +import shlex +import socket +import argparse +import tempfile +import urlparse +from plugins.plugin import PluginTerminalOutput +from plugins.plugin_utils import get_vulnweb_url_fields + + +__author__ = "Matías Lang" +__copyright__ = "Copyright (c) 2013, Infobyte LLC" +__credits__ = ["Matías Lang"] +__license__ = "" +__version__ = "0.0.1" +__maintainer__ = "Matías Lang" +__email__ = "matiasl@infobytesec.com" +__status__ = "Development" + + +status_codes = { + 200: "OK", 201: "Created", 202: "Accepted", + 203: "Non-Authoritative Information", 204: "No Content", + 205: "Reset Content", 206: "Partial Content", 207: "Multi-Status", + 208: "Already Reported", 226: "IM Used", 300: "Multiple Choices", + 301: "Moved Permanently", 302: "Found", 303: "See Other", + 304: "Not Modified", 305: "Use Proxy", 306: "Switch Proxy", + 307: "Temporary Redirect", 308: "Permanent Redirect", + 400: "Bad Request", 401: "Unauthorized", 402: "Payment Required", + 403: "Forbidden", 404: "Not Found", 405: "Method Not Allowed", + 406: "Not Acceptable", 407: "Proxy Authentication Required", + 408: "Request Timeout", 409: "Conflict", 410: "Gone", + 411: "Length Required", 412: "Precondition Failed", + 413: "Payload Too Large", 414: "URI Too Long", + 415: "Unsupported Media Type", 416: "Range Not Satisfiable", + 417: "Expectation Failed", 418: "I'm a teapot", + 421: "Misdirected Request", 422: "Unprocessable Entity", 423: "Locked", + 424: "Failed Dependency", 426: "Upgrade Required", + 428: "Precondition Required", 429: "Too Many Requests", + 431: "Request Header Fields Too Large", + 451: "Unavailable For Legal Reasons", 500: "Internal Server Error", + 501: "Not Implemented", 502: "Bad Gateway", 503: "Service Unavailable", + 504: "Gateway Timeout", 505: "HTTP Version Not Supported", + 506: "Variant Also Negotiates", 507: "Insufficient Storage", + 508: "Loop Detected", 510: "Not Extended", + 511: "Network Authentication Required", +} + + +class DirsearchPlugin(PluginTerminalOutput): + def __init__(self): + super(DirsearchPlugin, self).__init__() + self.id = "dirsearch" + self.name = "dirsearch" + self.plugin_version = "0.0.1" + self.version = "0.0.1" + self._command_regex = re.compile( + r'^(sudo )?(python[0-9\.]? )?dirsearch(\.py)?') + self.ignore_parsing = False + self.json_report_file = None + self.addSetting("Ignore 403", str, "1") + + def parseOutputString(self, output, debug=False): + if self.ignore_parsing: + return + if self.json_report_file: + # We ran the plugin via command line + try: + fp = open(self.json_report_file) + except IOError: + self.log('Error opening JSON in the file {}'.format( + self.json_report_file + ), 'ERROR') + else: + self.parse_json(fp.read()) + if self.remove_report: + os.unlink(self.json_report_file) + else: + # We are importing a report + self.parse_json(output) + + def resolve(self, domain): + return socket.gethostbyname(domain) + + @property + def should_ignore_403(self): + val = self.getSetting('Ignore 403') + if not val or not int(val): + return False + return True + + def parse_json(self, contents): + try: + data = json.loads(contents) + except ValueError: + self.log('Error parsing report. Make sure the file has valid ' + 'JSON', 'ERROR') + return + for (base_url, items) in data.items(): + base_split = urlparse.urlsplit(base_url) + ip = self.resolve(base_split.hostname) + h_id = self.createAndAddHost(ip) + + i_id = self.createAndAddInterface( + h_id, + name=ip, + ipv4_address=ip, + hostname_resolution=[base_split.hostname]) + + s_id = self.createAndAddServiceToInterface( + h_id, + i_id, + base_split.scheme, + 'tcp', + [base_split.port], + status="open") + + for item in items: + self.parse_found_url(base_url, h_id, s_id, item) + + def parse_found_url(self, base_url, h_id, s_id, item): + if self.should_ignore_403 and item['status'] == 403: + return + url = urlparse.urlsplit(urlparse.urljoin(base_url, item['path'])) + response = "HTTP/1.1 {} {}\nContent-Length: {}".format( + item['status'], status_codes.get(item['status'], 'unknown'), + item['content-length']) + redirect = item.get('redirect') + if redirect is not None: + response += '\nLocation: {}'.format(redirect) + self.createAndAddVulnWebToService( + h_id, + s_id, + name='Path found: {} ({})'.format(item['path'], item['status']), + desc="Dirsearch tool found the following URL: {}".format( + url.geturl()), + severity="info", + method='GET', + response=response, + **get_vulnweb_url_fields(url.geturl())) + + def processCommandString(self, username, current_path, command_string): + parser = argparse.ArgumentParser(conflict_handler='resolve') + parser.add_argument('-h', '--help', action='store_true') + parser.add_argument('--json-report') + args, unknown = parser.parse_known_args(shlex.split(command_string)) + + if args.help: + self.devlog('help detected, ignoring parsing') + return command_string + if args.json_report: + # The user already defined a path to the JSON report + self.json_report_file = args.json_report + self.remove_report = False + return command_string + else: + # Use temporal file to save the report data + # TODO: use tempfile + self.json_report_file = tempfile.mktemp( + prefix="dirsearch_report_", suffix=".json") + self.devlog('Setting report file to {}'.format( + self.json_report_file)) + self.remove_report = True + return '{} --json-report {}'.format(command_string, + self.json_report_file) + + +def createPlugin(): + return DirsearchPlugin() From 192cb278cc06224c995c08f270fcabbd58e2e800 Mon Sep 17 00:00:00 2001 From: Winna_Z Date: Thu, 23 Nov 2017 17:55:19 -0300 Subject: [PATCH 0556/1506] severity fix on netspeaker parser --- plugins/repo/netsparkercloud/plugin.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/repo/netsparkercloud/plugin.py b/plugins/repo/netsparkercloud/plugin.py index 5ddd61ade7b..f56e503dd3e 100644 --- a/plugins/repo/netsparkercloud/plugin.py +++ b/plugins/repo/netsparkercloud/plugin.py @@ -37,6 +37,7 @@ __email__ = "famato@infobytesec.com" __status__ = "Development" + def cleaner_unicode(string): if string is not None: return string.encode('ascii', errors='backslashreplace') @@ -109,6 +110,10 @@ class Item(object): @param item_node A item_node taken from an netsparkercloud xml tree """ + def re_map_severity(self, severity): + if severity == "Important": + return "high" + return severity def __init__(self, item_node): self.node = item_node @@ -128,7 +133,7 @@ def __init__(self, item_node): self.type = self.get_text_from_subnode("type") self.name = self.get_text_from_subnode("name") - self.severity = self.get_text_from_subnode("severity") + self.severity = self.re_map_severity(self.get_text_from_subnode("severity")) self.certainty = self.get_text_from_subnode("certainty") From 295c22a0b218f71550a1deaf726e7b9e6b493f4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 23 Nov 2017 18:02:00 -0300 Subject: [PATCH 0557/1506] Add sublist3r plugin --- plugins/repo/dirsearch/plugin.py | 1 - plugins/repo/sublist3r/__init__.py | 6 ++ plugins/repo/sublist3r/plugin.py | 109 +++++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 plugins/repo/sublist3r/__init__.py create mode 100644 plugins/repo/sublist3r/plugin.py diff --git a/plugins/repo/dirsearch/plugin.py b/plugins/repo/dirsearch/plugin.py index c92bb308a2f..48e5a2ca33d 100644 --- a/plugins/repo/dirsearch/plugin.py +++ b/plugins/repo/dirsearch/plugin.py @@ -166,7 +166,6 @@ def processCommandString(self, username, current_path, command_string): return command_string else: # Use temporal file to save the report data - # TODO: use tempfile self.json_report_file = tempfile.mktemp( prefix="dirsearch_report_", suffix=".json") self.devlog('Setting report file to {}'.format( diff --git a/plugins/repo/sublist3r/__init__.py b/plugins/repo/sublist3r/__init__.py new file mode 100644 index 00000000000..004c49be6c6 --- /dev/null +++ b/plugins/repo/sublist3r/__init__.py @@ -0,0 +1,6 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' diff --git a/plugins/repo/sublist3r/plugin.py b/plugins/repo/sublist3r/plugin.py new file mode 100644 index 00000000000..9d93d1e22b5 --- /dev/null +++ b/plugins/repo/sublist3r/plugin.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information +''' + +import re +import os +import shlex +import socket +import argparse +import tempfile +from plugins.plugin import PluginTerminalOutput + + +__author__ = "Matías Lang" +__copyright__ = "Copyright (c) 2013, Infobyte LLC" +__credits__ = ["Matías Lang"] +__license__ = "" +__version__ = "0.0.1" +__maintainer__ = "Matías Lang" +__email__ = "matiasl@infobytesec.com" +__status__ = "Development" + + +class Sublist3rPlugin(PluginTerminalOutput): + def __init__(self): + super(Sublist3rPlugin, self).__init__() + self.id = "sublist3r" + self.name = "sublist3r" + self.plugin_version = "0.0.1" + self.version = "0.0.1" + self._command_regex = re.compile( + r'^(sudo )?(python[0-9\.]? )?sublist3r(\.py)?') + self.ignore_parsing = False + self.report_file = None + + def parseOutputString(self, output, debug=False): + if self.ignore_parsing: + return + if self.report_file: + # We ran the plugin via command line + try: + fp = open(self.report_file) + except IOError: + self.log('Error opening report file {}'.format( + self.report_file + ), 'ERROR') + else: + self.parse_report(fp.read()) + if self.remove_report: + os.unlink(self.report_file) + else: + # We are importing a report + self.parse_report(output) + + def resolve(self, domain): + return socket.gethostbyname(domain) + + def parse_report(self, contents): + for line in contents.splitlines(): + hostname = line.strip() + if not hostname: + continue + try: + ip = self.resolve(hostname) + except socket.gaierror: + self.log('Error resolving hostname {}. Skipping.'.format( + hostname + ), 'ERROR') + continue + h_id = self.createAndAddHost(ip) + + self.createAndAddInterface( + h_id, + name=ip, + ipv4_address=ip, + hostname_resolution=[hostname]) + + def processCommandString(self, username, current_path, command_string): + parser = argparse.ArgumentParser(conflict_handler='resolve') + parser.add_argument('-h', '--help', action='store_true') + parser.add_argument('-o', '--output') + args, unknown = parser.parse_known_args(shlex.split(command_string)) + + if args.help: + self.devlog('help detected, ignoring parsing') + return command_string + if args.output: + # The user already defined a path to the report + self.report_file = args.output + self.remove_report = False + return command_string + else: + # Use temporal file to save the report data + self.report_file = tempfile.mktemp( + prefix="sublist3r_report_", suffix=".txt") + self.devlog('Setting report file to {}'.format( + self.report_file)) + self.remove_report = True + return '{} --output {}'.format(command_string, + self.report_file) + + +def createPlugin(): + return Sublist3rPlugin() From e0ea4779b515de799757c49bf6e8568b63d726cf Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 23 Nov 2017 19:07:27 -0300 Subject: [PATCH 0558/1506] Import hosts and vulns from recon-ng --- plugins/repo/reconng/plugin.py | 42 ++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/plugins/repo/reconng/plugin.py b/plugins/repo/reconng/plugin.py index 0478df0c225..2f5059a59f3 100644 --- a/plugins/repo/reconng/plugin.py +++ b/plugins/repo/reconng/plugin.py @@ -6,6 +6,8 @@ See the file 'doc/LICENSE' for the license information ''' import re +import json +import logging from plugins.core import PluginBase @@ -18,6 +20,8 @@ __email__ = 'leonardol@infobytesec.com' __status__ = 'Development' +logger = logging.getLogger(__name__) + class ReconngPlugin(PluginBase): """ @@ -36,11 +40,45 @@ def __init__(self): self._current_output = None self._command_regex = re.compile( r'records added to') - self.importing_report = False + self.importing_report = True - def parseOutputString(self, output): + def load_from_report(self, output): + # TODO: add credentials and ports + reconng_data = json.loads(output) + hosts_id_mapper = {} + for host in reconng_data.get('hosts', []): + h_id = self.createAndAddHost( + host['ip_address'] or host['host'] + ) + hosts_id_mapper[host['host']] = h_id + + severity_mapper = { + 'Information Disclosure': 'informational' + } + for vulnerability in reconng_data.get('vulnerabilities', []): + if vulnerability['host'] not in hosts_id_mapper: + logger.warn('Could not find host_id, skipping vulnerability') + continue + severity = 'info' + if 'SSL' in vulnerability['category']: + severity = 'med' + self.createAndAddVulnToHost( + name='Recon-ng found: ' + vulnerability['example'], + desc='Found by module: ' + vulnerability['module'], + severity=severity, + ref=[vulnerability['reference']], + host_id=hosts_id_mapper[vulnerability['host']] + ) + + def load_from_shell(self, output): pass + def parseOutputString(self, output): + if self.importing_report: + self.load_from_report(output) + else: + self.load_from_shell(output) + def parseCommandString(self, username, current_path, command_string): self.importing_report = False From fd2c6dd5aa947f882b46e52dcf1463a482e40902 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 23 Nov 2017 19:40:33 -0300 Subject: [PATCH 0559/1506] Add support for json files in report directory --- managers/reports_managers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/managers/reports_managers.py b/managers/reports_managers.py index e641230e479..2bb456af21a 100755 --- a/managers/reports_managers.py +++ b/managers/reports_managers.py @@ -276,6 +276,10 @@ def getRootTag(self, file_path): result = "maltego" elif report_type == "dat": result = 'lynis' + elif report_type == 'json': + # this will work since recon-ng is the first plugin to use json. + # we need to add json detection here! + result = 'reconng' else: try: @@ -355,5 +359,7 @@ def rType(self, tag, output): return "Maltego" elif "lynis" == tag: return "Lynis" + elif "reconng" == tag: + return "Reconng" else: return None From e4bf600d3e8ae55a65594088f1ccf83945b324b4 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 24 Nov 2017 12:57:48 -0300 Subject: [PATCH 0560/1506] Add redirect from root to _ui --- server/web.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/web.py b/server/web.py index fb6bc9e36f0..5d33f46ae1e 100644 --- a/server/web.py +++ b/server/web.py @@ -12,6 +12,7 @@ from twisted.internet import ssl, reactor, error from twisted.web.static import File +from twisted.web.util import Redirect from twisted.web.wsgi import WSGIResource from server.utils import logger from server.app import create_app @@ -20,6 +21,7 @@ logger = server.utils.logger.get_logger(__name__) class WebServer(object): + HOME = '' UI_URL_PATH = '_ui' API_URL_PATH = '_api' WEB_UI_LOCAL_PATH = os.path.join(server.config.FARADAY_BASE, 'server/www') @@ -48,11 +50,15 @@ def __load_ssl_certs(self): def __build_server_tree(self): self.__root_resource = Resource() + self.__root_resource.putChild(WebServer.HOME, self.__build_web_redirect()) self.__root_resource.putChild( WebServer.UI_URL_PATH, self.__build_web_resource()) self.__root_resource.putChild( WebServer.API_URL_PATH, self.__build_api_resource()) + def __build_web_redirect(self): + return Redirect(WebServer.UI_URL_PATH) + def __build_web_resource(self): return File(WebServer.WEB_UI_LOCAL_PATH) From 95171924dfc97567f535c971ac022b9a5252ea61 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 24 Nov 2017 15:44:59 -0300 Subject: [PATCH 0561/1506] Remove session from persistence --- managers/mapper_manager.py | 8 ++++---- persistence/server/models.py | 8 ++++---- persistence/server/server.py | 28 +++++++++++++--------------- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/managers/mapper_manager.py b/managers/mapper_manager.py index e9a51bf5e01..2f932d3b11b 100644 --- a/managers/mapper_manager.py +++ b/managers/mapper_manager.py @@ -23,17 +23,17 @@ def createMappers(self, workpace_name): self.workspace_name = workpace_name def save(self, obj): - if create_object(self.workspace_name, obj.class_signature, obj, self.session): + if create_object(self.workspace_name, obj.class_signature, obj): return True return False def update(self, obj): - if update_object(self.workspace_name, obj.class_signature, obj, self.session): + if update_object(self.workspace_name, obj.class_signature, obj): return True return False def find(self, class_signature, obj_id): - return get_object(self.workspace_name, class_signature, obj_id, self.session) + return get_object(self.workspace_name, class_signature, obj_id) def remove(self, obj_id, class_signature): - return delete_object(self.workspace_name, class_signature, obj_id, self.session) \ No newline at end of file + return delete_object(self.workspace_name, class_signature, obj_id) \ No newline at end of file diff --git a/persistence/server/models.py b/persistence/server/models.py index 91d8cd0bb80..3033b8114ae 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -408,9 +408,9 @@ def update_credential(workspace_name, credential): return server.update_credential(workspace_name, **credential_properties) @_ignore_in_changes -def create_command(workspace_name, command, session): +def create_command(workspace_name, command): command_properties = get_command_properties(command) - return server.create_command(workspace_name, session, **command_properties) + return server.create_command(workspace_name, **command_properties) @_ignore_in_changes def update_command(workspace_name, command): @@ -420,7 +420,7 @@ def update_command(workspace_name, command): command_properties = get_command_properties(command) return server.update_command(workspace_name, **command_properties) -def create_object(workspace_name, object_signature, obj, session): +def create_object(workspace_name, object_signature, obj): """Given a workspace name, an object_signature as string and obj, a Faraday object, save that object on the server. @@ -442,7 +442,7 @@ def create_object(workspace_name, object_signature, obj, session): except KeyError: raise WrongObjectSignature(object_signature) - return appropiate_function(workspace_name, obj, session) + return appropiate_function(workspace_name, obj) def update_object(workspace_name, object_signature, obj): """Given a workspace name, an object_signature as string and obj, a Faraday diff --git a/persistence/server/server.py b/persistence/server/server.py index 2c039765c08..685bc03726a 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -138,7 +138,7 @@ def _parse_json(response_object): return {} -def _get(request_url, session, **params): +def _get(request_url, **params): """Get from the request_url. Takes an arbitrary number of parameters to customize the request_url if necessary. @@ -147,12 +147,12 @@ def _get(request_url, session, **params): Return a dictionary with the information in the json. """ - return _parse_json(_unsafe_io_with_server(session.get, + return _parse_json(_unsafe_io_with_server(requests.get, 200, request_url, params=params)) -def _put(post_url, session, update=False, expected_response=201, **params): +def _put(post_url, update=False, expected_response=201, **params): """Put to the post_url. If update is True, try to get the object revision first so as to update the object in Couch. You can customize the expected response (it should be 201, but Couchdbkit returns @@ -168,27 +168,27 @@ def _put(post_url, session, update=False, expected_response=201, **params): if update: last_rev = _get(post_url)['_rev'] params['_rev'] = last_rev - return _parse_json(_unsafe_io_with_server(session.put, + return _parse_json(_unsafe_io_with_server(requests.put, expected_response, post_url, json=params)) -def _post(post_url, session, update=False, expected_response=201, **params): - return _parse_json(_unsafe_io_with_server(session.post, +def _post(post_url, update=False, expected_response=201, **params): + return _parse_json(_unsafe_io_with_server(requests.post, expected_response, post_url, json=params)) -def _delete(delete_url, session, database=False): +def _delete(delete_url, database=False): """Deletes the object on delete_url. If you're deleting a database, specify the database parameter to True""" params = {} if not database: last_rev = _get(delete_url)['_rev'] params = {'rev': last_rev} - return _parse_json(_unsafe_io_with_server(session.delete, + return _parse_json(_unsafe_io_with_server(requests.delete, 200, delete_url, params=params)) @@ -253,13 +253,13 @@ def _update_in_couch(workspace_name, faraday_object_id, **params): post_url = _create_server_post_url(workspace_name, faraday_object_id) return _put(post_url, update=True, **params) -def _save_to_server(workspace_name, session, **params): +def _save_to_server(workspace_name, **params): post_url = _create_server_post_url(workspace_name, params['type']) - return _post(post_url, session, update=False, expected_response=200, **params) + return _post(post_url, update=False, expected_response=200, **params) -def _update_in_server(workspace_name, faraday_object_id, session, **params): +def _update_in_server(workspace_name, faraday_object_id, **params): post_url = _create_server_post_url(workspace_name, faraday_object_id) - return _put(post_url, session, update=True, expected_response=200, **params) + return _put(post_url, update=True, expected_response=200, **params) def _save_db_to_server(db_name, **params): post_url = _create_server_db_url(db_name) @@ -1347,14 +1347,13 @@ def update_credential(workspace_name, id, name, username, password, password=password, type="Cred") -def create_command(workspace_name, session, command, duration=None, hostname=None, +def create_command(workspace_name, command, duration=None, hostname=None, ip=None, itime=None, params=None, user=None): """Creates a command. Args: workspace_name (str): the name of the workspace where the vuln web will be saved. command (str): the command to be created - session (obj): request authenticated Session instance duration (str). the command's duration hostname (str): the hostname where the command was executed ip (str): the ip of the host where the command was executed @@ -1366,7 +1365,6 @@ def create_command(workspace_name, session, command, duration=None, hostname=Non A dictionary with the server's response. """ return _save_to_server(workspace_name, - session, command=command, duration=duration, hostname=hostname, From 83b2af40ed2565d5f422ffe978396a709c2a24ab Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 24 Nov 2017 16:17:24 -0300 Subject: [PATCH 0562/1506] When checking deps avoid adding version conflict packages as missing. --- faraday-server.py | 5 ++++- faraday.py | 5 ++++- utils/dependencies.py | 7 +++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/faraday-server.py b/faraday-server.py index e86ec390352..75da83f0d98 100755 --- a/faraday-server.py +++ b/faraday-server.py @@ -25,11 +25,14 @@ def setup_environment(check_deps=False): if check_deps: # Check dependencies - installed_deps, missing_deps = dependencies.check_dependencies( + installed_deps, missing_deps, conflict_deps = dependencies.check_dependencies( requirements_file=server.config.REQUIREMENTS_FILE) logger.info("Checking dependencies...") + if conflict_deps: + logger.info("Some dependencies are old. Update them with \"pip install -rrequirements_server.txt -U\"") + if missing_deps: install_deps = query_yes_no("Do you want to install them?", default="no") diff --git a/faraday.py b/faraday.py index f62773ea9d4..72006b4e033 100755 --- a/faraday.py +++ b/faraday.py @@ -205,10 +205,13 @@ def check_dependencies_or_exit(): """ - installed_deps, missing_deps = dependencies.check_dependencies(requirements_file=FARADAY_REQUIREMENTS_FILE) + installed_deps, missing_deps, conflict_deps = dependencies.check_dependencies(requirements_file=FARADAY_REQUIREMENTS_FILE) logger.info("Checking dependencies...") + if conflict_deps: + logger.info("Some dependencies are old. Update them with \"pip install -rrequirements_server.txt -U\"") + if missing_deps: install_deps = query_yes_no("Do you want to install them?", default="no") diff --git a/utils/dependencies.py b/utils/dependencies.py index 6893aee198a..a974752a087 100644 --- a/utils/dependencies.py +++ b/utils/dependencies.py @@ -16,15 +16,18 @@ def check_dependencies(requirements_file='requirements.txt'): installed = [] missing = [] + conflict = [] for package in requirements: try: pkg_resources.working_set.resolve([package]) installed += [package] - except (pkg_resources.DistributionNotFound, pkg_resources.VersionConflict): + except pkg_resources.DistributionNotFound: missing += [package.key] + except pkg_resources.VersionConflict: + conflict += [package.key] - return installed, missing + return installed, missing, conflict def install_packages(packages): From 2ce420ae24cea7fe27c9f56dc091cbda7578b6ac Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 24 Nov 2017 17:13:17 -0300 Subject: [PATCH 0563/1506] Add auth with cookies in persistence/server/server.py --- config/configuration.py | 8 +++++ persistence/server/server.py | 66 ++++++++++++++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/config/configuration.py b/config/configuration.py index 3587985f10b..0469c29240c 100644 --- a/config/configuration.py +++ b/config/configuration.py @@ -157,6 +157,8 @@ def _getConfig(self): self._plugin_settings = json.loads(self._getValue(tree, CONST_PLUGIN_SETTINGS, default = "{}")) self._osint = json.loads(self._getValue(tree, CONST_OSINT, default = "{\"host\": \"shodan.io\",\"icon\": \"shodan\",\"label\": \"Shodan\", \"prefix\": \"/search?query=\", \"suffix\": \"\", \"use_external_icon\": false}")) + self._session_cookies = {} + self._updates_uri = self._getValue(tree, CONST_UPDATEURI, default = "https://www.faradaysec.com/scripts/updates.php") self._tkts_uri = self._getValue(tree, CONST_TKTURI,default = "https://www.faradaysec.com/scripts/listener.php") self._tkt_api_params = self._getValue(tree, CONST_TKTAPIPARAMS,default ="{}") @@ -253,6 +255,9 @@ def getCouchReplics(self): def getCouchIsReplicated(self): return self._couch_is_replicated + def getDBSessionCookies(self): + return self._session_cookies + def getRepoPassword(self): return self._repo_password @@ -371,6 +376,9 @@ def setPersistencePath(self, val): def setPerspectiveView(self, val): self._perspective_view = val + def setDBSessionCookies(self, val=None): + self._session_cookies = val + def setRepoPassword(self, val): self._repo_password = val diff --git a/persistence/server/server.py b/persistence/server/server.py index 685bc03726a..af9015774b1 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -39,11 +39,24 @@ # NOTE: Change is you want to use this module by itself. # If FARADAY_UP is False, SERVER_URL must be a valid faraday server url FARADAY_UP = True -SERVER_URL = "http://127.0.0.1:5984" +SERVER_URL = "http://127.0.0.1:5985" +AUTH_USER = "" +AUTH_PASS = "" + def _conf(): + from config.configuration import getInstanceConfiguration CONF = getInstanceConfiguration() + + # If you are running this libs outside of Faraday, cookies are not setted. + # you need get a valid cookie auth and set that. + # Fplugin run in other instance, so this dont generate any trouble. + if not CONF.getDBSessionCookies(): + server_url = CONF.getCouchURI() if FARADAY_UP else SERVER_URL + cookie = login_user(server_url, AUTH_USER, AUTH_PASS) + CONF.setDBSessionCookies(cookie) + return CONF def _get_base_server_url(): @@ -106,6 +119,17 @@ def _create_server_db_url(workspace_name): db_url = '{0}/ws/{1}'.format(server_api_url, workspace_name) return db_url +def _add_session_cookies(func): + """A decorator which wrapps a function dealing with I/O with the server and + adds authentication to the parameters. + """ + def wrapper(*args, **kwargs): + kwargs['cookies'] = _conf().getDBSessionCookies() + response = func(*args, **kwargs) + return response + return wrapper if FARADAY_UP else func + +@_add_session_cookies def _unsafe_io_with_server(server_io_function, server_expected_response, server_url, **payload): """A wrapper for functions which deals with I/O to or from the server. @@ -271,6 +295,7 @@ def _delete_from_couch(workspace_name, faraday_object_id): return _delete(delete_url) # XXX: COUCH IT! +@_add_session_cookies def _couch_changes(workspace_name, **params): return CouchChangesStream(workspace_name, _create_couch_db_url(workspace_name), @@ -1473,10 +1498,35 @@ def server_info(): except: return None +def login_user(uri, uname, upass): + auth = {"email": uname, "password": upass} + try: + resp = requests.post(uri + "/_api/login", json=auth, timeout=1) + if resp.status_code == 400: + return None + else: + return resp.cookies + except requests.adapters.ConnectionError: + return None + except requests.adapters.ReadTimeout: + return None + +def is_authenticated(uri, cookies): + try: + resp = requests.get(uri + "/_api/session", cookies=cookies, timeout=1) + if resp.status_code != 403: + user_info = resp.json() + return bool(user_info.get('name', {})) + else: + return False + except requests.adapters.ConnectionError: + return False + except requests.adapters.ReadTimeout: + return False + def check_faraday_version(): """Raise RuntimeError if client and server aren't running the same version""" info = server_info() - #print "INFO", infok faraday_directory = os.path.dirname(os.path.realpath('faraday.py')) @@ -1498,3 +1548,15 @@ def test_server_url(url_to_test): except: test_okey = False return test_okey + +def get_user_info(): + try: + resp = requests.get(_get_base_server_url() + "/_api/session", cookies=_conf().getDBSessionCookies(), timeout=1) + if resp.status_code != 403: + return resp.json() + else: + return False + except requests.adapters.ConnectionError: + return False + except requests.adapters.ReadTimeout: + return False \ No newline at end of file From 00b6837d0d6f09a2e6365d9ea2bd00d7b84caa64 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 24 Nov 2017 18:11:38 -0300 Subject: [PATCH 0564/1506] Update js code to use send_data instead of put --- server/www/scripts/commons/providers/server.js | 4 ++-- test_cases/test_api_workspace.py | 0 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 test_cases/test_api_workspace.py diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index 1bebfb0498e..6ba36efbec9 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -484,12 +484,12 @@ angular.module("faradayApp") ServerAPI.createWorkspace = function(wsName, data) { var dbUrl = createDbUrl(wsName); - return put(dbUrl, data, false) + return send_data(dbUrl, data, true, "POST"); } ServerAPI.updateWorkspace = function(workspace) { var putUrl = createDbUrl(workspace.name); - return put(putUrl, workspace, true) + return send_data(dbUrl, workspace, true, "PUT"); } ServerAPI.deleteWorkspace = function(wsName) { diff --git a/test_cases/test_api_workspace.py b/test_cases/test_api_workspace.py new file mode 100644 index 00000000000..e69de29bb2d From 44fd7b82dfaef8b68ffd5223c58cc0289d6914e5 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 24 Nov 2017 21:07:46 -0300 Subject: [PATCH 0565/1506] Add auth to white version cli and GTK --- config/configuration.py | 27 +++--- faraday.py | 86 +++++++++++++++++-- gui/gtk/application.py | 87 +++++++++++++------ gui/gtk/dialogs.py | 159 +++++++++++++++++++++++++++++++++-- model/application.py | 20 +++++ persistence/server/server.py | 4 +- plugins/fplugin_utils.py | 2 +- updates/updater.py | 8 +- 8 files changed, 337 insertions(+), 56 deletions(-) diff --git a/config/configuration.py b/config/configuration.py index 0469c29240c..91c678a7223 100644 --- a/config/configuration.py +++ b/config/configuration.py @@ -39,7 +39,7 @@ CONST_PERSISTENCE_PATH = "persistence_path" CONST_PERSPECTIVE_VIEW = "perspective_view" CONST_REPO_PASSWORD = "repo_password" -CONST_COUCH_URI = "couch_uri" +CONST_SERVER_URI = "couch_uri" CONST_COUCH_REPLICS = "couch_replics" CONST_COUCH_ISREPLICATED = "couch_is_replicated" CONST_REPO_URL = "repo_url" @@ -145,7 +145,7 @@ def _getConfig(self): self._persistence_path = self._getValue(tree, CONST_PERSISTENCE_PATH) self._perspective_view = self._getValue(tree, CONST_PERSISTENCE_PATH) self._repo_password = self._getValue(tree, CONST_REPO_PASSWORD) - self._couch_uri = self._getValue(tree, CONST_COUCH_URI, default = "") + self._server_uri = self._getValue(tree, CONST_SERVER_URI, default = "") self._couch_replics = self._getValue(tree, CONST_COUCH_REPLICS, default = "") self._couch_is_replicated = bool(self._getValue(tree, CONST_COUCH_ISREPLICATED, default = False)) self._repo_url = self._getValue(tree, CONST_REPO_URL) @@ -157,6 +157,7 @@ def _getConfig(self): self._plugin_settings = json.loads(self._getValue(tree, CONST_PLUGIN_SETTINGS, default = "{}")) self._osint = json.loads(self._getValue(tree, CONST_OSINT, default = "{\"host\": \"shodan.io\",\"icon\": \"shodan\",\"label\": \"Shodan\", \"prefix\": \"/search?query=\", \"suffix\": \"\", \"use_external_icon\": false}")) + self._db_user = "" self._session_cookies = {} self._updates_uri = self._getValue(tree, CONST_UPDATEURI, default = "https://www.faradaysec.com/scripts/updates.php") @@ -243,11 +244,11 @@ def getPersistencePath(self): def getPerspectiveView(self): return self._perspective_view - def getCouchURI(self): - if self._couch_uri and self._couch_uri.endswith('/'): - return self._couch_uri[:-1] + def getServerURI(self): + if self._server_uri and self._server_uri.endswith('/'): + return self._server_uri[:-1] else: - return self._couch_uri + return self._server_uri def getCouchReplics(self): return self._couch_replics @@ -258,6 +259,9 @@ def getCouchIsReplicated(self): def getDBSessionCookies(self): return self._session_cookies + def getDBUser(self): + return self._db_user + def getRepoPassword(self): return self._repo_password @@ -379,6 +383,9 @@ def setPerspectiveView(self, val): def setDBSessionCookies(self, val=None): self._session_cookies = val + def setDBUser(self, val=None): + self._db_user = val + def setRepoPassword(self, val): self._repo_password = val @@ -398,7 +405,7 @@ def setVersion(self, val): self._version = val def setCouchUri(self, uri): - self._couch_uri = uri + self._server_uri = uri def setCouchIsReplicated(self, is_it): self._couch_is_replicated = is_it @@ -550,9 +557,9 @@ def saveConfig(self, xml_file="~/.faraday/config/user.xml"): LAST_WORKSPACE.text = self.getLastWorkspace() ROOT.append(LAST_WORKSPACE) - COUCH_URI = Element(CONST_COUCH_URI) - COUCH_URI.text = self.getCouchURI() - ROOT.append(COUCH_URI) + SERVER_URI = Element(CONST_SERVER_URI) + SERVER_URI.text = self.getServerURI() + ROOT.append(SERVER_URI) COUCH_IS_REPLICATED = Element(CONST_COUCH_ISREPLICATED) COUCH_IS_REPLICATED.text = str(self.getCouchIsReplicated()) diff --git a/faraday.py b/faraday.py index f62773ea9d4..6cb0cc4f624 100755 --- a/faraday.py +++ b/faraday.py @@ -39,6 +39,7 @@ from utils.user_input import query_yes_no from persistence.server import server +from persistence.server.server import is_authenticated, login_user, get_user_info USER_HOME = os.path.expanduser(CONST_USER_HOME) FARADAY_BASE = os.path.dirname(os.path.realpath(__file__)) @@ -130,6 +131,12 @@ def getParserArgs(): default=False, help="Disable the application exception hook that allows to send error reports to developers.") + parser.add_argument('--login', + action="store_true", + dest="login", + default=False, + help="Enable prompt for authentication Database credentials") + parser.add_argument('--dev-mode', action="store_true", dest="dev_mode", @@ -186,7 +193,15 @@ def getParserArgs(): default=False, help="Enables debug mode. Default = disabled") - parser.add_argument('--nodeps', action='store_true', help='Skip dependency check') + parser.add_argument('--creds-file', + action="store", + dest="creds_file", + default=None, + help="File containing user's credentials to be used in cli mode") + + parser.add_argument('--nodeps', + action="store_true", + help='Skip dependency check') f = open(FARADAY_VERSION_FILE) f_version = f.read().strip() @@ -301,9 +316,9 @@ def startFaraday(): start = main_app.start from colorama import Fore, Back, Style import string - couchURL = getInstanceConfiguration().getCouchURI() - if couchURL: - url = "%s/_ui" % couchURL + serverURL = getInstanceConfiguration().getServerURI() + if serverURL: + url = "%s/_ui" % serverURL print(Fore.WHITE + Style.BRIGHT + "\n*" + string.center("faraday ui is ready", 53 - 6)) print( Fore.WHITE + Style.BRIGHT + "Make sure you got couchdb up and running.\nIf couchdb is up, point your browser to: \n[%s]" % url) @@ -454,6 +469,12 @@ def update(): """ if args.update: + CONF = getInstanceConfiguration() + + if not is_authenticated(CONF.getServerURI(), CONF.getDBSessionCookies()): + logger.warning("Credentials needed to update.") + doLoginLoop() + from updates.updater import Updater Updater().doUpdates() logger.info("Update process finished with no errors") @@ -485,7 +506,7 @@ def checkUpdates(): def checkCouchUrl(): import requests try: - requests.get(getInstanceConfiguration().getCouchURI(), timeout=5) + requests.get(getInstanceConfiguration().getServerURI(), timeout=5) except requests.exceptions.SSLError: print """ SSL certificate validation failed. @@ -527,6 +548,46 @@ def check_faraday_version(): sys.exit(2) +def doLoginLoop(): + """ Sets the username and passwords from the command line. + If --login flag is set then username and password is set """ + + import getpass + + print("""\nTo login please provide your valid DB Credentials.\n +You have 3 attempts.""") + + try: + + CONF = getInstanceConfiguration() + + for attempt in range(1, 4): + + username = raw_input("Username: ") + password = getpass.getpass('Password: ') + session_cookie = login_user(CONF.getServerURI(), username, password) + if session_cookie: + CONF.setDBUser(username) + CONF.setDBSessionCookies(session_cookie) + + user_info = get_user_info() + + if user_info is None or 'username' not in user_info or 'roles' not in user_info or 'client' in user_info['roles']: + print "You can't login as a client. You have %s attempt(s) left." % (3 - attempt) + continue + + logger.info('Login successful') + + break + print('Login failed, please try again. You have %d more attempts' % (3 - attempt)) + + else: + logger.fatal('Invalid credentials, 3 attempts failed. Quitting Faraday...') + sys.exit(-1) + except KeyboardInterrupt: + sys.exit(0) + + def main(): """Main. @@ -554,6 +615,21 @@ def main(): checkCouchUrl() checkVersion() + if args.login: + CONF = getInstanceConfiguration() + + if not CONF.getServerURI(): + couchURI = raw_input("Enter the Faraday server [http://127.0.0.1:5985]: ") or "http://127.0.0.1:5985" + + if couchURI: + CONF.setCouchUri(couchURI) + checkCouchUrl() + else: + logger.fatal('Please configure couchdb server to authenticate (--login)') + sys.exit(-1) + + doLoginLoop() + check_faraday_version() update() diff --git a/gui/gtk/application.py b/gui/gtk/application.py index f6c30a9d82c..89459a445e7 100644 --- a/gui/gtk/application.py +++ b/gui/gtk/application.py @@ -48,7 +48,8 @@ from config.configuration import getInstanceConfiguration from utils.logs import getLogger from appwindow import AppWindow -from persistence.server.server import check_faraday_version + +from persistence.server.server import is_authenticated, check_faraday_version, Unauthorized from server import ServerIO from dialogs import PreferenceWindowDialog @@ -63,6 +64,7 @@ from dialogs import ForcePreferenceWindowDialog from dialogs import errorDialog from dialogs import ImportantErrorDialog +from dialogs import LoginDialog, ForceLoginDialog from dialogs import FaradayPluginsDialog from mainwidgets import Sidebar @@ -161,27 +163,37 @@ def createWorkspace(self, name, description=""): return creation_ok def remove_workspace(self, button, ws_name): - """Removes a workspace. If the workspace to be deleted is the one - selected, it moves you to the one above it on the list. If there - aren't more workspaces left, you will be forced to create one. - The clears and refreshes - sidebar""" - model.api.log("Removing Workspace: %s" % ws_name) - server_response = self.getWorkspaceManager().removeWorkspace(ws_name) - if server_response: + """Removes a workspace. If the workspace deleted is currently active, + a signal will be incoming vis postUpdates() and force the user to + select another workspace.""" + try: + model.api.log("Removing Workspace: %s" % ws_name) + self.getWorkspaceManager().removeWorkspace(ws_name) self.ws_sidebar.clear_sidebar() self.ws_sidebar.refresh_sidebar() - available_workspaces = self.serverIO.get_workspaces_names() - if available_workspaces: - self.select_last_workspace_in_list(available_workspaces) - else: - self.handle_no_active_workspace() + except restkit.errors.Unauthorized: + notification_center.showDialog( + "You're not authorized to delete this workspace.\n" + "Make sure you're an admin and that you're logged in.", + "ERROR") + except Exception: + traceback_str = traceback.format_exc() + model.api.log("An exception was captured while deleting " + "workspace %s\n%s" % (ws_name, traceback_str), + "ERROR") + + available_workspaces = self.serverIO.get_workspaces_names() + if available_workspaces: + self.select_last_workspace_in_list(available_workspaces) + else: + self.handle_no_active_workspace() def lost_db_connection(self, explanatory_message=None, handle_connection_lost=None, connect_to_a_different_couch=None): """Creates a simple dialog with an error message to inform the user some kind of problem has happened and the connection was lost. + Returns whether the login dialog should be shown or not """ # NOTE: if we start faraday without CouchDB, both the signal coming @@ -226,10 +238,22 @@ def do_nothing_on_key_stroke(event, key): cancel_button = dialog.add_button("Exit Faraday", 0) cancel_button.connect("clicked", self.on_quit) + if hasattr(self, 'force_new_workspace_dialog'): + # The dialog to create a new workspace is open. Lets close it + new_workspace_dialog = getattr(self, 'force_new_workspace_dialog') + + new_workspace_dialog.destroy() + setattr(self, 'force_new_workspace_dialog', None) + response = dialog.run() if response == Gtk.ResponseType.DELETE_EVENT: GObject.idle_add(self.exit_faraday_without_confirm) + elif response in [0, 42, 43]: + return False + + return True + def handle_no_active_workspace(self): """If there's been a problem opening a workspace or for some reason we suddenly find our selves without one, force the user @@ -264,6 +288,7 @@ def change_flag(widget): self.workspace_manager, self.ws_sidebar, self.exit_faraday) + self.force_new_workspace_dialog = dialog dialog.connect("destroy", change_flag) dialog.show_all() @@ -346,7 +371,7 @@ def connect_to_couch(self, server_uri, parent=None): def handle_connection_lost(self, button=None, dialog=None): """Tries to connect to Couch using the same URI""" - couch_uri = CONF.getCouchURI() + couch_uri = CONF.getServerURI() if self.connect_to_couch(couch_uri, parent=dialog): reconnected = True if dialog is not None: @@ -737,10 +762,20 @@ def do_activate(self): model.guiapi.notification_center.registerWidget(self.window) if self.serverIO.server_info() is None: - self.lost_db_connection( + + should_login = self.lost_db_connection( handle_connection_lost=self.handle_connection_lost, connect_to_a_different_couch=self.force_change_couch_url) + if not should_login: + return + + if not is_authenticated(CONF.getServerURI(), CONF.getDBSessionCookies()): + loginDialog = ForceLoginDialog(self.window, + self.exit_faraday_without_confirm) + loginDialog.run(3, CONF.getServerURI(), self.window) + self.reload_workspaces() + workspace_argument_set = self.open_workspace_from_args() if not workspace_argument_set: self.open_last_workspace() @@ -842,9 +877,11 @@ def select_plugin(): def on_file_selected(plugin_id, report): """Send the plugin_id and the report file to be processed""" - self.report_manager.sendReportToPluginById(plugin_id, report) - - plugin_response, plugin_id = select_plugin() + try: + self.report_manager.sendReportToPluginById(plugin_id, report) + except Unauthorized: + self.show_normal_error("You are not authorized to write data " + "to this workspace.") while plugin_response == Gtk.ResponseType.ACCEPT and plugin_id is None: # force user to select a plugin if he did not do it @@ -884,15 +921,9 @@ def on_preferences(self, action=None, param=None): preference_window.show_all() def on_click_go_to_web_ui_button(self, action=None, param=None): - """Opens the dashboard of the current workspace on a new tab of - the user's default browser - """ - couch_url = CONF.getCouchURI() - ws_name = self.workspace_manager.getActiveWorkspace() - if not ws_name: - ws_url = couch_url + "/_ui/" - else: - ws_url = couch_url + "/_ui/#/dashboard/ws/" + ws_name.name + """Opens the login on a new tab of the user's default browser""" + + ws_url = CONF.getServerURI() + "/_ui/#/login" webbrowser.open(ws_url, new=2) def on_help_dispatch(self, action, param=None): diff --git a/gui/gtk/dialogs.py b/gui/gtk/dialogs.py index 96f96d047ed..a59c60e697d 100644 --- a/gui/gtk/dialogs.py +++ b/gui/gtk/dialogs.py @@ -14,6 +14,7 @@ from gi.repository import Gtk, GdkPixbuf, Gdk from config.configuration import getInstanceConfiguration +from persistence.server.server import is_authenticated, login_user, get_user_info, test_server_url from model import guiapi from decorators import scrollable @@ -49,7 +50,7 @@ def __init__(self, reload_ws_callback, connect_to_couch, parent): ip_label.set_text("Faraday Server IP or URL") main_box.pack_start(ip_label, True, False, 10) - couch_uri = CONF.getCouchURI() + couch_uri = CONF.getServerURI() self.ip_entry = Gtk.Entry() text = couch_uri if couch_uri else "http://127.0.0.1:5050" self.ip_entry.set_text(text) @@ -74,9 +75,34 @@ def on_click_ok(self, button=None): repourl (Couch IP) from self.ip_entry and connect to it if possible. """ repourl = self.ip_entry.get_text() - if self.connectCouchCallback(repourl, parent=self): # success! + + if not test_server_url(repourl): + errorDialog(self, "Could not connect to Faraday Server.", + ("Are you sure it is running and that the URL is correct?")) + return False + + + credentials_ok = self.credentialsOK(repourl) + couch_connection_ok = self.connectCouchCallback(repourl, parent=self) + if credentials_ok and couch_connection_ok: self.destroy() + def credentialsOK(self, repourl): + """Pops up a dialog (if necessary) to set up Faraday + credentials. Dialog is a LoginDialog which emits a signal marked + by 42 when the user clicks its button. The run method returns 42 + on that click. Function will listen for that 42 at most three times. + It's a boolean function, return True if auth ok, False if not. + Number 42 was chosen for obvious reasons :) """ + + if is_authenticated(repourl, CONF.getDBSessionCookies()): + return True + + # if that didn't work... + loginDialog = LoginDialog(self) + return loginDialog.run(3, repourl, self) + + def on_click_cancel(self, button=None): self.destroy() @@ -109,6 +135,124 @@ def on_click_cancel(self, button=None): self.exit_faraday(parent=self) +class LoginDialog(Gtk.Dialog): + """A simple login dialog with a user and password""" + def __init__(self, parent): + Gtk.Dialog.__init__(self, + flags=Gtk.DialogFlags.MODAL, + buttons=("OK", Gtk.ResponseType.OK, + "Cancel", Gtk.ResponseType.CANCEL)) + + self.set_default_response(Gtk.ResponseType.OK) + + self.set_keep_above(True) + + self.set_transient_for(parent) + content_area = self.get_content_area() + + instructions = Gtk.Label.new("Credentials needed to proceed." + "You've got 3 tries.") + instructions.set_line_wrap(True) + instructions.set_max_width_chars(38) + content_area.pack_start(instructions, True, True, 10) + + userBox = Gtk.Box() + user_label = Gtk.Label() + user_label.set_text("User:") + self.user_entry = Gtk.Entry() + self.user_entry.set_width_chars(24) + self.user_entry.set_activates_default(True) + userBox.pack_start(user_label, True, True, 3) + userBox.pack_start(self.user_entry, False, False, 5) + content_area.pack_start(userBox, True, True, 10) + + passwordBox = Gtk.Box() + password_label = Gtk.Label() + password_label.set_text("Password:") + self.password_entry = Gtk.Entry() + self.password_entry.set_visibility(False) + self.password_entry.set_width_chars(24) + self.password_entry.set_activates_default(True) + passwordBox.pack_start(password_label, True, True, 3) + passwordBox.pack_start(self.password_entry, False, False, 5) + content_area.pack_start(passwordBox, True, True, 10) + + self.show_all() + + def getUser(self): + if self.user_entry.get_text() is not None: + res = self.user_entry.get_text() + else: + res = "" + return res + + def getPassword(self): + if self.password_entry.get_text() is not None: + res = self.password_entry.get_text() + else: + res = "" + return res + + def run(self, attempts, url, parent): + for attempt in range(attempts): + run = Gtk.Dialog.run(self) + + if run == Gtk.ResponseType.OK: + newUser = self.getUser() + newPass = self.getPassword() + session_cookie = login_user(url, newUser, newPass) + if not session_cookie: + if attempt != attempts-1: + errorDialog(self, ("Invalid credentials!. You " + "have " + + str(attempts-1-attempt) + + " attempt(s) left.")) + else: + + CONF.setDBUser(newUser) + CONF.setDBSessionCookies(session_cookie) + + user_info = get_user_info() + + if user_info is None or 'userCtx' not in user_info or 'roles' not in user_info['userCtx'] or 'client' in user_info['userCtx']['roles']: + errorDialog(self, ("You can't login as a client. You have " + + str(attempts - 1 - attempt) + + " attempt(s) left.")) + continue + + self.destroy() + + return True + + if run == Gtk.ResponseType.CANCEL or run == -4: + # run returns -4 when escape key pressed + self.exit() + return False + else: + errorDialog(self, ("Invalid credentials after " + + str(attempts) + " tries. " + + "Check your credentials and try again.")) + self.exit() + return False + + def exit(self): + self.destroy() + + +class ForceLoginDialog(LoginDialog): + """A simple login dialog with a user and password""" + def __init__(self, parent, exit_faraday_callback): + LoginDialog.__init__(self, parent) + + self.set_deletable(False) + + self.exit_faraday = exit_faraday_callback + + def exit(self, button=None): + """Override destroy to make it exit Faraday.""" + self.exit_faraday() + + class NewWorkspaceDialog(Gtk.Window): """Sets up the New Workspace Dialog, where the user can set a name, a description and a type for a new workspace. Also checks that the @@ -591,7 +735,7 @@ def __init__(self, parent, active_ws_name, host): host_info = self.model[0] host_id = self.model[0][0] - couch_url = CONF.getCouchURI() + couch_url = CONF.getServerURI() base_url = couch_url + "/_ui/#/host/ws/" self.edit_url = base_url + active_ws_name + "/hid/" + host_id @@ -722,8 +866,8 @@ def create_model(self, host): several columns HOST - ------------> SERVICE1 - ------------> SERVICE2 + ------------> SERVICE1 + ------------> SERVICE2 And so on and so on, like Zizek says. """ @@ -1518,7 +1662,10 @@ def create_explanation_message(self): message = Gtk.Label() message.set_text("Your last workspace is not accessible. \n" "You must select one of the below workspaces to " - "continue using Faraday.") + "continue using Faraday. \n" + "If your problem persists, double check you are " + "logged in. You may want to start faraday with the " + "--login flag.") return message @scrollable(height=200) diff --git a/model/application.py b/model/application.py index f8a6c872cae..74aea8bfe1f 100644 --- a/model/application.py +++ b/model/application.py @@ -8,6 +8,7 @@ import os import sys import signal +import json import threading import requests @@ -17,6 +18,7 @@ from model.controller import ModelController from managers.workspace_manager import WorkspaceManager from plugins.controller import PluginController +from persistence.server.server import login_user import model.api import model.guiapi @@ -78,6 +80,24 @@ def __init__(self, args): self.args = args + logger = getLogger(self) + if args.creds_file: + try: + with open(args.creds_file, 'r') as fp: + creds = json.loads(fp.read()) + username = creds.get('username') + password = creds.get('password') + session_cookie = login_user(CONF.getServerURI(), + username, password) + if session_cookie: + logger.info('Login successful') + CONF.setDBUser(username) + CONF.setDBSessionCookies(session_cookie) + else: + logger.error('Login failed') + except (IOError, ValueError): + logger.error("Credentials file couldn't be loaded") + self._mappers_manager = MapperManager() self._model_controller = ModelController(self._mappers_manager) diff --git a/persistence/server/server.py b/persistence/server/server.py index af9015774b1..be703884bc1 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -53,7 +53,7 @@ def _conf(): # you need get a valid cookie auth and set that. # Fplugin run in other instance, so this dont generate any trouble. if not CONF.getDBSessionCookies(): - server_url = CONF.getCouchURI() if FARADAY_UP else SERVER_URL + server_url = CONF.getServerURI() if FARADAY_UP else SERVER_URL cookie = login_user(server_url, AUTH_USER, AUTH_PASS) CONF.setDBSessionCookies(cookie) @@ -61,7 +61,7 @@ def _conf(): def _get_base_server_url(): if FARADAY_UP: - server_url = _conf().getCouchURI() + server_url = _conf().getServerURI() else: server_url = SERVER_URL return server_url[:-1] if server_url[-1] == "/" else server_url diff --git a/plugins/fplugin_utils.py b/plugins/fplugin_utils.py index ea6eb3bb109..1427d435f06 100644 --- a/plugins/fplugin_utils.py +++ b/plugins/fplugin_utils.py @@ -75,6 +75,6 @@ def build_faraday_plugin_command(plugin, workspace_name, absolute_path=False): return '{path}fplugin {command} -u {url} -w {workspace} '.format( path='"%s"' % path if absolute_path else '', command=plugin, - url=CONF.getCouchURI(), + url=CONF.getServerURI(), workspace=workspace_name ) diff --git a/updates/updater.py b/updates/updater.py index f76072117a1..4fcda0e4950 100644 --- a/updates/updater.py +++ b/updates/updater.py @@ -50,7 +50,7 @@ class Update(object): class CouchViews(Update): def run(self): - source_server = CONF.getCouchURI() + source_server = CONF.getServerURI() if not source_server: logger.info("""No DB configuration found. To upgrade your DB please configure a valid CouchDB URI in: @@ -84,7 +84,7 @@ def update_db(self, db_name): logger.info('Database [%s] is a backup, ignoring' % db_name) return - source_server = CONF.getCouchURI() + source_server = CONF.getServerURI() # Levanto los servidores db_source = couchdbkit.Database("/".join((source_server, db_name))) if db_source.doc_exist(db_name): @@ -102,7 +102,7 @@ def update_db(self, db_name): # Crear documento 'workspace' logger.info('Creating workspace document') - + create_workspace(db_name, 'Migrated Workspace', int(time.time() * 1000), @@ -145,7 +145,7 @@ def update_db(self, db_name): def run(self): - source_server = CONF.getCouchURI() + source_server = CONF.getServerURI() if not source_server: logger.info("""No DB configuration found. To upgrade your DB please configure a valid CouchDB URI in: From d63a035d9b3a1acab61893cda697f42574d0b695 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 27 Nov 2017 14:32:19 -0300 Subject: [PATCH 0566/1506] Add support to update commands and execute process report from command line * Disable infinite loop when report processing was called from command line. * Commands are not updated when the import finished. --- manage.py | 8 ++++-- managers/mapper_manager.py | 5 ++-- managers/reports_managers.py | 13 +++++---- model/commands_history.py | 3 +-- model/controller.py | 4 +-- persistence/server/models.py | 2 +- persistence/server/server.py | 30 ++++++++++++--------- persistence/server/server_io_exceptions.py | 2 +- plugins/controller.py | 2 +- server/commands/reports.py | 31 +++++++++++++++++----- 10 files changed, 65 insertions(+), 35 deletions(-) diff --git a/manage.py b/manage.py index 24ccddfd0a6..4313250ddd3 100755 --- a/manage.py +++ b/manage.py @@ -11,6 +11,7 @@ from server.commands.reports import import_external_reports from server.web import app +from utils.logs import setUpLogger @click.group() @@ -18,10 +19,13 @@ def cli(): pass @click.command() -def process_reports(): +@click.option('--debug/--no-debug', default=False) +@click.option('--workspace', default=None) +def process_reports(debug, workspace): + setUpLogger(debug) with app.app_context(): try: - import_external_reports() + import_external_reports(workspace) except OperationalError as ex: print('{0}'.format(ex)) print('Please verify database is running or configuration on server.ini!') diff --git a/managers/mapper_manager.py b/managers/mapper_manager.py index 2f932d3b11b..06641c08b08 100644 --- a/managers/mapper_manager.py +++ b/managers/mapper_manager.py @@ -23,8 +23,9 @@ def createMappers(self, workpace_name): self.workspace_name = workpace_name def save(self, obj): - if create_object(self.workspace_name, obj.class_signature, obj): - return True + saved_raw_obj = create_object(self.workspace_name, obj.class_signature, obj) + if '_id' in saved_raw_obj: + return saved_raw_obj['_id'] return False def update(self, obj): diff --git a/managers/reports_managers.py b/managers/reports_managers.py index 9c80b462f30..9140bd25c58 100755 --- a/managers/reports_managers.py +++ b/managers/reports_managers.py @@ -9,10 +9,9 @@ import os import re import time +import logging import traceback from multiprocessing import Process - - from utils.logs import getLogger try: @@ -24,6 +23,7 @@ from config.configuration import getInstanceConfiguration CONF = getInstanceConfiguration() +logger = logging.getLogger(__name__) class ReportProcessor(): @@ -64,10 +64,11 @@ def onlinePlugin(self, cmd): class ReportManager(Process): - def __init__(self, timer, ws_name, plugin_controller): + def __init__(self, timer, ws_name, plugin_controller, polling=True): Process.__init__(self) + self.polling = polling self.ws_name = ws_name - self.daemon = True + self.daemon = False self.timer = timer self._stop = False self._report_path = os.path.join(CONF.getReportPath(), ws_name) @@ -100,6 +101,8 @@ def run(self): if tmp_timer >= self.timer: try: self.syncReports() + if not self.polling: + break except Exception: getLogger(self).error( "An exception was captured while saving reports\n%s" @@ -138,7 +141,7 @@ def syncReports(self): # If plugin not is detected... move to unprocessed if self.processor.processReport(filename) is False: - + logger.info('Plugin not detected. Moving {0} to unprocessed'.format(filename)) os.rename( filename, os.path.join(self._report_upath, name)) diff --git a/model/commands_history.py b/model/commands_history.py index 268c5146d3d..742a42fbf39 100644 --- a/model/commands_history.py +++ b/model/commands_history.py @@ -46,7 +46,6 @@ class CommandRunInformation(object): class_signature = "CommandRunInformation" def __init__(self, **kwargs): - self._id = uuid.uuid4().hex self.type = self.__class__.__name__ self.user = get_user() self.ip = get_private_ip() @@ -63,7 +62,7 @@ def getID(self): return self._id def setID(self, id): - return self._id + self._id = id def toDict(self): return self.__dict__ diff --git a/model/controller.py b/model/controller.py index ae446979565..a031131ec31 100644 --- a/model/controller.py +++ b/model/controller.py @@ -175,7 +175,6 @@ def _registerObjectTypes(self): # This could be done in hosts module, but it seems easier to maintain # if we have all in one place inside the controller self._object_factory.register(models.Host) - self._object_factory.register(models.Interface) self._object_factory.register(models.Service) self._object_factory.register(models.Vuln) self._object_factory.register(models.VulnWeb) @@ -448,7 +447,8 @@ def addHostSYNC(self, host): def _save_new_object(self, new_object): res = self.mappers_manager.save(new_object) - if res: notifier.addObject(new_object) + if res: + notifier.addObject(new_object) return res def _handle_conflict(self, old_obj, new_obj): diff --git a/persistence/server/models.py b/persistence/server/models.py index 3033b8114ae..9d1f9e31066 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -418,7 +418,7 @@ def update_command(workspace_name, command): Return the server's json response as a dictionary. """ command_properties = get_command_properties(command) - return server.update_command(workspace_name, **command_properties) + return server.update_command(workspace_name, command.getID(), **command_properties) def create_object(workspace_name, object_signature, obj): """Given a workspace name, an object_signature as string and obj, a Faraday diff --git a/persistence/server/server.py b/persistence/server/server.py index be703884bc1..4f234e51cfa 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -89,11 +89,18 @@ def _create_server_get_url(workspace_name, object_name=None): def _create_server_post_url(workspace_name, obj_type): server_api_url = _create_server_api_url() - object_end_point_name = OBJECT_END_POINT_MAPPER[obj_type] + object_end_point_name = OBJECT_TYPE_END_POINT_MAPPER[obj_type] post_url = '{0}/ws/{1}/{2}/'.format(server_api_url, workspace_name, object_end_point_name) return post_url +def _create_server_put_url(workspace_name, obj_type, obj_id): + server_api_url = _create_server_api_url() + object_end_point_name = OBJECT_TYPE_END_POINT_MAPPER[obj_type] + pust_url = '{0}/ws/{1}/{2}/{3}/'.format(server_api_url, workspace_name, object_end_point_name, obj_id) + return pust_url + + def _create_server_delete_url(workspace_name, object_id): return _create_server_post_url(workspace_name, object_id) @@ -176,7 +183,7 @@ def _get(request_url, **params): request_url, params=params)) -def _put(post_url, update=False, expected_response=201, **params): +def _put(post_url, expected_response=201, **params): """Put to the post_url. If update is True, try to get the object revision first so as to update the object in Couch. You can customize the expected response (it should be 201, but Couchdbkit returns @@ -189,9 +196,6 @@ def _put(post_url, update=False, expected_response=201, **params): Return a dictionary with the response from couchdb, which looks like this: {u'id': u'61', u'ok': True, u'rev': u'1-967a00dff5e02add41819138abb3284d'} """ - if update: - last_rev = _get(post_url)['_rev'] - params['_rev'] = last_rev return _parse_json(_unsafe_io_with_server(requests.put, expected_response, post_url, @@ -274,16 +278,16 @@ def _save_to_couch(workspace_name, faraday_object_id, **params): return _post(post_url, update=False, **params) def _update_in_couch(workspace_name, faraday_object_id, **params): - post_url = _create_server_post_url(workspace_name, faraday_object_id) - return _put(post_url, update=True, **params) + post_url = _create_server_put_url(workspace_name, faraday_object_id) + return _put(post_url, **params) def _save_to_server(workspace_name, **params): post_url = _create_server_post_url(workspace_name, params['type']) - return _post(post_url, update=False, expected_response=200, **params) + return _post(post_url, update=False, expected_response=201, **params) def _update_in_server(workspace_name, faraday_object_id, **params): - post_url = _create_server_post_url(workspace_name, faraday_object_id) - return _put(post_url, update=True, expected_response=200, **params) + put_url = _create_server_put_url(workspace_name, params['type'], faraday_object_id) + return _put(put_url, expected_response=200, **params) def _save_db_to_server(db_name, **params): post_url = _create_server_db_url(db_name) @@ -1400,7 +1404,7 @@ def create_command(workspace_name, command, duration=None, hostname=None, workspace=workspace_name, type="CommandRunInformation") -def update_command(workspace_name, command, duration=None, hostname=None, +def update_command(workspace_name, command_id, command, duration=None, hostname=None, ip=None, itime=None, params=None, user=None): """Updates a command. @@ -1419,7 +1423,7 @@ def update_command(workspace_name, command, duration=None, hostname=None, A dictionary with the server's response. """ return _update_in_server(workspace_name, - id, + command_id, command=command, duration=duration, hostname=hostname, @@ -1501,7 +1505,7 @@ def server_info(): def login_user(uri, uname, upass): auth = {"email": uname, "password": upass} try: - resp = requests.post(uri + "/_api/login", json=auth, timeout=1) + resp = requests.post(uri + "/_api/login", json=auth) if resp.status_code == 400: return None else: diff --git a/persistence/server/server_io_exceptions.py b/persistence/server/server_io_exceptions.py index e88a8b337c0..6d78ec979b4 100644 --- a/persistence/server/server_io_exceptions.py +++ b/persistence/server/server_io_exceptions.py @@ -55,7 +55,7 @@ def __init__(self, answer): def __str__(self): return ("You're not authorized to make this request. " - "The answer from the server was {0}".format(self.answer)) + "The answer from the server was {0}. Plase check that your domain is the correct one.".format(self.answer)) class CouchDBException(Exception): def __init__(self): diff --git a/plugins/controller.py b/plugins/controller.py index 97a4499c716..2d4ef5f6c7f 100644 --- a/plugins/controller.py +++ b/plugins/controller.py @@ -246,7 +246,7 @@ def processReport(self, plugin, filepath, ws_name=None): 'command': 'Import %s:' % plugin, 'params': filepath}) self._mapper_manager.createMappers(ws_name) - self._mapper_manager.save(cmd_info) + cmd_info.setID(self._mapper_manager.save(cmd_info)) if plugin in self._plugins: self.processOutput(self._plugins[plugin], filepath, cmd_info.getID(), True) diff --git a/server/commands/reports.py b/server/commands/reports.py index bc297849d4c..07a32b8d877 100644 --- a/server/commands/reports.py +++ b/server/commands/reports.py @@ -1,9 +1,13 @@ import os import logging +from tqdm import tqdm + from managers.mapper_manager import MapperManager from managers.reports_managers import ReportManager, CONF - +from managers.workspace_manager import WorkspaceManager +from model.api import setUpAPIs +from model.controller import ModelController from plugins.controller import PluginController from plugins.manager import PluginManager @@ -12,25 +16,40 @@ logger = logging.getLogger(__name__) -def import_external_reports(): - processes = [] +def import_external_reports(workspace_name=None): + plugin_manager = PluginManager( os.path.join(CONF.getConfigPath(), "plugins")) mappers_manager = MapperManager() + controller = ModelController(mappers_manager) + workspace_manager = WorkspaceManager(mappers_manager) + setUpAPIs(controller, workspace_manager, hostname=None, port=None) plugin_controller = PluginController( 'PluginController', plugin_manager, mappers_manager ) - for workspace in Workspace.query.all(): + + if workspace_name: + query = Workspace.query.filter_by(name=workspace_name) + else: + query = Workspace.query + + process_workspaces(plugin_controller, query) + + +def process_workspaces(plugin_controller, query): + processes = [] + for workspace in query.all(): report_manager = ReportManager( 0.1, workspace.name, - plugin_controller + plugin_controller, + polling=False ) processes.append(report_manager) report_manager.start() - for thread in processes: + for thread in tqdm(processes): thread.join() From 8d76a4ec505e3bf2bb0ac1cc3c105aeb5fba793e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 27 Nov 2017 14:49:45 -0300 Subject: [PATCH 0567/1506] Add +OBJECT_TYPE_END_POINT_MAPPER and some logging on plugin controller --- persistence/server/server.py | 4 ++++ plugins/controller.py | 12 +++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/persistence/server/server.py b/persistence/server/server.py index 4f234e51cfa..32781f50cbf 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -42,6 +42,9 @@ SERVER_URL = "http://127.0.0.1:5985" AUTH_USER = "" AUTH_PASS = "" +OBJECT_TYPE_END_POINT_MAPPER = { + 'CommandRunInformation': 'commands', +} def _conf(): @@ -59,6 +62,7 @@ def _conf(): return CONF + def _get_base_server_url(): if FARADAY_UP: server_url = _conf().getServerURI() diff --git a/plugins/controller.py b/plugins/controller.py index 2d4ef5f6c7f..d9ccbf530ee 100644 --- a/plugins/controller.py +++ b/plugins/controller.py @@ -7,14 +7,13 @@ See the file 'doc/LICENSE' for the license information ''' - -import errno -from cStringIO import StringIO -import multiprocessing import os +import time import Queue import shlex -import time +import errno +import logging +import multiprocessing from plugins.plugin import PluginProcess import model.api @@ -26,6 +25,8 @@ CONST_FARADAY_HOME_PATH, CONST_FARADAY_ZSH_OUTPUT_PATH) +logger = logging.getLogger(__name__) + class PluginController(object): """ @@ -249,6 +250,7 @@ def processReport(self, plugin, filepath, ws_name=None): cmd_info.setID(self._mapper_manager.save(cmd_info)) if plugin in self._plugins: + logger.info('Processing report with plugin {0}'.format(plugin)) self.processOutput(self._plugins[plugin], filepath, cmd_info.getID(), True) cmd_info.duration = time.time() - cmd_info.itime self._mapper_manager.update(cmd_info) From 3b6e5f6cf77032844f0fa45df51410710aa0b8d0 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 27 Nov 2017 14:51:53 -0300 Subject: [PATCH 0568/1506] Add NotImplementedError on base class --- plugins/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/plugin.py b/plugins/plugin.py index f1253b14551..f5755a17f44 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -123,7 +123,7 @@ def parseOutputString(self, output): Using the output the plugin can create and add hosts, interfaces, services, etc. """ - pass + raise NotImplementedError('This method must be implemented.') def processCommandString(self, username, current_path, command_string): """ From 02c2e8119978deaa065d44f84e22451043aa3da1 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 27 Nov 2017 15:13:35 -0300 Subject: [PATCH 0569/1506] Add warning when no ws_name was detected --- model/common.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/model/common.py b/model/common.py index f6936a22a33..8fd827ddbed 100644 --- a/model/common.py +++ b/model/common.py @@ -7,8 +7,9 @@ import sys import traceback import threading -import SimpleXMLRPCServer +import logging import xmlrpclib +import SimpleXMLRPCServer try: import model.api as api @@ -17,6 +18,7 @@ from config.configuration import getInstanceConfiguration CONF = getInstanceConfiguration() +logger = logging.getLogger(__name__) # ------------------------------------------------------------------------------- @@ -94,6 +96,7 @@ def createModelObject(self, classname, object_name, workspace_name=None, parent_ """ if not workspace_name: workspace_name = CONF.getLastWorkspace() + logger.warn('No workspace selected. Using last workspace {0}'.format(workspace_name)) if classname in self._registered_objects: if object_name is not None: objargs['name'] = object_name From b3028a92025ee0726588e1ebd0a491873003ff42 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 27 Nov 2017 15:13:57 -0300 Subject: [PATCH 0570/1506] Set end_date when duration is sent --- server/api/modules/commandsrun.py | 3 ++- test_cases/test_api_commands.py | 21 ++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/server/api/modules/commandsrun.py b/server/api/modules/commandsrun.py index 850665698eb..7238a0d0743 100644 --- a/server/api/modules/commandsrun.py +++ b/server/api/modules/commandsrun.py @@ -44,9 +44,10 @@ def get_duration(self, obj): @post_load def post_load_set_end_date_with_duration(self, data): + # there is a potential bug when updating, the start_date can be changed. duration = data.pop('duration', None) if duration: - pass + data['end_date'] = data['start_date'] + datetime.timedelta(seconds=120) class Meta: model = Command diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index 7d6956b4331..70ecc9a3cf8 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -315,4 +315,23 @@ def test_create_command(self, test_client): } res = test_client.post(self.url(), data=raw_data) - assert res.status_code == 201 \ No newline at end of file + assert res.status_code == 201 + + + def test_update_command(self, test_client, session): + command = self.factory() + session.commit() + raw_data ={ + 'command': 'Import Nessus:', + 'duration': 120, + 'hostname': 'mandarina', + 'ip': '192.168.20.53', + 'itime': 1511387720.048548, + 'params': u'/home/lcubo/.faraday/report/airbnb/nessus_report_Remote.nessus', + 'user': 'lcubo' + } + + res = test_client.put(self.url(command, workspace=command.workspace), data=raw_data) + assert res.status_code == 200 + updated_command = self.model.query.get(command.id) + assert updated_command.end_date == datetime.datetime.fromtimestamp(1511387720.048548) + datetime.timedelta(seconds=120) \ No newline at end of file From 1dc91263551c5f6abac7e0a79fc4bbfda8f4ceb6 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 27 Nov 2017 19:37:35 -0300 Subject: [PATCH 0571/1506] Fix typo --- faraday-server.py | 2 +- faraday.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/faraday-server.py b/faraday-server.py index 75da83f0d98..0e685b7eb2f 100755 --- a/faraday-server.py +++ b/faraday-server.py @@ -31,7 +31,7 @@ def setup_environment(check_deps=False): logger.info("Checking dependencies...") if conflict_deps: - logger.info("Some dependencies are old. Update them with \"pip install -rrequirements_server.txt -U\"") + logger.info("Some dependencies are old. Update them with \"pip install -r requirements_server.txt -U\"") if missing_deps: diff --git a/faraday.py b/faraday.py index 72006b4e033..0a2c2da2d1e 100755 --- a/faraday.py +++ b/faraday.py @@ -210,7 +210,7 @@ def check_dependencies_or_exit(): logger.info("Checking dependencies...") if conflict_deps: - logger.info("Some dependencies are old. Update them with \"pip install -rrequirements_server.txt -U\"") + logger.info("Some dependencies are old. Update them with \"pip install -r requirements_server.txt -U\"") if missing_deps: From d4a2824d38182d4e6c25ac9ce6aa5a82c463baf9 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 27 Nov 2017 19:41:17 -0300 Subject: [PATCH 0572/1506] Delete programming: remove interface --- model/api.py | 103 ++------------------------ model/controller.py | 42 +---------- plugins/controller.py | 6 +- plugins/modelactions.py | 10 +-- test_cases/test_ModelObjectFactory.py | 0 5 files changed, 11 insertions(+), 150 deletions(-) create mode 100644 test_cases/test_ModelObjectFactory.py diff --git a/model/api.py b/model/api.py index cb3059a2ceb..f6ffdd54e14 100644 --- a/model/api.py +++ b/model/api.py @@ -90,8 +90,6 @@ def _setUpAPIServer(hostname=None, port=None): # register all the api functions to be exposed by the server _xmlrpc_api_server.register_function(createAndAddHost) - _xmlrpc_api_server.register_function(createAndAddInterface) - _xmlrpc_api_server.register_function(createAndAddServiceToInterface) _xmlrpc_api_server.register_function(createAndAddNoteToService) _xmlrpc_api_server.register_function(createAndAddNoteToHost) _xmlrpc_api_server.register_function(createAndAddNoteToNote) @@ -99,10 +97,7 @@ def _setUpAPIServer(hostname=None, port=None): _xmlrpc_api_server.register_function(createAndAddVulnToService) _xmlrpc_api_server.register_function(createAndAddVulnToHost) _xmlrpc_api_server.register_function(addHost) - _xmlrpc_api_server.register_function(addInterface) - _xmlrpc_api_server.register_function(addServiceToInterface) _xmlrpc_api_server.register_function(newHost) - _xmlrpc_api_server.register_function(newInterface) _xmlrpc_api_server.register_function(newService) _xmlrpc_api_server.register_function(devlog) @@ -143,48 +138,17 @@ def createAndAddHost(name, os="Unknown"): return host.getID() return None -def createAndAddInterface(host_id, name = "", mac = "00:00:00:00:00:00", - ipv4_address = "0.0.0.0", ipv4_mask = "0.0.0.0", - ipv4_gateway = "0.0.0.0", ipv4_dns = [], - ipv6_address = "0000:0000:0000:0000:0000:0000:0000:0000", ipv6_prefix = "00", - ipv6_gateway = "0000:0000:0000:0000:0000:0000:0000:0000", ipv6_dns = [], - network_segment = "", hostname_resolution = []): - """ - Creates a new interface object with the parameters provided and adds it to - the host selected. - If interface is successfuly created and the host exists, it returns the inteface id - It returns None otherwise - """ - interface = newInterface(name, mac, ipv4_address, ipv4_mask, ipv4_gateway, - ipv4_dns,ipv6_address,ipv6_prefix,ipv6_gateway,ipv6_dns, - network_segment, hostname_resolution, parent_id=host_id) - if addInterface(host_id, interface): - return interface.getID() - return None - -def createAndAddServiceToInterface(host_id, interface_id, name, protocol = "tcp?", - ports = [], status = "running", version = "unknown", description = ""): - service = newService(name, protocol, ports, status, version, description, parent_id=interface_id) - if addServiceToInterface(host_id, interface_id, service): - return service.getID() - return None # Vulnerability - def createAndAddVulnToHost(host_id, name, desc, ref, severity, resolution): vuln = newVuln(name, desc, ref, severity, resolution, parent_id=host_id) if addVulnToHost(host_id, vuln): return vuln.getID() return None -def createAndAddVulnToInterface(host_id, interface_id, name, desc, ref, severity, resolution): - vuln = newVuln(name, desc, ref, severity, resolution, parent_id=interface_id) - if addVulnToInterface(host_id, interface_id, vuln): - return vuln.getID() - return None def createAndAddVulnToService(host_id, service_id, name, desc, ref, severity, resolution): - #we should give the interface_id or de application_id too? I think we should... + #we should give the application_id too? I think we should... vuln = newVuln(name, desc, ref, severity, resolution, parent_id=service_id) if addVulnToService(host_id, service_id, vuln): return vuln.getID() @@ -194,7 +158,7 @@ def createAndAddVulnToService(host_id, service_id, name, desc, ref, severity, re def createAndAddVulnWebToService(host_id, service_id, name, desc, ref, severity, resolution, website, path, request, response, method,pname, params,query,category): - #we should give the interface_id or de application_id too? I think we should... + #we should give the application_id too? I think we should... vuln = newVulnWeb(name, desc, ref, severity, resolution, website, path, request, response, method,pname, params, query, category, parent_id=service_id) if addVulnWebToService(host_id, service_id, vuln): @@ -209,11 +173,6 @@ def createAndAddNoteToHost(host_id, name, text): return note.getID() return None -def createAndAddNoteToInterface(host_id, interface_id, name, text): - note = newNote(name, text, parent_id=interface_id) - if addNoteToInterface(host_id, interface_id, note): - return note.getID() - return None def createAndAddNoteToService(host_id, service_id, name, text): note = newNote(name, text, parent_id=service_id) @@ -245,18 +204,9 @@ def addHost(host): return True return False -def addInterface(host_id, interface): - if interface is not None: - __model_controller.addInterfaceASYNC(host_id, interface) - return True - return False - -def addServiceToInterface(host_id, interface_id, service): - if service is not None: - __model_controller.addServiceToInterfaceASYNC(host_id, interface_id, service) - return True - return False +def addServiceToHost(host_id, service): + pass # Vulnerability @@ -266,11 +216,6 @@ def addVulnToHost(host_id, vuln): return True return False -def addVulnToInterface(host_id, interface_id, vuln): - if vuln is not None: - __model_controller.addVulnToInterfaceASYNC(host_id, interface_id, vuln) - return True - return False def addVulnToService(host_id, service_id, vuln): if vuln is not None: @@ -285,19 +230,14 @@ def addVulnWebToService(host_id, service_id, vuln): return True return False -# Notes +# Notes def addNoteToHost(host_id, note): if note is not None: __model_controller.addNoteToHostASYNC(host_id, note) return True return False -def addNoteToInterface(host_id, interface_id, note): - if note is not None: - __model_controller.addNoteToInterfaceASYNC(host_id, interface_id, note) - return True - return False def addNoteToService(host_id, service_id, note): if note is not None: @@ -326,24 +266,11 @@ def delHost(hostname): return True -def delInterface(hostname,intname): - __model_controller.delInterfaceASYNC(hostname,intname) - return True - def delServiceFromHost(hostname, service): __model_controller.delServiceFromHostASYNC(hostname, service) return True -def delServiceFromInterface(hostname, intname, service, remote = True): - __model_controller.delServiceFromInterfaceASYNC(hostname,intname,service) - return True - -# Vulnerability -#------------------------------------------------------------------------------- -def delVulnFromInterface(vuln, hostname, intname): - __model_controller.delVulnFromInterfaceASYNC(hostname,intname, vuln) - return True #------------------------------------------------------------------------------- def delVulnFromHost(vuln, hostname): __model_controller.delVulnFromHostASYNC(hostname,vuln) @@ -354,11 +281,7 @@ def delVulnFromService(vuln, hostname, srvname): __model_controller.delVulnFromServiceASYNC(hostname,srvname, vuln) return True -# Notes -#------------------------------------------------------------------------------- -def delNoteFromInterface(note, hostname, intname): - __model_controller.delNoteFromInterfaceASYNC(hostname,intname, note) - return True + #------------------------------------------------------------------------------- def delNoteFromHost(note, hostname): __model_controller.delNoteFromHostASYNC(hostname, note) @@ -385,20 +308,6 @@ def newHost(name, os = "Unknown"): """ return __model_controller.newHost(name, os) -def newInterface(name = "", mac = "00:00:00:00:00:00", - ipv4_address = "0.0.0.0", ipv4_mask = "0.0.0.0", - ipv4_gateway = "0.0.0.0", ipv4_dns = [], - ipv6_address = "0000:0000:0000:0000:0000:0000:0000:0000", ipv6_prefix = "00", - ipv6_gateway = "0000:0000:0000:0000:0000:0000:0000:0000", ipv6_dns = [], - network_segment = "", hostname_resolution = [], parent_id=None): - """ - It creates and returns an Interface object. - The created object is not added to the model. - """ - return __model_controller.newInterface( - name, mac, ipv4_address, ipv4_mask, ipv4_gateway, ipv4_dns, - ipv6_address, ipv6_prefix, ipv6_gateway, ipv6_dns, network_segment, - hostname_resolution, parent_id) def newService(name, protocol = "tcp?", ports = [], status = "running", version = "unknown", description = "", parent_id=None): diff --git a/model/controller.py b/model/controller.py index a031131ec31..903f421c179 100644 --- a/model/controller.py +++ b/model/controller.py @@ -22,24 +22,19 @@ # and things are really messy CONF = getInstanceConfiguration() +logger = logging.getLogger(__name__) class modelactions: ADDHOST = 2000 DELHOST = 2001 - ADDINTERFACE = 2002 - DELINTERFACE = 2003 - ADDSERVICEINT = 2004 - DELSERVICEINT = 2006 + ADDSERVICEHOST = 2008 + ADDSERVICEHOST = 20008 ADDCATEGORY = 2011 - ADDVULNINT = 2013 - DELVULNINT = 2014 ADDVULNHOST = 2017 DELVULNHOST = 2018 ADDVULNSRV = 2019 DELVULNSRV = 2020 - ADDNOTEINT = 2021 - DELNOTEINT = 2022 ADDNOTEHOST = 2025 DELNOTEHOST = 2026 ADDNOTESRV = 2027 @@ -71,13 +66,7 @@ class modelactions: __descriptions = { ADDHOST: "ADDHOST", DELHOST: "DELHOST", - ADDINTERFACE: "ADDINTERFACE", - DELINTERFACE: "DELINTERFACE", - ADDSERVICEINT: "ADDSERVICEINT", - DELSERVICEINT: "DELSERVICEINT", ADDCATEGORY: "ADDCATEGORY", - ADDVULNINT: "ADDVULNINT", - DELVULNINT: "DELVULNINT", ADDVULNHOST: "ADDVULNHOST", DELVULNHOST: "DELVULNHOST", ADDVULNSRV: "ADDVULNSRV", @@ -213,15 +202,9 @@ def _setupActionDispatcher(self): modelactions.ADDHOST: self.__add, modelactions.DELHOST: self.__del, modelactions.EDITHOST: self.__edit, - modelactions.ADDINTERFACE: checkParentHost(self.__add), - modelactions.DELINTERFACE: self.__del, modelactions.EDITINTERFACE: self.__edit, - modelactions.ADDSERVICEINT: checkParentInterface(self.__add), - modelactions.DELSERVICEINT: self.__del, modelactions.EDITSERVICE: self.__edit, # Vulnerability - modelactions.ADDVULNINT: checkParentInterface(self.__add), - modelactions.DELVULNINT: self.__del, modelactions.ADDVULNHOST: checkParentHost(self.__add), modelactions.DELVULNHOST: self.__del, modelactions.ADDVULNSRV: checkParentService(self.__add), @@ -231,8 +214,6 @@ def _setupActionDispatcher(self): modelactions.ADDVULNWEBSRV: checkParentService(self.__add), modelactions.EDITVULN: self.__edit, # Note - modelactions.ADDNOTEINT: checkParentInterface(self.__add), - modelactions.DELNOTEINT: self.__del, modelactions.ADDNOTEHOST: checkParentHost(self.__add), modelactions.DELNOTEHOST: self.__del, modelactions.ADDNOTESRV: checkParentService(self.__add), @@ -808,23 +789,6 @@ def newHost(self, name, os="Unknown"): models.Host.class_signature, name, self.mappers_manager.workspace_name, os=os, parent_id=None) - def newInterface(self, name, mac="00:00:00:00:00:00", - ipv4_address="0.0.0.0", - ipv4_mask="0.0.0.0", ipv4_gateway="0.0.0.0", ipv4_dns=[], - ipv6_address="0000:0000:0000:0000:0000:0000:0000:0000", - ipv6_prefix="00", - ipv6_gateway="0000:0000:0000:0000:0000:0000:0000:0000", - ipv6_dns=[], network_segment="", hostname_resolution=[], - parent_id=None): - return model.common.factory.createModelObject( - models.Interface.class_signature, name, - self.mappers_manager.workspace_name, mac=mac, ipv4_address=ipv4_address, - ipv4_mask=ipv4_mask, ipv4_gateway=ipv4_gateway, ipv4_dns=ipv4_dns, - ipv6_address=ipv6_address, ipv6_prefix=ipv6_prefix, - ipv6_gateway=ipv6_gateway, ipv6_dns=ipv6_dns, - network_segment=network_segment, - hostnames=hostname_resolution, parent_id=parent_id) - def newService(self, name, protocol="tcp?", ports=[], status="running", version="unknown", description="", parent_id=None): return model.common.factory.createModelObject( diff --git a/plugins/controller.py b/plugins/controller.py index d9ccbf530ee..0d6fa4868d7 100644 --- a/plugins/controller.py +++ b/plugins/controller.py @@ -160,17 +160,13 @@ def _processAction(self, action, parameters): def _setupActionDispatcher(self): self._actionDispatcher = { modelactions.ADDHOST: model.api.addHost, - modelactions.ADDINTERFACE: model.api.addInterface, - modelactions.ADDSERVICEINT: model.api.addServiceToInterface, - modelactions.DELSERVICEINT: model.api.delServiceFromInterface, + modelactions.ADDSERVICEHOST: model.api.addServiceToHost, #Vulnerability - modelactions.ADDVULNINT: model.api.addVulnToInterface, modelactions.ADDVULNHOST: model.api.addVulnToHost, modelactions.ADDVULNSRV: model.api.addVulnToService, #VulnWeb modelactions.ADDVULNWEBSRV: model.api.addVulnWebToService, #Note - modelactions.ADDNOTEINT: model.api.addNoteToInterface, modelactions.ADDNOTEHOST: model.api.addNoteToHost, modelactions.ADDNOTESRV: model.api.addNoteToService, modelactions.ADDNOTENOTE: model.api.addNoteToNote, diff --git a/plugins/modelactions.py b/plugins/modelactions.py index 8e8c773d9f1..402f9d56108 100644 --- a/plugins/modelactions.py +++ b/plugins/modelactions.py @@ -12,25 +12,18 @@ class modelactions: ADDHOST = 2000 CADDHOST = 2001 - ADDINTERFACE = 2002 - CADDINTERFACE = 2003 - ADDSERVICEINT = 2004 ADDSERVICEAPP = 2005 - CADDSERVICEINT = 2006 CADDSERVICEAPP = 2007 CADDSERVICEHOST = 2008 + ADDSERVICEHOST = 20008 ADDAPPLICATION = 2009 CADDAPPLICATION = 2010 - ADDVULNINT = 2013 - CADDVULNINT = 2014 ADDVULNAPP = 2015 CADDVULNAPP = 2016 ADDVULNHOST = 2017 CADDVULNHOST = 2018 ADDVULNSRV = 2019 CADDVULNSRV = 2020 - ADDNOTEINT = 2021 - CADDNOTEINT = 2022 ADDNOTEAPP = 2023 CADDNOTEAPP = 2024 ADDNOTEHOST = 2025 @@ -41,7 +34,6 @@ class modelactions: CADDNOTEVULN = 2031 LOG = 2032 DEVLOG = 2033 - DELSERVICEINT = 2034 ADDCREDSRV = 2035 CADDCREDSRV = 2036 ADDVULNWEBSRV = 2037 diff --git a/test_cases/test_ModelObjectFactory.py b/test_cases/test_ModelObjectFactory.py new file mode 100644 index 00000000000..e69de29bb2d From 3bf87d5adabf2c3c70236bfee246a23142cac389 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 27 Nov 2017 19:42:16 -0300 Subject: [PATCH 0573/1506] Add support to create object without interface --- model/controller.py | 26 ++++++++++++++------------ persistence/server/server.py | 1 + plugins/controller.py | 3 ++- plugins/plugin.py | 32 +++++++++++++++++++++++++------- requirements_server.txt | 1 + server/commands/reports.py | 17 ++++++++++------- 6 files changed, 53 insertions(+), 27 deletions(-) diff --git a/model/controller.py b/model/controller.py index 903f421c179..2c81eb80041 100644 --- a/model/controller.py +++ b/model/controller.py @@ -4,11 +4,13 @@ See the file 'doc/LICENSE' for the license information ''' -import time -import threading -import Queue +#import Queue +import logging import traceback import model.common # this is to make sure the factory is created +from multiprocessing import Process, Lock, Queue +from Queue import Empty +from threading import Thread, RLock from config.configuration import getInstanceConfiguration from utils.logs import getLogger @@ -99,31 +101,31 @@ def getDescription(action): return modelactions.__descriptions.get(action, "") -class ModelController(threading.Thread): +class ModelController(Thread): def __init__(self, mappers_manager): - threading.Thread.__init__(self) + Thread.__init__(self) self.mappers_manager = mappers_manager # set as daemon - self.setDaemon(True) +# self.setDaemon(True) # flag to stop daemon thread self._stop = False # locks needed to make model thread-safe - self._hosts_lock = threading.RLock() + self._hosts_lock = Lock() # count of plugins sending actions self.active_plugins_count = 0 - self.active_plugins_count_lock = threading.RLock() + self.active_plugins_count_lock = Lock() # TODO: check if it is better using collections.deque # a performance analysis should be done # http://docs.python.org/library/collections.html#collections.deque # the actions queue - self._pending_actions = Queue.Queue() + self._pending_actions = Queue() # a reference to the ModelObjectFactory self._object_factory = model.common.factory @@ -136,7 +138,7 @@ def __init__(self, mappers_manager): # This flag & lock are used when the complete model is being persisted self._saving_model_flag = False - self._saving_model_lock = threading.RLock() + self._saving_model_lock = Lock() self._actionDispatcher = None self._setupActionDispatcher() @@ -358,12 +360,12 @@ def processAction(self): parameters = current_action[1:] # dispatch the action self._processAction(action, parameters) - except Queue.Empty: + except Empty: # if timeout was reached, just let the daemon run again # this is done just to be able to test the stop flag # because if we don't do it, the daemon will be blocked forever pass - except Exception: + except Exception as ex: getLogger(self).debug( "something strange happened... unhandled exception?") getLogger(self).debug(traceback.format_exc()) diff --git a/persistence/server/server.py b/persistence/server/server.py index 32781f50cbf..08d6b736acb 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -1519,6 +1519,7 @@ def login_user(uri, uname, upass): except requests.adapters.ReadTimeout: return None + def is_authenticated(uri, cookies): try: resp = requests.get(uri + "/_api/session", cookies=cookies, timeout=1) diff --git a/plugins/controller.py b/plugins/controller.py index 0d6fa4868d7..3eb825a539e 100644 --- a/plugins/controller.py +++ b/plugins/controller.py @@ -139,7 +139,8 @@ def processOutput(self, plugin, output, command_id, isReport=False): "something strange happened... " "unhandled exception?") break - except Exception: + except Exception as ex: + logger.exception(ex) getLogger(self).debug( "new_elem_queue Exception - " "something strange happened... " diff --git a/plugins/plugin.py b/plugins/plugin.py index f5755a17f44..62280177180 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -8,12 +8,15 @@ ''' -import multiprocessing import os import re +import time import Queue import traceback +import multiprocessing +import deprecation +import server.config import model.api import model.common from model.common import factory @@ -29,6 +32,7 @@ from config.configuration import getInstanceConfiguration CONF = getInstanceConfiguration() +VERSION = server.config.__get_version() class PluginBase(object): @@ -147,10 +151,13 @@ def createAndAddHost(self, name, os="unknown"): Host.class_signature, name, os=os, parent_id=None) - host_obj._metadata.creator = self.id + host_obj._metadata.creatoserverr = self.id self.__addPendingAction(modelactions.ADDHOST, host_obj) return host_obj.getID() + @deprecation.deprecated(deprecated_in="3.0", removed_in="3.5", + current_version=VERSION, + details="Interface object removed. Use host or service instead") def createAndAddInterface( self, host_id, name="", mac="00:00:00:00:00:00", ipv4_address="0.0.0.0", ipv4_mask="0.0.0.0", ipv4_gateway="0.0.0.0", @@ -163,6 +170,9 @@ def createAndAddInterface( # backwards compatibility return host_id + @deprecation.deprecated(deprecated_in="3.0", removed_in="3.5", + current_version=VERSION, + details="Interface object removed. Use host or service instead. This will attach Service to Host!") def createAndAddServiceToInterface(self, host_id, interface_id, name, protocol="tcp?", ports=[], status="running", version="unknown", @@ -171,10 +181,10 @@ def createAndAddServiceToInterface(self, host_id, interface_id, name, serv_obj = model.common.factory.createModelObject( Service.class_signature, name, protocol=protocol, ports=ports, status=status, - version=version, description=description, parent_id=interface_id) + version=version, description=description, parent_id=host_id) serv_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDSERVICEINT, host_id, interface_id, serv_obj) + self.__addPendingAction(modelactions.ADDSERVICEHOST, host_id, serv_obj) return serv_obj.getID() def createAndAddVulnToHost(self, host_id, name, desc="", ref=[], @@ -189,6 +199,9 @@ def createAndAddVulnToHost(self, host_id, name, desc="", ref=[], self.__addPendingAction(modelactions.ADDVULNHOST, host_id, vuln_obj) return vuln_obj.getID() + @deprecation.deprecated(deprecated_in="3.0", removed_in="3.5", + current_version=VERSION, + details="Interface object removed. Use host or service instead. Vuln will be added to Host") def createAndAddVulnToInterface(self, host_id, interface_id, name, desc="", ref=[], severity="", resolution=""): @@ -199,7 +212,7 @@ def createAndAddVulnToInterface(self, host_id, interface_id, name, confirmed=False, parent_id=interface_id) vuln_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDVULNINT, host_id, interface_id, vuln_obj) + self.__addPendingAction(modelactions.ADDVULNHOST, host_id, vuln_obj) return vuln_obj.getID() def createAndAddVulnToService(self, host_id, service_id, name, desc="", @@ -240,6 +253,9 @@ def createAndAddNoteToHost(self, host_id, name, text): self.__addPendingAction(modelactions.ADDNOTEHOST, host_id, note_obj) return note_obj.getID() + @deprecation.deprecated(deprecated_in="3.0", removed_in="3.5", + current_version=VERSION, + details="Interface object removed. Use host or service instead. Note will be added to Host") def createAndAddNoteToInterface(self, host_id, interface_id, name, text): note_obj = model.common.factory.createModelObject( @@ -247,7 +263,7 @@ def createAndAddNoteToInterface(self, host_id, interface_id, name, text): name, text=text, parent_id=interface_id) note_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDNOTEINT, host_id, interface_id, note_obj) + self.__addPendingAction(modelactions.ADDNOTEHOST, host_id, note_obj) return note_obj.getID() def createAndAddNoteToService(self, host_id, service_id, name, text): @@ -340,12 +356,13 @@ def run(self): try: self.new_elem_queue.put( self.plugin._pending_actions.get(block=False)) + time.sleep(0) # potentially allowing another process to run except Queue.Empty: model.api.devlog( ("PluginProcess run _pending_actions" " queue Empty. Breaking loop")) break - except Exception: + except Exception as ex: model.api.devlog( ("PluginProcess run getting from " "_pending_action queue - something strange " @@ -353,6 +370,7 @@ def run(self): model.api.devlog(traceback.format_exc()) break + else: done = True diff --git a/requirements_server.txt b/requirements_server.txt index fd02d837bd0..830b1fb965f 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -20,3 +20,4 @@ marshmallow-sqlalchemy filteralchemy filedepot==0.5.0 nplusone==0.8.1 +deprecation==1.0.1 diff --git a/server/commands/reports.py b/server/commands/reports.py index 07a32b8d877..ac3bc6a40fd 100644 --- a/server/commands/reports.py +++ b/server/commands/reports.py @@ -21,9 +21,6 @@ def import_external_reports(workspace_name=None): plugin_manager = PluginManager( os.path.join(CONF.getConfigPath(), "plugins")) mappers_manager = MapperManager() - controller = ModelController(mappers_manager) - workspace_manager = WorkspaceManager(mappers_manager) - setUpAPIs(controller, workspace_manager, hostname=None, port=None) plugin_controller = PluginController( 'PluginController', @@ -36,12 +33,18 @@ def import_external_reports(workspace_name=None): else: query = Workspace.query - process_workspaces(plugin_controller, query) + process_workspaces(mappers_manager, plugin_controller, query) + #controller._pending_actions.join() -def process_workspaces(plugin_controller, query): +def process_workspaces(mappers_manager, plugin_controller, query): processes = [] for workspace in query.all(): + mappers_manager.createMappers(workspace.name) + controller = ModelController(mappers_manager) + workspace_manager = WorkspaceManager(mappers_manager) + setUpAPIs(controller, workspace_manager, hostname=None, port=None) + controller.start() report_manager = ReportManager( 0.1, workspace.name, @@ -51,5 +54,5 @@ def process_workspaces(plugin_controller, query): processes.append(report_manager) report_manager.start() - for thread in tqdm(processes): - thread.join() + #for thread in tqdm(processes): + # thread.join() \ No newline at end of file From fc043d8e9422a619b09ddc8d509281a077cd29ef Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Mon, 27 Nov 2017 19:59:01 -0300 Subject: [PATCH 0574/1506] Add credentials.json file --- credentials.json | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 credentials.json diff --git a/credentials.json b/credentials.json new file mode 100644 index 00000000000..1df8ea97e23 --- /dev/null +++ b/credentials.json @@ -0,0 +1,4 @@ +{ + "username": "your_user_here", + "password": "your_password_here" +} From 0034fe23929aa11ec97266a26ca143853dacb7c8 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 28 Nov 2017 15:07:35 -0300 Subject: [PATCH 0575/1506] Changes to create a host from plugins --- model/common.py | 1 - model/controller.py | 10 ++++++---- persistence/server/models.py | 15 +++------------ persistence/server/server.py | 10 ++++++---- persistence/server/utils.py | 8 +++++--- server/api/modules/hosts.py | 2 +- 6 files changed, 21 insertions(+), 25 deletions(-) diff --git a/model/common.py b/model/common.py index 8fd827ddbed..a7fa6ad5bf4 100644 --- a/model/common.py +++ b/model/common.py @@ -103,7 +103,6 @@ def createModelObject(self, classname, object_name, workspace_name=None, parent_ objargs['_id'] = -1 # they still don't have a server id objargs['id'] = -1 # we'll generate it after making sure the objects are okey tmpObj = self._registered_objects[classname](objargs, workspace_name) - tmpObj.setID(parent_id) return tmpObj else: raise Exception("Object name parameter missing. Cannot create object class: %s" % classname) diff --git a/model/controller.py b/model/controller.py index 2c81eb80041..b92550c3ce1 100644 --- a/model/controller.py +++ b/model/controller.py @@ -430,6 +430,7 @@ def addHostSYNC(self, host): def _save_new_object(self, new_object): res = self.mappers_manager.save(new_object) + new_object.setID(res) if res: notifier.addObject(new_object) return res @@ -439,10 +440,11 @@ def _handle_conflict(self, old_obj, new_obj): return self.addUpdate(old_obj, new_obj) def __add(self, new_obj, parent_id=None, *args): - old_obj = self.mappers_manager.find(new_obj.class_signature, new_obj.getID()) - if not old_obj: - return self._save_new_object(new_obj) - return self._handle_conflict(old_obj, new_obj) + new_obj_id = self._save_new_object(new_obj) + if new_obj_id: + new_obj + # conflict + #return self._handle_conflict(old_obj, new_obj) def __edit(self, obj, *args, **kwargs): obj.updateAttributes(*args, **kwargs) diff --git a/persistence/server/models.py b/persistence/server/models.py index 9d1f9e31066..79b76e19b16 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -229,7 +229,7 @@ def get_services(workspace_name, **params): def get_service(workspace_name, service_id): """Return the Service of id service_id. None if not found.""" - return force_unique(get_services(workspace_name, couchid=service_id)) + return force_unique(get_services(workspace_name, service_id=service_id)) def get_credentials(workspace_name, **params): """Take a workspace name and a arbitrary number of params to customize the @@ -639,13 +639,8 @@ def __init__(self, obj, workspace_name): self._metadata = obj.get('metadata', Metadata(self.owner)) self.updates = [] - def setID(self, parent_id, *args): - if self.id and self.id != -1: - return None - objid = get_hash(args) - if parent_id: - objid = '.'.join([parent_id, objid]) - self.id = objid + def setID(self, id): + self.id = id @staticmethod def publicattrsrefs(): @@ -729,10 +724,6 @@ def __init__(self, host, workspace_name): self.os = host.get('os') if host.get('os') else 'unknown' self.vuln_amount = int(host.get('vulns', 0)) - def setID(self, _): - # empty arg so as to share same interface as other classes' generateID - ModelBase.setID(self, '', self.name) - @staticmethod def publicattrsrefs(): publicattrs = dict(ModelBase.publicattrsrefs(), **{ diff --git a/persistence/server/server.py b/persistence/server/server.py index 08d6b736acb..f0a451bf80e 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -44,6 +44,7 @@ AUTH_PASS = "" OBJECT_TYPE_END_POINT_MAPPER = { 'CommandRunInformation': 'commands', + 'Host': 'hosts', } @@ -75,7 +76,7 @@ def _create_server_api_url(): """Return the server's api url.""" return "{0}/_api/v2".format(_get_base_server_url()) -def _create_server_get_url(workspace_name, object_name=None): +def _create_server_get_url(workspace_name, object_name=None, object_id=None): """Creates a url to get from the server. Takes the workspace name as a string, an object_name paramter which is the object you want to query as a string ('hosts', 'interfaces', etc) . @@ -85,6 +86,7 @@ def _create_server_get_url(workspace_name, object_name=None): Return the get_url as a string. """ object_name = "/{0}".format(object_name) if object_name else "" + object_name += "/{0}/".format(object_id) if object_id else "" get_url = '{0}/ws/{1}{2}'.format(_create_server_api_url(), workspace_name, object_name) @@ -140,6 +142,7 @@ def wrapper(*args, **kwargs): return response return wrapper if FARADAY_UP else func + @_add_session_cookies def _unsafe_io_with_server(server_io_function, server_expected_response, server_url, **payload): @@ -826,7 +829,7 @@ def get_commands_number(workspace_name, **params): """ return int(_get_raw_commands(workspace_name, **params)) -def create_host(workspace_name, id, name, os, default_gateway, +def create_host(workspace_name, ip, os, default_gateway=None, description="", metadata=None, owned=False, owner="", parent=None): """Create a host. @@ -849,8 +852,7 @@ def create_host(workspace_name, id, name, os, default_gateway, A dictionary with the server's response. """ return _save_to_server(workspace_name, - id, - name=name, os=os, + ip=ip, os=os, default_gateway=default_gateway, owned=owned, metadata=metadata, diff --git a/persistence/server/utils.py b/persistence/server/utils.py index 8221f87b535..bec779fa147 100644 --- a/persistence/server/utils.py +++ b/persistence/server/utils.py @@ -31,8 +31,8 @@ def get_object_properties(obj): if not isinstance(obj.getMetadata(), dict): metadata = metadata.toDict() - return {'id': obj.getID(), - 'name': obj.getName(), + return { + 'ip': obj.getName(), 'description': obj.getDescription(), 'metadata': metadata, 'owned': obj.isOwned(), @@ -41,7 +41,9 @@ def get_object_properties(obj): def get_host_properties(host): host_dict = {'os': host.getOS(), - 'default_gateway': host.getDefaultGateway()} + } + if host.getDefaultGateway(): + host['default_gateway'] = host.getDefaultGateway() host_dict.update(get_object_properties(host)) return host_dict diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 9fd73d2114d..a86a921e178 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -29,7 +29,7 @@ class HostSchema(AutoSchema): _rev = fields.String(default='') ip = fields.String(default='') description = fields.String(required=True) # Explicitly set required=True - default_gateway = fields.String(attribute="default_gateway_ip") + default_gateway = fields.String(attribute="default_gateway_ip", allow_none=True) name = fields.String(dump_only=True, attribute='ip', default='') os = fields.String(default='') owned = fields.Boolean(default=False) From 4d3fcd3bac7e1567bb37fe07ec45f8530eb88814 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 28 Nov 2017 15:49:39 -0300 Subject: [PATCH 0576/1506] Add missing imports --- gui/gtk/application.py | 5 ++++- server/{ => api}/modules/info.py | 0 2 files changed, 4 insertions(+), 1 deletion(-) rename server/{ => api}/modules/info.py (100%) diff --git a/gui/gtk/application.py b/gui/gtk/application.py index 89459a445e7..3f4a246cf68 100644 --- a/gui/gtk/application.py +++ b/gui/gtk/application.py @@ -6,12 +6,15 @@ See the file 'doc/LICENSE' for the license information ''' +import traceback import os import sys import threading import webbrowser +import restkit + try: import gi except ImportError as e: @@ -172,7 +175,7 @@ def remove_workspace(self, button, ws_name): self.ws_sidebar.clear_sidebar() self.ws_sidebar.refresh_sidebar() except restkit.errors.Unauthorized: - notification_center.showDialog( + model.notification_center.showDialog( "You're not authorized to delete this workspace.\n" "Make sure you're an admin and that you're logged in.", "ERROR") diff --git a/server/modules/info.py b/server/api/modules/info.py similarity index 100% rename from server/modules/info.py rename to server/api/modules/info.py From 47d16e55c4651ecb8647d80bd0cc6de0dee0d12b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 28 Nov 2017 15:50:32 -0300 Subject: [PATCH 0577/1506] Use ip instead of names --- model/api.py | 8 ++++---- model/controller.py | 4 ++-- persistence/server/utils.py | 16 ++++++---------- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/model/api.py b/model/api.py index f6ffdd54e14..e569b82094f 100644 --- a/model/api.py +++ b/model/api.py @@ -132,8 +132,8 @@ def _setUpAPIServer(hostname=None, port=None): # plugin created the object -def createAndAddHost(name, os="Unknown"): - host = newHost(name, os) +def createAndAddHost(ip, os="Unknown"): + host = newHost(ip, os) if addHost(host): return host.getID() return None @@ -301,12 +301,12 @@ def delCredFromService(cred, hostname, srvname): # CREATION APIS #------------------------------------------------------------------------------- -def newHost(name, os = "Unknown"): +def newHost(ip, os = "Unknown"): """ It creates and returns a Host object. The object created is not added to the model. """ - return __model_controller.newHost(name, os) + return __model_controller.newHost(ip, os) def newService(name, protocol = "tcp?", ports = [], status = "running", diff --git a/model/controller.py b/model/controller.py index b92550c3ce1..bdea7fb1cf1 100644 --- a/model/controller.py +++ b/model/controller.py @@ -788,9 +788,9 @@ def addCredSYNC(self, model_object_id, newCred): def delCredSYNC(self, model_object, cred_id): self._processAction(modelactions.DELCRED, [cred_id], sync=True) - def newHost(self, name, os="Unknown"): + def newHost(self, ip, os="Unknown"): return model.common.factory.createModelObject( - models.Host.class_signature, name, + models.Host.class_signature, ip, self.mappers_manager.workspace_name, os=os, parent_id=None) def newService(self, name, protocol="tcp?", ports=[], status="running", diff --git a/persistence/server/utils.py b/persistence/server/utils.py index bec779fa147..7dbac15c95d 100644 --- a/persistence/server/utils.py +++ b/persistence/server/utils.py @@ -32,7 +32,7 @@ def get_object_properties(obj): metadata = metadata.toDict() return { - 'ip': obj.getName(), + 'name': obj.getName(), 'description': obj.getDescription(), 'metadata': metadata, 'owned': obj.isOwned(), @@ -45,17 +45,12 @@ def get_host_properties(host): if host.getDefaultGateway(): host['default_gateway'] = host.getDefaultGateway() host_dict.update(get_object_properties(host)) + # name was removed from host and changed to ip + ip = host_dict.pop('name') + # regex ip y lanzar warning si no es ip. + host_dict['ip'] = ip return host_dict -def get_interface_properties(interface): - interface_dict = {'mac': interface.getMAC(), - 'hostnames': interface.getHostnames(), - 'network_segment': interface.getNetworkSegment(), - 'ipv4': interface.getIPv4(), - 'ipv6': interface.getIPv6() - } - interface_dict.update(get_object_properties(interface)) - return interface_dict def get_service_properties(service): service_dict = {'ports': service.getPorts(), @@ -66,6 +61,7 @@ def get_service_properties(service): service_dict.update(get_object_properties(service)) return service_dict + def get_vuln_properties(vuln): vuln_dict = {'confirmed': vuln.getConfirmed(), 'data': vuln.getData(), From adebfec3257ba1138ffb5bde18110ebe9f47cd98 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 28 Nov 2017 15:50:50 -0300 Subject: [PATCH 0578/1506] Remove verify parent decorator --- model/controller.py | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/model/controller.py b/model/controller.py index bdea7fb1cf1..fa912c37d0f 100644 --- a/model/controller.py +++ b/model/controller.py @@ -193,13 +193,6 @@ def addWrapper(new_obj, parent_id=None, *args): def _setupActionDispatcher(self): - # these are decorators for the __add method. - checkParentHost = self._checkParent('Host') - checkParentInterface = self._checkParent('Interface') - checkParentService = self._checkParent('Service') - checkParentVuln = self._checkParent('Vuln') - checkParentNote = self._checkParent('Note') - self._actionDispatcher = { modelactions.ADDHOST: self.__add, modelactions.DELHOST: self.__del, @@ -207,28 +200,28 @@ def _setupActionDispatcher(self): modelactions.EDITINTERFACE: self.__edit, modelactions.EDITSERVICE: self.__edit, # Vulnerability - modelactions.ADDVULNHOST: checkParentHost(self.__add), + modelactions.ADDVULNHOST: self.__add, modelactions.DELVULNHOST: self.__del, - modelactions.ADDVULNSRV: checkParentService(self.__add), + modelactions.ADDVULNSRV: self.__add, modelactions.DELVULNSRV: self.__del, modelactions.ADDVULN: self.__add, modelactions.DELVULN: self.__del, - modelactions.ADDVULNWEBSRV: checkParentService(self.__add), + modelactions.ADDVULNWEBSRV: self.__add, modelactions.EDITVULN: self.__edit, # Note - modelactions.ADDNOTEHOST: checkParentHost(self.__add), + modelactions.ADDNOTEHOST: self.__add, modelactions.DELNOTEHOST: self.__del, - modelactions.ADDNOTESRV: checkParentService(self.__add), + modelactions.ADDNOTESRV: self.__add, modelactions.DELNOTESRV: self.__del, - modelactions.ADDNOTEVULN: checkParentVuln(self.__add), + modelactions.ADDNOTEVULN: self.__add, modelactions.ADDNOTE: self.__add, modelactions.DELNOTE: self.__del, - modelactions.ADDCREDSRV: checkParentService(self.__add), + modelactions.ADDCREDSRV: self.__add, modelactions.DELCREDSRV: self.__del, - modelactions.ADDNOTENOTE: checkParentNote(self.__add), + modelactions.ADDNOTENOTE: self.__add, modelactions.EDITNOTE: self.__edit, modelactions.EDITCRED: self.__edit, - modelactions.ADDCRED: checkParentHost(self.__add), + modelactions.ADDCRED: self.__add, modelactions.DELCRED: self.__del, # Plugin states modelactions.PLUGINSTART: self._pluginStart, From e9b7e526ea994cbbe027a80a4708c7ccc1cfa259 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 28 Nov 2017 15:51:27 -0300 Subject: [PATCH 0579/1506] Fix info api --- persistence/server/models.py | 2 +- server/api/modules/info.py | 2 +- server/app.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index 79b76e19b16..9a1978274bf 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -592,7 +592,7 @@ def delete_workspace(workspace_name): def get_workspaces_names(): """Return a list with all the workspace names available.""" - return server.get_workspaces_names()['workspaces'] + return map(lambda ws: ws['name'], server.get_workspaces_names()) def server_info(): """True if server is up, False otherwise.""" diff --git a/server/api/modules/info.py b/server/api/modules/info.py index 4bcf7e09390..ffb5a596fff 100644 --- a/server/api/modules/info.py +++ b/server/api/modules/info.py @@ -9,7 +9,7 @@ info_api = Blueprint('info_api', __name__) -@info_api.route('/info', methods=['GET']) +@info_api.route('/v2/info', methods=['GET']) def show_info(): faraday_directory = os.path.dirname(os.path.realpath('faraday.py')) diff --git a/server/app.py b/server/app.py index 7cb6e3d70b9..6973ae38982 100644 --- a/server/app.py +++ b/server/app.py @@ -47,7 +47,7 @@ def setup_storage_path(): def register_blueprints(app): - from server.modules.info import info_api + from server.api.modules.info import info_api from server.api.modules.commandsrun import commandsrun_api from server.api.modules.credentials import credentials_api from server.api.modules.doc import doc_api From ec2fb9005b164c3bb9d5951381149d9312a95a59 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 28 Nov 2017 19:58:17 -0300 Subject: [PATCH 0580/1506] Handle conflicts with new api * Currently using __class__ and workspace from _workspace. if the idea works this could be improved. * API returns old object when 409 status code is returned. --- model/controller.py | 11 ++++++----- server/api/base.py | 23 +++++++++++++---------- server/app.py | 1 + test_cases/test_api_hosts.py | 20 ++++++++++++++++++++ 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/model/controller.py b/model/controller.py index fa912c37d0f..64ab756a189 100644 --- a/model/controller.py +++ b/model/controller.py @@ -13,6 +13,7 @@ from threading import Thread, RLock from config.configuration import getInstanceConfiguration +from persistence.server.server_io_exceptions import ConflictInDatabase from utils.logs import getLogger import model.api as api from model.guiapi import notification_center as notifier @@ -433,11 +434,11 @@ def _handle_conflict(self, old_obj, new_obj): return self.addUpdate(old_obj, new_obj) def __add(self, new_obj, parent_id=None, *args): - new_obj_id = self._save_new_object(new_obj) - if new_obj_id: - new_obj - # conflict - #return self._handle_conflict(old_obj, new_obj) + try: + self._save_new_object(new_obj) + except ConflictInDatabase as conflict: + old_obj = new_obj.__class__(conflict.answer.json()['object'], new_obj._workspace_name) + return self._handle_conflict(old_obj, new_obj) def __edit(self, obj, *args, **kwargs): obj.updateAttributes(*args, **kwargs) diff --git a/server/api/base.py b/server/api/base.py index e4de426870b..df48a3c1f9b 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -157,10 +157,10 @@ def _validate_uniqueness(self, obj, object_id=None): @classmethod def register(cls, app, *args, **kwargs): """Register and add JSON error handler. Use error code - 400 instead of 422""" + 400 instead of 409""" super(GenericView, cls).register(app, *args, **kwargs) - @app.errorhandler(422) - def handle_unprocessable_entity(err): + @app.errorhandler(409) + def handle_conflict(err): # webargs attaches additional metadata to the `data` attribute exc = getattr(err, 'exc') if exc: @@ -168,9 +168,7 @@ def handle_unprocessable_entity(err): messages = exc.messages else: messages = ['Invalid request'] - return flask.jsonify({ - 'messages': messages, - }), 400 + return flask.jsonify(messages), 409 @app.errorhandler(InvalidUsage) def handle_invalid_usage(error): @@ -231,11 +229,16 @@ def _validate_uniqueness(self, obj, object_id=None): # The object already exists in DB, we want to fetch an object # different to this one but with the same unique field query = query.filter(primary_key_field != object_id) - if query.one_or_none(): + obj = query.one_or_none() + conflict_data = self._get_schema_class()().dump(obj).data + if obj: db.session.rollback() - abort(422, ValidationError('Existing value for %s field: %s' % ( - field_name, value - ))) + abort(409, ValidationError( + { + 'message': 'Existing value for %s field: %s' % (field_name, value), + 'object': conflict_data, + } + )) class ListMixin(object): diff --git a/server/app.py b/server/app.py index 6973ae38982..baccb694b31 100644 --- a/server/app.py +++ b/server/app.py @@ -2,6 +2,7 @@ # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information import logging + import os from os.path import join, expanduser diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 6037202c779..1fc58719251 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -422,3 +422,23 @@ def test_sort_by_services(self, test_client, session, second_workspace, '?sort=services&sort_dir=asc') assert res.status_code == 200 assert [h['_id'] for h in res.json['data']] == expected_ids + + def test_create_a_host_twice_returns_conflict(self, test_client): + res = test_client.post(self.url(), data={ + "ip": "127.0.0.1", + "description": "aaaaa", + }) + assert res.status_code == 201 + assert Host.query.count() == HOSTS_COUNT + 1 + host_id = res.json['id'] + host = Host.query.get(host_id) + assert host.ip == "127.0.0.1" + assert host.description == "aaaaa" + assert host.os is None + assert host.workspace == self.workspace + res = test_client.post(self.url(), data={ + "ip": "127.0.0.1", + "description": "aaaaa", + }) + assert res.status_code == 409 + assert res.json['object']['_id'] == host_id \ No newline at end of file From 822192b287a0e9a68f3f457c5365220fa70c41d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 29 Nov 2017 15:50:21 -0300 Subject: [PATCH 0581/1506] Fix cross-workspace bug when filtering vulns by command_id I was able to set the command id of other workspace and see all its related vulnerabilities, despite the fact that they are from other workspace. I fixed it and added a test to avoid regressions. --- server/api/modules/vulns.py | 7 +++++-- test_cases/test_api_vulnerability.py | 15 +++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 362bc984899..0e2e3b2b571 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -284,12 +284,15 @@ def filter(self): """ command_id = request.args.get('command_id') if command_id: - self.query = db.session.query(VulnerabilityGeneric).join(CommandObject, and_(VulnerabilityWeb.id == CommandObject.object_id, CommandObject.object_type=='vulnerability')) + self.query = self.query.join( + CommandObject, and_( + VulnerabilityWeb.id == CommandObject.object_id, + CommandObject.object_type == 'vulnerability')) query = super(VulnerabilityFilterSet, self).filter() if command_id: - query = query.filter(CommandObject.command_id==int(command_id)) + query = query.filter(CommandObject.command_id == int(command_id)) return query diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 020a4e39c3c..9c2401a5587 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -729,10 +729,11 @@ def test_target(self, test_client, session, second_workspace, @pytest.mark.usefixtures('mock_envelope_list') @pytest.mark.usefixtures('ignore_nplusone') def test_filter_by_command_id(self, test_client, session, - second_workspace, - vulnerability_factory, - vulnerability_web_factory, - ): + second_workspace, + workspace, + vulnerability_factory, + vulnerability_web_factory, + ): expected_ids = set() web_expected_ids = set() host = HostFactory.create(workspace=second_workspace) @@ -804,6 +805,12 @@ def test_filter_by_command_id(self, test_client, session, vuln['metadata']['command_id'] == command_object.command.id assert set(vuln['_id'] for vuln in res.json['data']) == web_expected_ids + # Check for cross-workspace bugs + res = test_client.get(self.url( + workspace=workspace) + '?command_id={0}'.format(web_command.id)) + assert res.status_code == 200 + assert len(res.json['data']) == 0 + def test_vulnerability_metadata(self, session, test_client, workspace): owner = UserFactory.create() service = ServiceFactory.create(workspace=workspace) From a50c9b9cce203c6b4ea7bdc647ec3c2b9461f620 Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 29 Nov 2017 16:05:09 -0300 Subject: [PATCH 0582/1506] Add release information for CouchDB's death --- RELEASE.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index ed48d9c7146..6196fca70f0 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -8,6 +8,14 @@ Make sure you run ```./faraday.py --update``` the first time after an update! New features in the latest update ===================================== +TBA: +--- +* CouchDB was replaced by PostgreSQL :) +* Host object changed, now the name property is called ip +* Interface object was removed +* Note object was removed and replaced with Comment +* Communication object was removed and replaced with Comment + November 17, 2017: --- * Fix bug with tags in models. From 26723e8d65eaba023427d2232a5b2247afa965f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 29 Nov 2017 16:25:52 -0300 Subject: [PATCH 0583/1506] Add service filter in hosts API So the host by service modal of the dashboard works properly --- server/api/modules/hosts.py | 12 ++++++++++-- test_cases/test_api_hosts.py | 20 ++++++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 9fd73d2114d..74f3246bb8a 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -6,7 +6,7 @@ from flask import Blueprint from flask_classful import route from marshmallow import fields -from filteralchemy import FilterSet, operators +from filteralchemy import Filter, FilterSet, operators from server.utils.database import get_or_create @@ -53,11 +53,19 @@ class Meta: ) +class ServiceFilter(Filter): + """Filter hosts by service name""" + + def filter(self, query, model, attr, value): + return query.filter(model.services.any(Service.name == value)) + + class HostFilterSet(FilterSet): class Meta(FilterSetMeta): model = Host - fields = ('os',) + fields = ('os', 'service') operators = (operators.Equal, operators.Like, operators.ILike) + service = ServiceFilter(fields.Str()) class ServiceSchema(AutoSchema): diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 6037202c779..57dc5c579ab 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -273,9 +273,25 @@ def test_filter_by_os_like_ilike(self, test_client, session, workspace, assert res.status_code == 200 self.compare_results(hosts + [case_insensitive_host], res) + def test_filter_by_service(self, test_client, session, workspace, + service_factory, host_factory): + services = service_factory.create_batch(10, workspace=workspace, + name="IRC") + hosts = [service.host for service in services] + + # Hosts that shouldn't be shown + host_factory.create_batch(5, workspace=workspace) + + res = test_client.get(self.url() + '?service=IRC') + assert res.status_code == 200 + shown_hosts_ids = set(obj['id'] for obj in res.json['rows']) + expected_host_ids = set(host.id for host in hosts) + assert shown_hosts_ids == expected_host_ids + def test_host_with_open_vuln_count_verification(self, test_client, session, - workspace, host_factory, vulnerability_factory, - service_factory): + workspace, host_factory, + vulnerability_factory, + service_factory): host = host_factory.create(workspace=workspace) service = service_factory.create(host=host, workspace=workspace) From a4e59e2ea5a90cd2d513375f5d881a4e60eb3005 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 29 Nov 2017 17:08:26 -0300 Subject: [PATCH 0584/1506] Fix services modal of the dashboard --- .../controllers/summarizedCtrlServicesModal.js | 8 +++----- .../dashboard/partials/modal-services-by-host.html | 14 +++++++------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/server/www/scripts/dashboard/controllers/summarizedCtrlServicesModal.js b/server/www/scripts/dashboard/controllers/summarizedCtrlServicesModal.js index 5ab671798d7..455b0595467 100644 --- a/server/www/scripts/dashboard/controllers/summarizedCtrlServicesModal.js +++ b/server/www/scripts/dashboard/controllers/summarizedCtrlServicesModal.js @@ -29,14 +29,12 @@ angular.module('faradayApp') } ServerAPI.getServicesByHost(workspace, host._id).then(function(response){ - dashboardSrv.getName(workspace, host._id).then(function(name){ - $scope.name = name; - $scope.services = response.data.services; - }) + $scope.name = host.name; + $scope.services = response.data; }); $scope.ok = function(){ $modalInstance.close(); } - }]); \ No newline at end of file + }]); diff --git a/server/www/scripts/dashboard/partials/modal-services-by-host.html b/server/www/scripts/dashboard/partials/modal-services-by-host.html index 217e7d679ad..a5b9690870c 100644 --- a/server/www/scripts/dashboard/partials/modal-services-by-host.html +++ b/server/www/scripts/dashboard/partials/modal-services-by-host.html @@ -20,16 +20,16 @@

    Services for {{name}} ({{services.length}} total)

    - {{srv.value.name}} - + {{srv.name}} + - {{srv.value.description}} + {{srv.description}} - {{srv.value.ports.toString().replace("[]", "")}} - + {{srv.ports.toString().replace("[]", "")}} + - {{srv.value.protocol}} - {{srv.value.status}} + {{srv.protocol}} + {{srv.status}}
    From ac3a6f2b699fb1fe61835f9e949d793d6191b8a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 29 Nov 2017 18:52:17 -0300 Subject: [PATCH 0585/1506] Align user dropdown menu --- server/www/estilos.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/estilos.css b/server/www/estilos.css index 043eb882008..45c7e436785 100644 --- a/server/www/estilos.css +++ b/server/www/estilos.css @@ -1190,6 +1190,6 @@ a.button-disable{cursor: not-allowed;pointer-events: none;opacity: 0.5} } .header_right { - /* TODO: this is a fast hask, maybe there is a better way to do this */ - margin-top: -3%; + /* TODO: this is a fast hack, maybe there is a better way to do this */ + margin-top: -2.25%; } From 7a9fadb28769d8e8e6823848286eed3b748d1a92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 30 Nov 2017 12:17:59 -0300 Subject: [PATCH 0586/1506] Fix vuln and host deleting in the web ui --- server/www/scripts/commons/providers/server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index 6ba36efbec9..df921c70055 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -413,7 +413,7 @@ angular.module("faradayApp") } ServerAPI.deleteHost = function(wsName, hostId, rev) { - var deleteUrl = createDeleteUrl(wsName, hostId, rev); + var deleteUrl = createDeleteUrl(wsName, hostId, 'hosts'); if (typeof rev === "undefined") { return _delete(deleteUrl, false) } @@ -433,7 +433,7 @@ angular.module("faradayApp") } ServerAPI.deleteVuln = function(wsName, vulnId, rev) { - var deleteUrl = createDeleteUrl(wsName, vulnId, rev); + var deleteUrl = createDeleteUrl(wsName, vulnId, 'vulns'); if (typeof rev === "undefined") { return _delete(deleteUrl, false) } From 805342e1157173dc85eb3c72c873196e1797fc18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 30 Nov 2017 12:32:38 -0300 Subject: [PATCH 0587/1506] Show hostnaems in host list --- server/www/scripts/hosts/partials/list.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/www/scripts/hosts/partials/list.html b/server/www/scripts/hosts/partials/list.html index e42a8a3b1e2..dbb649de71d 100644 --- a/server/www/scripts/hosts/partials/list.html +++ b/server/www/scripts/hosts/partials/list.html @@ -54,6 +54,9 @@

    Name + + Hostnames + Open Services @@ -85,6 +88,11 @@

    {{host.name}} + + + {{hostname}}
    +     + From 7817a51e53c59ca6d1003fb4f3907ca9663248ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 30 Nov 2017 16:10:19 -0300 Subject: [PATCH 0588/1506] Fix host creation and hostname showing --- server/www/scripts/hosts/controllers/new.js | 2 +- server/www/scripts/services/partials/list.html | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/www/scripts/hosts/controllers/new.js b/server/www/scripts/hosts/controllers/new.js index 9a8be36c6b2..08005b67c0b 100644 --- a/server/www/scripts/hosts/controllers/new.js +++ b/server/www/scripts/hosts/controllers/new.js @@ -38,7 +38,7 @@ angular.module('faradayApp') $scope.insert = function(hostdata) { hostsManager.createHost(hostdata, $scope.workspace).then(function(host) { - $location.path('/host/ws/' + $scope.workspace + '/hid/' + $scope.host.id); + $location.path('/host/ws/' + $scope.workspace + '/hid/' + host.data.id); }, function(message) { $uibModal.open({ templateUrl: 'scripts/commons/partials/modalKO.html', diff --git a/server/www/scripts/services/partials/list.html b/server/www/scripts/services/partials/list.html index fad7a52125f..6a53f19d105 100644 --- a/server/www/scripts/services/partials/list.html +++ b/server/www/scripts/services/partials/list.html @@ -187,7 +187,7 @@
    Hostnames
    - +
    @@ -195,7 +195,7 @@
    Hostnames
      -
    • {{hostname.key}}
      • +
    • {{hostname}}
     From d4fe1df9b4c3c85e5f80c496a30725c56f3cf53e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 1 Dec 2017 11:55:53 -0300 Subject: [PATCH 0589/1506] Fix vuln template updates and severities Update wasn't updating everything and medium and info severities were not working because a lack of translation between javascript and the value in the DB. I fixed it by adding a SeverityField both in vuln templates and vulns API --- server/api/modules/vulnerability_template.py | 5 ++-- server/api/modules/vulns.py | 17 ++---------- server/schemas.py | 27 ++++++++++++++++++- .../www/scripts/vulndb/providers/vulnModel.js | 2 +- test_cases/test_api_vulnerability.py | 20 ++++++++++++++ 5 files changed, 52 insertions(+), 19 deletions(-) diff --git a/server/api/modules/vulnerability_template.py b/server/api/modules/vulnerability_template.py index aa1d412a0d6..e560d83619b 100644 --- a/server/api/modules/vulnerability_template.py +++ b/server/api/modules/vulnerability_template.py @@ -21,7 +21,7 @@ ReadWriteView, ) from server.models import VulnerabilityTemplate -from server.schemas import PrimaryKeyRelatedField +from server.schemas import PrimaryKeyRelatedField, SeverityField vulnerability_template_api = Blueprint('vulnerability_template_api', __name__) logger = server.utils.logger.get_logger(__name__) @@ -31,7 +31,7 @@ class VulnerabilityTemplateSchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') _rev = fields.String(default='', dump_only=True) cwe = fields.String(dump_only=True, default='') # deprecated field, the legacy data is added to refs on import - exploitation = fields.String(attribute='severity', validate=OneOf(VulnerabilityTemplate.SEVERITIES), required=True) + exploitation = SeverityField(attribute='severity', required=True) references = fields.Method('get_references', deserialize='load_references', required=True) class Meta: @@ -80,6 +80,7 @@ def _envelope_list(self, objects, pagination_metadata=None): }) return { 'rows': vuln_tpls, + 'total_rows': len(objects) } diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 0e2e3b2b571..e77a3b41ad1 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -42,6 +42,7 @@ MutableField, PrimaryKeyRelatedField, SelfNestedField, + SeverityField, MetadataSchema) vulns_api = Blueprint('vulns_api', __name__) @@ -99,7 +100,7 @@ class VulnerabilitySchema(AutoSchema): '_id', 'ports', 'status', 'protocol', 'name', 'version', 'summary' ]), dump_only=True) host = fields.Integer(dump_only=True, attribute='host_id') - severity = fields.Method(serialize='get_severity', deserialize='load_severity') + severity = SeverityField(required=True) status = fields.Method(serialize='get_status', deserialize='load_status') # TODO: this breaks enum validation. type = fields.Method(serialize='get_type', deserialize='load_type') obj_id = fields.String(dump_only=True, attribute='id') @@ -145,13 +146,6 @@ def get_parent_type(self, obj): assert obj.service_id is not None or obj.host_id is not None return 'Service' if obj.service_id is not None else 'Host' - def get_severity(self, obj): - if obj.severity == 'medium': - return 'med' - if obj.severity == 'informational': - return 'info' - return obj.severity - def get_status(self, obj): if obj.status == 'open': return 'opened' @@ -166,13 +160,6 @@ def get_target(self, obj): else: return obj.host.ip - def load_severity(self, value): - if value == 'med': - return 'medium' - if value == 'info': - return 'informational' - return value - def load_status(self, value): if value == 'opened': return 'open' diff --git a/server/schemas.py b/server/schemas.py index 002d1f1417d..8258e65f178 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -3,7 +3,7 @@ from marshmallow.exceptions import ValidationError from server.api.base import AutoSchema -from server.models import CommandObject +from server.models import CommandObject, VulnerabilityABC class JSTimestampField(fields.Field): @@ -102,6 +102,31 @@ def _add_to_schema(self, field_name, schema): self.write_field._add_to_schema(field_name, schema) +class SeverityField(fields.String): + """ + Custom field for the severity, with the proper mappings to make + it compatible with the web UI + """ + + def _serialize(self, value, attr, obj): + ret = super(SeverityField, self)._serialize(value, attr, obj) + if ret == 'medium': + return 'med' + elif ret == 'informational': + return 'info' + return ret + + def _deserialize(self, value, attr, data): + ret = super(SeverityField, self)._serialize(value, attr, data) + if ret == 'med': + return 'medium' + elif ret == 'info': + return 'informational' + if ret not in VulnerabilityABC.SEVERITIES: + raise ValidationError("Invalid severity type.") + return ret + + class MetadataSchema(Schema): command_id = fields.Method('get_command_id', dump_only=True) diff --git a/server/www/scripts/vulndb/providers/vulnModel.js b/server/www/scripts/vulndb/providers/vulnModel.js index 9346136bac0..dbdaae6b015 100644 --- a/server/www/scripts/vulndb/providers/vulnModel.js +++ b/server/www/scripts/vulndb/providers/vulnModel.js @@ -61,7 +61,7 @@ angular.module('faradayApp'). var deferred = $q.defer(); var self = this; - ServerAPI.updateVulnerabilityTemplate(self) + ServerAPI.updateVulnerabilityTemplate(data) .then(function(res) { self.set(res.data); deferred.resolve(self); diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 9c2401a5587..5853e2e88cd 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -645,6 +645,26 @@ def test_create_vuln_with_invalid_type(self, host_with_hostnames, test_client, s assert vuln_count_previous == session.query(Vulnerability).count() assert res.json['message'] == 'Invalid vulnerability type.' + def test_create_vuln_with_invalid_severity(self, + host_with_hostnames, + test_client, session): + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='Vulnerability', + parent_id=host_with_hostnames.id, + parent_type='Host', + refs=[], + policyviolations=[], + severity="invalid", + ) + ws_name = host_with_hostnames.workspace.name + vuln_count_previous = session.query(Vulnerability).count() + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) + assert res.status_code == 400 + assert vuln_count_previous == session.query(Vulnerability).count() + assert 'Invalid severity type.' in res.data + def test_create_vuln_with_invalid_ease_of_resolution(self, host_with_hostnames, test_client, session): session.commit() # flush host_with_hostnames raw_data = self._create_post_data_vulnerability( From d1386efab02970ac7de5ae1cc6de1736c3d2e20d Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 1 Dec 2017 12:42:47 -0300 Subject: [PATCH 0590/1506] PEP8 and cleanup commented code --- persistence/server/models.py | 435 ++++++++++++++++------------------- 1 file changed, 198 insertions(+), 237 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index 9a1978274bf..4bbd66ae935 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -45,6 +45,7 @@ def _conf(): else: raise CantAccessConfigurationWithoutTheClient + def _get_merge_strategy(): try: merge_strategy = _conf().getMergeStrategy() @@ -53,13 +54,18 @@ def _get_merge_strategy(): return merge_strategy _CHANGES_LOCK = Lock() + + def get_changes_lock(): return _CHANGES_LOCK _LOCAL_CHANGES_ID_TO_REV = {} + + def local_changes(): return _LOCAL_CHANGES_ID_TO_REV + def _ignore_in_changes(func): @wraps(func) def func_wrapper(*args, **kwargs): @@ -70,6 +76,7 @@ def func_wrapper(*args, **kwargs): return json return func_wrapper + def _flatten_dictionary(dictionary): """Given a dictionary with dictionaries inside, create a new flattened dictionary from that one and return it. @@ -98,6 +105,7 @@ def _flatten_dictionary(dictionary): # client, even if they come from the server: they should have the same # interface as the old style objects, from when we kept them on memory + def _get_faraday_ready_objects(workspace_name, faraday_ready_object_dictionaries, faraday_object_name): """Takes a workspace name, a faraday object ('hosts', 'vulns', @@ -124,10 +132,12 @@ def _get_faraday_ready_objects(workspace_name, faraday_ready_object_dictionaries faraday_objects.append(appropiate_class(flattened_object_dictionary, workspace_name)) return faraday_objects + def _get_faraday_ready_hosts(workspace_name, hosts_dictionaries): """Return a list of Hosts created with the information found on hosts_dictionaries""" return _get_faraday_ready_objects(workspace_name, hosts_dictionaries, 'hosts') + def _get_faraday_ready_vulns(workspace_name, vulns_dictionaries, vulns_type=None): """Return a list of Vuln or VulnWeb objects created with the information found on vulns_dictionaries. @@ -144,22 +154,27 @@ def _get_faraday_ready_vulns(workspace_name, vulns_dictionaries, vulns_type=None faraday_ready_web_vulns = _get_faraday_ready_objects(workspace_name, web_vulns, 'vulns_web') return faraday_ready_vulns + faraday_ready_web_vulns + def _get_faraday_ready_services(workspace_name, services_dictionaries): """Return a list of Services created with the information found on services_dictionaries""" return _get_faraday_ready_objects(workspace_name, services_dictionaries, 'services') + def _get_faraday_ready_credentials(workspace_name, credentials_dictionaries): """Return a list of Credentials created with the information found on credentials_dictionaries""" return _get_faraday_ready_objects(workspace_name, credentials_dictionaries, 'credentials') + def _get_faraday_ready_notes(workspace_name, notes_dictionaries): """Return a list of Notes created with the information found on notes_dictionaries""" return _get_faraday_ready_objects(workspace_name, notes_dictionaries, 'notes') + def _get_faraday_ready_commands(workspace_name, commands_dictionaries): """Return a list of Commands created with the information found on commands_dictionaries""" return _get_faraday_ready_objects(workspace_name, commands_dictionaries, 'commands') + def get_changes_stream(workspace_name): """Take a workspace_name as a string. Return a couchDB change_stream with the changes relevant to the workspace @@ -170,6 +185,7 @@ def get_changes_stream(workspace_name): return server.get_changes_stream(workspace_name, since=since, heartbeat='1000') + def get_hosts(workspace_name, **params): """Take a workspace name and a arbitrary number of params to customize the request. @@ -179,10 +195,12 @@ def get_hosts(workspace_name, **params): host_dictionaries = server.get_hosts(workspace_name, **params) return _get_faraday_ready_hosts(workspace_name, host_dictionaries) + def get_host(workspace_name, host_id): """Return the host by host_id. None if it can't be found.""" return force_unique(get_hosts(workspace_name, couchid=host_id)) + def get_all_vulns(workspace_name, **params): """Take a workspace name and a arbitrary number of params to customize the request. @@ -192,6 +210,7 @@ def get_all_vulns(workspace_name, **params): vulns_dictionaries = server.get_all_vulns(workspace_name, **params) return _get_faraday_ready_vulns(workspace_name, vulns_dictionaries) + def get_vulns(workspace_name, **params): """Take a workspace name and a arbitrary number of params to customize the request. @@ -201,10 +220,12 @@ def get_vulns(workspace_name, **params): vulns_dictionaries = server.get_vulns(workspace_name, **params) return _get_faraday_ready_vulns(workspace_name, vulns_dictionaries, vulns_type='vulns') + def get_vuln(workspace_name, vuln_id): """Return the Vuln of id vuln_id. None if not found.""" return force_unique(get_vulns(workspace_name, couchid=vuln_id)) + def get_web_vulns(workspace_name, **params): """Take a workspace name and a arbitrary number of params to customize the request. @@ -214,10 +235,12 @@ def get_web_vulns(workspace_name, **params): vulns_web_dictionaries = server.get_web_vulns(workspace_name, **params) return _get_faraday_ready_vulns(workspace_name, vulns_web_dictionaries, vulns_type='vulns_web') + def get_web_vuln(workspace_name, vuln_id): """Return the WebVuln of id vuln_id. None if not found.""" return force_unique(get_web_vulns(workspace_name, couchid=vuln_id)) + def get_services(workspace_name, **params): """Take a workspace name and a arbitrary number of params to customize the request. @@ -227,10 +250,12 @@ def get_services(workspace_name, **params): services_dictionary = server.get_services(workspace_name, **params) return _get_faraday_ready_services(workspace_name, services_dictionary) + def get_service(workspace_name, service_id): """Return the Service of id service_id. None if not found.""" return force_unique(get_services(workspace_name, service_id=service_id)) + def get_credentials(workspace_name, **params): """Take a workspace name and a arbitrary number of params to customize the request. @@ -240,10 +265,12 @@ def get_credentials(workspace_name, **params): credentials_dictionary = server.get_credentials(workspace_name, **params) return _get_faraday_ready_credentials(workspace_name, credentials_dictionary) + def get_credential(workspace_name, credential_id): """Return the Credential of id credential_id. None if not found.""" return force_unique(get_credentials(workspace_name, couchid=credential_id)) + def get_notes(workspace_name, **params): """Take a workspace name and a arbitrary number of params to customize the request. @@ -253,15 +280,18 @@ def get_notes(workspace_name, **params): notes_dictionary = server.get_notes(workspace_name, **params) return _get_faraday_ready_notes(workspace_name, notes_dictionary) + def get_note(workspace_name, note_id): """Return the Note of id note_id. None if not found.""" return force_unique(get_notes(workspace_name, couchid=note_id)) + def get_workspace(workspace_name): """Return the Workspace of id workspace_name. None if not found.""" workspace = server.get_workspace(workspace_name) return _Workspace(workspace, workspace_name) if workspace else None + def get_commands(workspace_name, **params): """Take a workspace name and a arbitrary number of params to customize the request. @@ -271,10 +301,12 @@ def get_commands(workspace_name, **params): commands_dictionary = server.get_commands(workspace_name, **params) return _get_faraday_ready_commands(workspace_name, commands_dictionary) + def get_command(workspace_name, command_id): """Return the Command of id command_id. None if not found.""" return force_unique(get_commands(workspace_name, couchid=command_id)) + def get_object(workspace_name, object_signature, object_id): """Given a workspace name, an object_signature as string and an arbitrary number of query params, return a list a dictionaries containg information @@ -298,12 +330,14 @@ def get_object(workspace_name, object_signature, object_id): return appropiate_function(workspace_name, object_id) + def get_deleted_object_name_and_type(workspace_name, object_id): """Return a tupe of (name, type) for the deleted object of object_id, if it can get around CouchDB to do it. Else None""" obj_dict = server.get_object_before_last_revision(workspace_name, object_id) return (obj_dict['name'], obj_dict['type']) if obj_dict else (None, None) + @_ignore_in_changes def create_host(workspace_name, host): """Take a workspace_name and a host object and save it to the sever. @@ -313,6 +347,7 @@ def create_host(workspace_name, host): host_properties = get_host_properties(host) return server.create_host(workspace_name, **host_properties) + @_ignore_in_changes def update_host(workspace_name, host): """Take a workspace_name and a host object and update it in the sever. @@ -322,6 +357,7 @@ def update_host(workspace_name, host): host_properties = get_host_properties(host) return server.update_host(workspace_name, **host_properties) + @_ignore_in_changes def create_service(workspace_name, service): """Take a workspace_name and a service object and save it to the sever. @@ -330,6 +366,7 @@ def create_service(workspace_name, service): service_properties = get_service_properties(service) return server.create_service(workspace_name, **service_properties) + @_ignore_in_changes def update_service(workspace_name, service): """Take a workspace_name and an service object and update it in the sever. @@ -339,6 +376,7 @@ def update_service(workspace_name, service): service_properties = get_service_properties(service) return server.update_service(workspace_name, **service_properties) + @_ignore_in_changes def create_vuln(workspace_name, vuln): """Take a workspace_name and an vulnerability object and save it to the @@ -348,6 +386,7 @@ def create_vuln(workspace_name, vuln): vuln_properties = get_vuln_properties(vuln) return server.create_vuln(workspace_name, **vuln_properties) + @_ignore_in_changes def update_vuln(workspace_name, vuln): """Take a workspace_name and a Vuln object and update it in the sever. @@ -357,6 +396,7 @@ def update_vuln(workspace_name, vuln): vuln_properties = get_vuln_properties(vuln) return server.update_vuln(workspace_name, **vuln_properties) + @_ignore_in_changes def create_vuln_web(workspace_name, vuln_web): """Take a workspace_name and an vulnerabilityWeb object and save it to the @@ -366,6 +406,7 @@ def create_vuln_web(workspace_name, vuln_web): vuln_web_properties = get_vuln_web_properties(vuln_web) return server.create_vuln_web(workspace_name, **vuln_web_properties) + @_ignore_in_changes def update_vuln_web(workspace_name, vuln_web): """Take a workspace_name and a VulnWeb object and update it in the sever. @@ -375,6 +416,7 @@ def update_vuln_web(workspace_name, vuln_web): vuln_web_properties = get_vuln_web_properties(vuln_web) return server.update_vuln_web(workspace_name, **vuln_web_properties) + @_ignore_in_changes def create_note(workspace_name, note): """Take a workspace_name and an note object and save it to the sever. @@ -383,6 +425,7 @@ def create_note(workspace_name, note): note_properties = get_note_properties(note) return server.create_note(workspace_name, **note_properties) + @_ignore_in_changes def update_note(workspace_name, note): """Take a workspace_name and a Note object and update it in the sever. @@ -391,6 +434,7 @@ def update_note(workspace_name, note): note_properties = get_note_properties(note) return server.update_note(workspace_name, **note_properties) + @_ignore_in_changes def create_credential(workspace_name, credential): """Take a workspace_name and an credential object and save it to the sever. @@ -399,6 +443,7 @@ def create_credential(workspace_name, credential): credential_properties = get_credential_properties(credential) return server.create_credential(workspace_name, **credential_properties) + @_ignore_in_changes def update_credential(workspace_name, credential): """Take a workspace_name and a Credential object and update it in the sever. @@ -407,11 +452,13 @@ def update_credential(workspace_name, credential): credential_properties = get_credential_properties(credential) return server.update_credential(workspace_name, **credential_properties) + @_ignore_in_changes def create_command(workspace_name, command): command_properties = get_command_properties(command) return server.create_command(workspace_name, **command_properties) + @_ignore_in_changes def update_command(workspace_name, command): """Take a workspace_name and a Command object and update it in the sever. @@ -420,6 +467,7 @@ def update_command(workspace_name, command): command_properties = get_command_properties(command) return server.update_command(workspace_name, command.getID(), **command_properties) + def create_object(workspace_name, object_signature, obj): """Given a workspace name, an object_signature as string and obj, a Faraday object, save that object on the server. @@ -444,6 +492,7 @@ def create_object(workspace_name, object_signature, obj): return appropiate_function(workspace_name, obj) + def update_object(workspace_name, object_signature, obj): """Given a workspace name, an object_signature as string and obj, a Faraday object, update that object on the server. @@ -482,27 +531,32 @@ def create_workspace(workspace_name, description, start_date, finish_date, return server.create_workspace(workspace_name, description, start_date, finish_date, customer) + def get_workspace_summary(workspace_name): """Return the workspace summary as a dictionary """ return server.get_workspace_summary(workspace_name) + def get_workspace_numbers(workspace_name): """Return a tuple with the number of hosts, services and vulns on the workspace of name workspace_name. """ return server.get_workspace_numbers(workspace_name) + def get_hosts_number(workspace_name, **params): """Return the number of hosts found on the workspace of name workspace_name """ return server.get_hosts_number(workspace_name, **params) + def get_services_number(workspace_name, **params): """Return the number of services found on the workspace of name workspace_name """ return server.get_services_number(workspace_name, **params) + def get_vulns_number(workspace_name, **params): """Return the number of vulns found on the workspace of name workspace_name """ @@ -513,6 +567,7 @@ def get_vulns_number(workspace_name, **params): # except for their names. # maybe implement some kind of validation in the future? + @_ignore_in_changes def delete_host(workspace_name, host_id): """Delete the host of id host_id on workspace workspace_name. @@ -520,6 +575,7 @@ def delete_host(workspace_name, host_id): """ return server.delete_host(workspace_name, host_id) + @_ignore_in_changes def delete_service(workspace_name, service_id): """Delete the service of id service_id on workspace workspace_name. @@ -527,6 +583,7 @@ def delete_service(workspace_name, service_id): """ return server.delete_service(workspace_name, service_id) + @_ignore_in_changes def delete_vuln(workspace_name, vuln_id): """Delete the vuln of id vuln_id on workspace workspace_name. @@ -534,6 +591,7 @@ def delete_vuln(workspace_name, vuln_id): """ return server.delete_vuln(workspace_name, vuln_id) + @_ignore_in_changes def delete_note(workspace_name, note_id): """Delete the note of id note_id on workspace workspace_name. @@ -541,6 +599,7 @@ def delete_note(workspace_name, note_id): """ return server.delete_note(workspace_name, note_id) + @_ignore_in_changes def delete_credential(workspace_name, credential_id): """Delete the credential of id credential_id on workspace workspace_name. @@ -548,6 +607,7 @@ def delete_credential(workspace_name, credential_id): """ return server.delete_credential(workspace_name, credential_id) + @_ignore_in_changes def delete_vuln_web(workspace_name, vuln_id): """Delete the vulnweb of id vulnweb_id on workspace workspace_name. @@ -555,6 +615,7 @@ def delete_vuln_web(workspace_name, vuln_id): """ return server.delete_vuln(workspace_name, vuln_id) + @_ignore_in_changes def delete_command(workspace_name, command_id): """Delete the command of id command_id on workspace workspace_name. @@ -562,6 +623,7 @@ def delete_command(workspace_name, command_id): """ return server.delete_command(workspace_name, command_id) + def delete_object(workspace_name, object_signature, obj_id): """Given a workspace name, an object_signature as string and an object id. @@ -583,6 +645,7 @@ def delete_object(workspace_name, object_signature, obj_id): return appropiate_function(workspace_name, obj_id) + def delete_workspace(workspace_name): """Tries to delete the worskpace workspace_name and returns the json response. You should always try/except this function, at least catching @@ -590,14 +653,17 @@ def delete_workspace(workspace_name): """ return server.delete_workspace(workspace_name) + def get_workspaces_names(): """Return a list with all the workspace names available.""" return map(lambda ws: ws['name'], server.get_workspaces_names()) + def server_info(): """True if server is up, False otherwise.""" return server.server_info() + def test_server_url(url_to_test): """Return True if url_to_test/_api/info is accessible, False otherwise""" return server.test_server_url(url_to_test) @@ -642,6 +708,9 @@ def __init__(self, obj, workspace_name): def setID(self, id): self.id = id + def getID(self): + return self.id + @staticmethod def publicattrsrefs(): return {'Description': 'description', @@ -656,10 +725,14 @@ def propertyTieBreaker(self, key, prop1, prop2): is a default value returns the good one. If neither returns the default value. If conflicting returns a tuple with the values """ - if prop1 in self.defaultValues(): return prop2 - elif prop2 in self.defaultValues(): return prop1 - elif self.tieBreakable(key): return self.tieBreak(key, prop1, prop2) - else: return (prop1, prop2) + if prop1 in self.defaultValues(): + return prop2 + elif prop2 in self.defaultValues(): + return prop1 + elif self.tieBreakable(key): + return self.tieBreak(key, prop1, prop2) + else: + return (prop1, prop2) def tieBreakable(self, key): """ @@ -703,11 +776,16 @@ def updateResolved(self, update): def needs_merge(self, new_obj): return ModelObjectDiff(self, new_obj).existDiff() - def getOwner(self): return self.owner - def isOwned(self): return self.owned - def getName(self): return self.name - def getMetadata(self): return self._metadata - def getDescription(self): return self.description + def getOwner(self): + return self.owner + def isOwned(self): + return self.owned + def getName(self): + return self.name + def getMetadata(self): + return self._metadata + def getDescription(self): + return self.description class Host(ModelBase): @@ -741,11 +819,15 @@ def updateAttributes(self, name=None, description=None, os=None, owned=None): if owned is not None: self.owned = owned - def __str__(self): return "{0} ({1})".format(self.name, self.vuln_amount) - def getOS(self): return self.os - def getVulnAmount(self): return self.vuln_amount - def getID(self): return self.id - def getDefaultGateway(self): return self.default_gateway + def __str__(self): + return "{0} ({1})".format(self.name, self.vuln_amount) + def getOS(self): + return self.os + def getVulnAmount(self): + return self.vuln_amount + + def getDefaultGateway(self): + return self.default_gateway def getVulns(self): return get_all_vulns(self._workspace_name, hostid=self._server_id) # def getInterface(self, interface_couch_id): @@ -757,140 +839,6 @@ def getServices(self): return get_services(self._workspace_name, hostid=self._server_id) -# class Interface(ModelBase): -# """A simple Interface class. Should implement all the methods of the -# Interface object in Model.Host -# Any method here more than a couple of lines long probably represent -# a search the server is missing. -# """ -# class_signature = 'Interface' - -# def __init__(self, interface, workspace_name): -# ModelBase.__init__(self, interface, workspace_name) -# self.hostnames = interface.get('hostnames', []) - -# # NOTE. i don't know why this is like this -# # probably a remnant of the old faraday style classes -# try: -# self.ipv4 = interface['ipv4'] -# self.ipv6 = interface['ipv6'] -# except KeyError: -# self.ipv4 = {'address': interface['ipv4_address'], -# 'gateway': interface['ipv4_gateway'], -# 'mask': interface['ipv4_mask'], -# 'DNS': interface['ipv4_dns']} -# self.ipv6 = {'address': interface['ipv6_address'], -# 'gateway': interface['ipv6_gateway'], -# 'prefix': interface['ipv6_prefix'], -# 'DNS': interface['ipv6_dns']} -# self.mac = interface.get('mac') -# self.network_segment = interface.get('network_segment') -# self.ports = interface.get('ports') - -# self.amount_ports_opened = 0 -# self.amount_ports_closed = 0 -# self.amount_ports_filtered = 0 - -# def setID(self, parent_id): -# try: -# ipv4_address = self.ipv4_address -# ipv6_address = self.ipv6_address -# except AttributeError: -# ipv4_address = self.ipv4['address'] -# ipv6_address = self.ipv6['address'] - -# ModelBase.setID(self, parent_id, self.network_segment, ipv4_address, ipv6_address) - -# @staticmethod -# def publicattrsrefs(): -# publicattrs = dict(ModelBase.publicattrsrefs(), **{ -# 'MAC Address' : 'mac', -# 'IPV4 Settings' : 'ipv4', -# 'IPV6 Settings' : 'ipv6', -# 'Network Segment' : 'network_segment', -# 'Hostnames' : 'hostnames' -# }) -# return publicattrs - -# def tieBreakable(self, property_key): -# """ -# Return true if we can auto resolve this conflict. -# """ -# if property_key in ["hostnames"]: -# return True -# return False - -# def tieBreak(self, key, prop1, prop2): -# """ -# Return the 'choosen one' -# Return a tuple with prop1, prop2 if we cant resolve conflict. -# """ -# if key == "hostnames": -# prop1.extend(prop2) -# # Remove duplicated with set... -# return list(set(prop1)) - -# return (prop1, prop2) - -# def updateAttributes(self, name=None, description=None, hostnames=None, mac=None, ipv4=None, ipv6=None, -# network_segment=None, amount_ports_opened=None, amount_ports_closed=None, -# amount_ports_filtered=None, owned=None): - -# if name is not None: -# self.name = name -# if description is not None: -# self.description = description -# if hostnames is not None: -# self.hostnames = hostnames -# if mac is not None: -# self.mac = mac -# if ipv4 is not None: -# self.ipv4 = ipv4 -# if ipv6 is not None: -# self.ipv6 = ipv6 -# if network_segment is not None: -# self.network_segment = network_segment -# if amount_ports_opened is not None: -# self.setPortsOpened(amount_ports_opened) -# if amount_ports_closed is not None: -# self.setPortsClosed(amount_ports_closed) -# if amount_ports_filtered is not None: -# self.setPortsFiltered(amount_ports_filtered) -# if owned is not None: -# self.owned = owned - -# def setPortsOpened(self, ports_opened): -# self.amount_ports_opened = ports_opened - -# def setPortsClosed(self, ports_closed): -# self.amount_ports_closed = ports_closed - -# def setPortsFiltered(self, ports_filtered): -# self.amount_ports_filtered = ports_filtered - -# def __str__(self): return "{0}".format(self.name) -# def getID(self): return self.id -# def getHostnames(self): return self.hostnames -# def getIPv4(self): return self.ipv4 -# def getIPv6(self): return self.ipv6 -# def getIPv4Address(self): return self.ipv4['address'] -# def getIPv4Mask(self): return self.ipv4['mask'] -# def getIPv4Gateway(self): return self.ipv4['gateway'] -# def getIPv4DNS(self): return self.ipv4['DNS'] -# def getIPv6Address(self): return self.ipv6['address'] -# def getIPv6Gateway(self): return self.ipv6['gateway'] -# def getIPv6DNS(self): return self.ipv6['DNS'] -# def getMAC(self): return self.mac -# def getNetworkSegment(self): return self.network_segment - -# def getService(self, service_couch_id): -# return get_service(self._workspace_name, service_couch_id) -# def getAllServices(self): -# return get_services(self._workspace_name, interface=self._server_id) -# def getVulns(self): -# return get_all_vulns(self._workspace_name, interfaceid=self._server_id) - - class Service(ModelBase): """A simple Service class. Should implement all the methods of the Service object in Model.Host @@ -907,11 +855,6 @@ def __init__(self, service, workspace_name): self.status = service['status'] self.vuln_amount = int(service.get('vulns', 0)) - def setID(self, parent_id): - # TODO: str from list? ERROR MIGRATION NEEDED - ports = ':'.join(str(self.ports)) - ModelBase.setID(self, parent_id, self.protocol, ports) - @staticmethod def publicattrsrefs(): publicattrs = dict(ModelBase.publicattrsrefs(), **{ @@ -939,14 +882,20 @@ def updateAttributes(self, name=None, description=None, protocol=None, ports=Non if owned is not None: self.owned = owned - def __str__(self): return "{0} ({1})".format(self.name, self.vuln_amount) - def getID(self): return self.id - def getStatus(self): return self.status - def getPorts(self): return self.ports # this is a list of one element in faraday - def getVersion(self): return self.version - def getProtocol(self): return self.protocol - def isOwned(self): return self.owned - def getVulns(self): return get_all_vulns(self._workspace_name, serviceid=self._server_id) + def __str__(self): + return "{0} ({1})".format(self.name, self.vuln_amount) + def getStatus(self): + return self.status + def getPorts(self): + return self.ports # this is a list of one element in faraday + def getVersion(self): + return self.version + def getProtocol(self): + return self.protocol + def isOwned(self): + return self.owned + def getVulns(self): + return get_all_vulns(self._workspace_name, serviceid=self._server_id) class Vuln(ModelBase): @@ -970,9 +919,6 @@ def __init__(self, vuln, workspace_name): self.status = vuln.get('status', "opened") self.policyviolations = vuln.get('policyviolations', list()) - def setID(self, parent_id): - ModelBase.setID(self, parent_id, self.name, self.description) - @staticmethod def publicattrsrefs(): publicattrs = dict(ModelBase.publicattrsrefs(), **{ @@ -1058,15 +1004,22 @@ def updateAttributes(self, name=None, desc=None, data=None, if policyviolations is not None: self.policyviolations = policyviolations - def getID(self): return self.id - def getDesc(self): return self.desc - def getData(self): return self.data - def getSeverity(self): return self.severity - def getRefs(self): return self.refs - def getConfirmed(self): return self.confirmed - def getResolution(self): return self.resolution - def getStatus(self): return self.status - def getPolicyViolations(self): return self.policyviolations + def getDesc(self): + return self.desc + def getData(self): + return self.data + def getSeverity(self): + return self.severity + def getRefs(self): + return self.refs + def getConfirmed(self): + return self.confirmed + def getResolution(self): + return self.resolution + def getStatus(self): + return self.status + def getPolicyViolations(self): + return self.policyviolations def setStatus(self, status): self.status = status @@ -1100,9 +1053,6 @@ def __init__(self, vuln_web, workspace_name): self.parent = vuln_web.get('parent') self.policyviolations = vuln_web.get('policyviolations', list()) - def setID(self, parent_id): - ModelBase.setID(self, parent_id, self.name, self.website, self.path, self.description) - @staticmethod def publicattrsrefs(): publicattrs = dict(ModelBase.publicattrsrefs(), **{ @@ -1147,26 +1097,46 @@ def updateAttributes(self, name=None, desc=None, data=None, website=None, path=N if policyviolations is not None: self.policyviolations = policyviolations - def getDescription(self): return self.description - def getPath(self): return self.path - def getWebsite(self): return self.website - def getRequest(self): return self.request - def getResponse(self): return self.response - def getMethod(self): return self.method - def getPname(self): return self.pname - def getParams(self): return self.params - def getQuery(self): return self.query - def getResolution(self): return self.resolution - def getAttachments(self): return self.attachments - def getEaseOfResolution(self): return self.easeofresolution - def getHostnames(self): return self.hostnames - def getImpact(self): return self.impact - def getService(self): return self.service - def getStatus(self): return self.status - def getTags(self): return self.tags - def getTarget(self): return self.target - def getParent(self): return self.parent - def getPolicyViolations(self): return self.policyviolations + def getDescription(self): + return self.description + def getPath(self): + return self.path + def getWebsite(self): + return self.website + def getRequest(self): + return self.request + def getResponse(self): + return self.response + def getMethod(self): + return self.method + def getPname(self): + return self.pname + def getParams(self): + return self.params + def getQuery(self): + return self.query + def getResolution(self): + return self.resolution + def getAttachments(self): + return self.attachments + def getEaseOfResolution(self): + return self.easeofresolution + def getHostnames(self): + return self.hostnames + def getImpact(self): + return self.impact + def getService(self): + return self.service + def getStatus(self): + return self.status + def getTags(self): + return self.tags + def getTarget(self): + return self.target + def getParent(self): + return self.parent + def getPolicyViolations(self): + return self.policyviolations def tieBreakable(self, key): """ @@ -1226,16 +1196,12 @@ def __init__(self, note, workspace_name): ModelBase.__init__(self, note, workspace_name) self.text = note['text'] - def setID(self, parent_id): - ModelBase.setID(self, parent_id, self.name, self.text) - def updateAttributes(self, name=None, text=None): if name is not None: self.name = name if text is not None: self.text = text - def getID(self): return self.id def getDescription(self): return self.description def getText(self): return self.text @@ -1251,18 +1217,16 @@ def __init__(self, credential, workspace_name): self.password = credential['password'] - def setID(self, parent_id): - ModelBase.setID(self, parent_id, self.name, self.username, self.password) - def updateAttributes(self, username=None, password=None): if username is not None: - self.username =username + self.username = username if password is not None: self.password = password - def getID(self): return self.id - def getUsername(self): return self.username - def getPassword(self): return self.password + def getUsername(self): + return self.username + def getPassword(self): + return self.password class Command: class_signature = 'CommandRunInformation' @@ -1278,15 +1242,22 @@ def __init__(self, command, workspace_name): self.user = command['user'] self.workspace = command['workspace'] - def getID(self): return self.id - def getCommand(self): return self.command - def getDuration(self): return self.duration - def getHostname(self): return self.hostname - def getIP(self): return self.ip - def getItime(self): return self.itime - def getParams(self): return self.params - def getUser(self): return self.user - def getWorkspace(self): return self.workspace + def getCommand(self): + return self.command + def getDuration(self): + return self.duration + def getHostname(self): + return self.hostname + def getIP(self): + return self.ip + def getItime(self): + return self.itime + def getParams(self): + return self.params + def getUser(self): + return self.user + def getWorkspace(self): + return self.workspace class _Workspace: class_signature = 'Workspace' @@ -1299,12 +1270,16 @@ def __init__(self, workspace, workspace_name): self.start_date = workspace['sdate'] self.finish_date = workspace['fdate'] - def getID(self): return self._id - def getName(self): return self.name - def getDescription(self): return self.description - def getCustomer(self): return self.customer - def getStartDate(self): return self.start_date - def getFinishDate(self): return self.finish_date + def getName(self): + return self.name + def getDescription(self): + return self.description + def getCustomer(self): + return self.customer + def getStartDate(self): + return self.start_date + def getFinishDate(self): + return self.finish_date class MetadataUpdateActions(object): @@ -1356,18 +1331,4 @@ def __getUpdateAction(self): if controller_funcallnames: return "ModelControler." + " ModelControler.".join(controller_funcallnames) - return "No model controller call" - -# NOTE: uncomment for test -# class SillyHost(): -# def __init__(self) : -# import random; self.id = random.randint(0, 1000) -# self.os = "Windows" -# def getID(self): return self.id -# def getOS(self): return self.os -# def getDefaultGateway(self): return '192.168.1.1' -# def getDescription(self): return "a description" -# def getName(self): return "my name" -# def isOwned(self): return False -# def getOwner(self): return False -# def getMetadata(self): return {'stuff': 'gives other stuff'} + return "No model controller call" \ No newline at end of file From 5380394c56a06f8584e0c08d8bea2f1c937f2ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 1 Dec 2017 14:21:03 -0300 Subject: [PATCH 0591/1506] Fix vuln template references Allow creating vuln templates with no references, and improve behavior when many spaces are used --- server/api/modules/vulnerability_template.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/server/api/modules/vulnerability_template.py b/server/api/modules/vulnerability_template.py index e560d83619b..4d9e063740d 100644 --- a/server/api/modules/vulnerability_template.py +++ b/server/api/modules/vulnerability_template.py @@ -42,12 +42,16 @@ def get_references(self, obj): return ', '.join(map(lambda ref_tmpl: ref_tmpl.name, obj.reference_template_instances)) def load_references(self, value): - if not isinstance(value, (unicode, str)): - raise ValidationError('references must be a string') - if len(value) == 0: - # Required because "".split(",") == [""] - return [] - references = value.split(',') + if isinstance(value, list): + references = value + elif isinstance(value, (unicode, str)): + if len(value) == 0: + # Required because "".split(",") == [""] + return [] + references = [ref.strip() for ref in value.split(',')] + else: + raise ValidationError('references must be a either a string ' + 'or a list') if any(len(ref) == 0 for ref in references): raise ValidationError('Empty name detected in reference') return references From b7a753a773edd7fa674e738fd3f5231a2b88d1d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 1 Dec 2017 15:17:44 -0300 Subject: [PATCH 0592/1506] Remove no longer useful vuln template tests --- test_cases/test_api_vulnerability_template.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test_cases/test_api_vulnerability_template.py b/test_cases/test_api_vulnerability_template.py index 3677331fd2b..ff1defe332f 100644 --- a/test_cases/test_api_vulnerability_template.py +++ b/test_cases/test_api_vulnerability_template.py @@ -85,9 +85,7 @@ def test_update_vulnerability_template(self, session, test_client): ',', ',,', 'a,', - ['a', 'b', 'c'], ['a', 'b', ''], - [], {"a": 1}, {} ]) From 73ae873d53367f20e07d2490cb18e232c7edc692 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 1 Dec 2017 20:34:04 -0300 Subject: [PATCH 0593/1506] Add support for metadata create time vulns in importer CSV --- bin/import_csv.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/bin/import_csv.py b/bin/import_csv.py index 8719b19bde2..4242b53b614 100644 --- a/bin/import_csv.py +++ b/bin/import_csv.py @@ -7,6 +7,8 @@ """ import csv +from time import mktime +from datetime import datetime from persistence.server import models WORKSPACE = "" @@ -179,6 +181,16 @@ def parse_vulnerability(register): if obj is None: return None vulnerability = models.Vuln(obj, WORKSPACE) + + try: + + date = register.get("vulnerability_metadata_create_time") + if date is not None: + datetime_object = datetime.strptime(date, "%d/%m/%Y") + vulnerability._metadata.create_time = mktime(datetime_object.timetuple()) + except Exception: + print "Invalid date", vulnerability.name + return vulnerability @@ -208,6 +220,15 @@ def parse_vulnerability_web(register): if obj is None: return None vulnerability_web = models.VulnWeb(obj, WORKSPACE) + + try: + date = register.get("vulnerability_web_metadata_create_time") + if date is not None: + datetime_object = datetime.strptime(date, "%d/%m/%Y") + vulnerability_web._metadata.create_time = mktime(datetime_object.timetuple()) + except Exception: + print "Invalid date", vulnerability_web.name + return vulnerability_web From 1b1663e052e2a5737e4e3ad93405d57e40a740b8 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 4 Dec 2017 14:43:47 -0300 Subject: [PATCH 0594/1506] Remove deprecated fields sdate, fdate in Workspace --- model/workspace.py | 12 ++++++------ persistence/server/models.py | 6 +++--- persistence/server/server.py | 2 -- .../workspaces/controllers/workspaces.js | 6 ------ .../scripts/workspaces/providers/workspaces.js | 17 ++++------------- .../workspaces/controllers/workspace_test.js | 10 ---------- .../workspaces/providers/workspaces_test.js | 2 -- 7 files changed, 13 insertions(+), 42 deletions(-) diff --git a/model/workspace.py b/model/workspace.py index 79476a93432..f207644785f 100644 --- a/model/workspace.py +++ b/model/workspace.py @@ -60,14 +60,14 @@ def setCustomer(self, customer): def getStartDate(self): return self.start_date - def setStartDate(self, sdate): - self.start_date = sdate + def setStartDate(self, start_date): + self.start_date = start_date - def getFinishDate(self): - return self.finish_date + def getEndDate(self): + return self.end_date - def setFinishDate(self, fdate): - self.finish_date = fdate + def setEndDate(self, edate): + self.end_date = edate def isActive(self): return self.name == self._workspace_manager.getActiveWorkspace().name diff --git a/persistence/server/models.py b/persistence/server/models.py index 3033b8114ae..9b2a41fc43e 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -1305,15 +1305,15 @@ def __init__(self, workspace, workspace_name): self.name = workspace['name'] self.description = workspace['description'] self.customer = workspace['customer'] - self.start_date = workspace['sdate'] - self.finish_date = workspace['fdate'] + self.start_date = workspace['start_date'] + self.end_date = workspace['end_date'] def getID(self): return self._id def getName(self): return self.name def getDescription(self): return self.description def getCustomer(self): return self.customer def getStartDate(self): return self.start_date - def getFinishDate(self): return self.finish_date + def getEndDate(self): return self.end_date class MetadataUpdateActions(object): diff --git a/persistence/server/server.py b/persistence/server/server.py index 46b5d2b2cf1..4d6cadf86cb 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -1422,8 +1422,6 @@ def create_workspace(workspace_name, description, start_date, finish_date, name=workspace_name, description=description, customer=customer, - sdate=start_date, - fdate=finish_date, duration=duration, type="Workspace") diff --git a/server/www/scripts/workspaces/controllers/workspaces.js b/server/www/scripts/workspaces/controllers/workspaces.js index a43799e4a6b..9d9085cbfa3 100644 --- a/server/www/scripts/workspaces/controllers/workspaces.js +++ b/server/www/scripts/workspaces/controllers/workspaces.js @@ -54,9 +54,6 @@ angular.module('faradayApp') }; $scope.onSuccessGet = function(workspace){ - if (workspace.sdate !== undefined) { - if(workspace.sdate.toString().indexOf(".") != -1) workspace.sdate = workspace.sdate * 1000; - } workspace.selected = false; $scope.workspaces.push(workspace); }; @@ -141,7 +138,6 @@ angular.module('faradayApp') "duration": duration, "name": ws.name, "scope": ws.scope, - "sdate": ws.sdate, "type": ws.type }; workspacesFact.update(workspace).then(function(workspace) { @@ -262,9 +258,7 @@ angular.module('faradayApp') workspace = { "_id": wname, "customer": "", - "sdate": (new Date).getTime(), "name": wname, - "fdate": undefined, "type": "Workspace", "children": [ ], diff --git a/server/www/scripts/workspaces/providers/workspaces.js b/server/www/scripts/workspaces/providers/workspaces.js index d0b0fcfe8cd..17e77585c06 100644 --- a/server/www/scripts/workspaces/providers/workspaces.js +++ b/server/www/scripts/workspaces/providers/workspaces.js @@ -45,19 +45,10 @@ angular.module('faradayApp') workspacesFact.getDuration = function(workspace_name) { var deferred = $q.defer(); ServerAPI.getWorkspace(workspace_name).then(function(workspace) { - var ws = workspace.data; - var dur = {}; - - if(ws.hasOwnProperty('duration')) { - if(ws.duration.hasOwnProperty('start') && ws.duration.hasOwnProperty('end')) { - dur.start = ws.duration.start; - dur.end = ws.duration.end; - } - } else if(ws.hasOwnProperty('sdate') && ws.hasOwnProperty('fdate')) { - dur.start = ws.sdate; - dur.end = ws.fdate; - } - deferred.resolve(dur); + deferred.resolve({ + "start": workspace.data.duration.start, + "end": workspace.data.duration.end + }); }); return deferred.promise; }; diff --git a/tests_web/faradayApp/components/workspaces/controllers/workspace_test.js b/tests_web/faradayApp/components/workspaces/controllers/workspace_test.js index 00a4090b8a8..c4b4c440f28 100644 --- a/tests_web/faradayApp/components/workspaces/controllers/workspace_test.js +++ b/tests_web/faradayApp/components/workspaces/controllers/workspace_test.js @@ -18,7 +18,6 @@ describe('workspacesCtrl', function() { "end": 1410832741.48194 }, "name": "ws1", - "sdate": 1410832741.48194, "scope": "", "selected": true, "type": "Workspace" @@ -34,7 +33,6 @@ describe('workspacesCtrl', function() { "endDate": 141083274148194 }, "name": "ws2", - "sdate": 1410832741.48194, "scope": "", "selected": true, "type": "Workspace", @@ -50,7 +48,6 @@ describe('workspacesCtrl', function() { "end": 1410832741.48194 }, "name": "ws3", - "sdate": 1410832741.48194, "scope": "", "selected": true, "type": "Workspace" @@ -99,7 +96,6 @@ describe('workspacesCtrl', function() { "end": 141083274148194 }, "name": "ws2", - "sdate": 1410832741.48194, "scope": "Nuevo Scope", "selected": true, "type": "Workspace", @@ -187,7 +183,6 @@ describe('workspacesCtrl', function() { expect(workspace.duration.start).toBeDefined(); expect(workspace.duration.end).toBeDefined(); expect(workspace.name).toBeDefined(); - expect(workspace.sdate).toBeDefined(); expect(workspace.scope).toBeDefined(); expect(workspace.selected).not.toBeDefined(); expect(workspace.type).toBeDefined(); @@ -204,7 +199,6 @@ describe('workspacesCtrl', function() { "end": 141083274148194 }, "name": "ws2", - "sdate": 1410832741.48194, "scope": "Nuevo Scope", "selected": true, "type": "Workspace", @@ -258,11 +252,7 @@ describe('workspacesCtrl', function() { expect(workspace._id).toBeDefined(); expect(workspace._rev).not.toBeDefined(); expect(workspace.customer).toBeDefined(); - expect(workspace.sdate).toBeDefined(); expect(workspace.name).toBeDefined(); - // find out if this variable is being used - // is defined as undefined - expect(workspace.fdate).toBeUndefined(); expect(workspace.type).toBeDefined(); expect(workspace.children).toBeDefined(); diff --git a/tests_web/faradayApp/components/workspaces/providers/workspaces_test.js b/tests_web/faradayApp/components/workspaces/providers/workspaces_test.js index fc9ee2c33b4..25ba1ab318b 100644 --- a/tests_web/faradayApp/components/workspaces/providers/workspaces_test.js +++ b/tests_web/faradayApp/components/workspaces/providers/workspaces_test.js @@ -46,9 +46,7 @@ describe('workspacesFact', function() { var workspace = { "_id": "test_workspace", "customer": "", - "sdate": 1415901244.040532, "name": "test_workspace", - "fdate": 1415901244.040532, "type": "Workspace", "children": [ ], From 5adad842eff081774087e452b11530fa28100a04 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 4 Dec 2017 14:44:44 -0300 Subject: [PATCH 0595/1506] Refactor Workspaces test cases Moved Workspace test cases from test_api_non_workspaced_various into its own proper file. --- test_cases/test_api_non_workspaced_various.py | 47 ---------------- test_cases/test_api_workspace.py | 53 +++++++++++++++++++ 2 files changed, 53 insertions(+), 47 deletions(-) diff --git a/test_cases/test_api_non_workspaced_various.py b/test_cases/test_api_non_workspaced_various.py index 192452769ee..1e111d315ff 100644 --- a/test_cases/test_api_non_workspaced_various.py +++ b/test_cases/test_api_non_workspaced_various.py @@ -6,10 +6,8 @@ from test_api_non_workspaced_base import ReadWriteAPITests, API_PREFIX from server.models import ( License, - Workspace, ) from server.api.modules.licenses import LicenseView -from server.api.modules.workspaces import WorkspaceView class LicenseEnvelopedView(LicenseView): """A custom view to test that enveloping on generic views work ok""" @@ -34,48 +32,3 @@ def test_envelope_list(self, test_client, app): assert new_res.status_code == 200 assert new_res.json == {"object_list": original_res.json} - - -class TestWorkspaceAPI(ReadWriteAPITests): - model = Workspace - factory = factories.WorkspaceFactory - api_endpoint = 'ws' - lookup_field = 'name' - - def test_host_count(self, host_factory, test_client, session): - host_factory.create(workspace=self.first_object) - session.commit() - res = test_client.get(self.url(self.first_object)) - assert res.status_code == 200 - assert res.json['stats']['hosts'] == 1 - - @pytest.mark.parametrize('querystring', [ - '', - '?confirmed=0', - '?confirmed=false' - ]) - def test_vuln_count(self, vulnerability_factory, test_client, session, - querystring): - vulnerability_factory.create_batch(8, workspace=self.first_object, - confirmed=False) - vulnerability_factory.create_batch(5, workspace=self.first_object, - confirmed=True) - session.commit() - res = test_client.get(self.url(self.first_object) + querystring) - assert res.status_code == 200 - assert res.json['stats']['total_vulns'] == 13 - - @pytest.mark.parametrize('querystring', [ - '?confirmed=1', - '?confirmed=true' - ]) - def test_vuln_count_confirmed(self, vulnerability_factory, test_client, - session, querystring): - vulnerability_factory.create_batch(8, workspace=self.first_object, - confirmed=False) - vulnerability_factory.create_batch(5, workspace=self.first_object, - confirmed=True) - session.commit() - res = test_client.get(self.url(self.first_object) + querystring) - assert res.status_code == 200 - assert res.json['stats']['total_vulns'] == 5 diff --git a/test_cases/test_api_workspace.py b/test_cases/test_api_workspace.py index e69de29bb2d..28edddbe4ed 100644 --- a/test_cases/test_api_workspace.py +++ b/test_cases/test_api_workspace.py @@ -0,0 +1,53 @@ +import pytest + +from server.models import Workspace +from server.api.modules.workspaces import WorkspaceView +from test_cases.test_api_non_workspaced_base import ReadWriteAPITests, API_PREFIX +from test_cases import factories + +class TestWorkspaceAPI(ReadWriteAPITests): + model = Workspace + factory = factories.WorkspaceFactory + api_endpoint = 'ws' + lookup_field = 'name' + view_class = WorkspaceView + + def test_host_count(self, host_factory, test_client, session): + host_factory.create(workspace=self.first_object) + session.commit() + res = test_client.get(self.url(self.first_object)) + assert res.status_code == 200 + assert res.json['stats']['hosts'] == 1 + + @pytest.mark.parametrize('querystring', [ + '', + '?confirmed=0', + '?confirmed=false' + ]) + + def test_vuln_count(self, vulnerability_factory, test_client, session, + querystring): + vulnerability_factory.create_batch(8, workspace=self.first_object, + confirmed=False) + vulnerability_factory.create_batch(5, workspace=self.first_object, + confirmed=True) + session.commit() + res = test_client.get(self.url(self.first_object) + querystring) + assert res.status_code == 200 + assert res.json['stats']['total_vulns'] == 13 + + @pytest.mark.parametrize('querystring', [ + '?confirmed=1', + '?confirmed=true' + ]) + + def test_vuln_count_confirmed(self, vulnerability_factory, test_client, + session, querystring): + vulnerability_factory.create_batch(8, workspace=self.first_object, + confirmed=False) + vulnerability_factory.create_batch(5, workspace=self.first_object, + confirmed=True) + session.commit() + res = test_client.get(self.url(self.first_object) + querystring) + assert res.status_code == 200 + assert res.json['stats']['total_vulns'] == 5 \ No newline at end of file From 00d32a43a8d67861636d71b05f6bce49719cea68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 4 Dec 2017 15:42:45 -0300 Subject: [PATCH 0596/1506] Link to credentials in the dashboard Put a link to view the credentials in the workspace summarized report of the dashboard --- RELEASE.md | 1 + server/www/scripts/dashboard/partials/summarized.html | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index 6196fca70f0..d3b273b41bb 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -15,6 +15,7 @@ TBA: * Interface object was removed * Note object was removed and replaced with Comment * Communication object was removed and replaced with Comment +* Show credentials count in summarized report on the dashboard November 17, 2017: --- diff --git a/server/www/scripts/dashboard/partials/summarized.html b/server/www/scripts/dashboard/partials/summarized.html index a12bbac0b9e..f3824898d7e 100644 --- a/server/www/scripts/dashboard/partials/summarized.html +++ b/server/www/scripts/dashboard/partials/summarized.html @@ -26,6 +26,7 @@

    Workspace summarized report
    {{obj.value}}
    {{obj.value}}
    {{obj.value}}
    +
    {{obj.value}}
    {{obj.value}}

    @@ -37,4 +38,4 @@

    Workspace summarized report

    -
    \ No newline at end of file +
    From 4ecc74731cccfe700610b6b438691d7d0d470d83 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Mon, 4 Dec 2017 16:06:38 -0300 Subject: [PATCH 0597/1506] Add support for host create time --- bin/import_csv.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bin/import_csv.py b/bin/import_csv.py index 4242b53b614..9bc65babc8d 100644 --- a/bin/import_csv.py +++ b/bin/import_csv.py @@ -115,6 +115,16 @@ def parse_host(register): if obj is None: return None host = models.Host(obj, WORKSPACE) + + try: + + date = register.get("host_metadata_create_time") + if date is not None: + datetime_object = datetime.strptime(date, "%d/%m/%Y") + host._metadata.create_time = mktime(datetime_object.timetuple()) + except Exception: + print "Invalid date", host.name + return host From 76bf33d4f1fbce226e0b1082d1dccfd5ce690c58 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 4 Dec 2017 17:27:04 -0300 Subject: [PATCH 0598/1506] Fix param names In the workspace object, duration dict has two keys named start_date and end_date now. --- .../workspaces/controllers/workspaces.js | 22 +++++++++---------- .../www/scripts/workspaces/partials/list.html | 4 ++-- .../workspaces/partials/modalEdit.html | 4 ++-- .../scripts/workspaces/partials/modalNew.html | 4 ++-- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/server/www/scripts/workspaces/controllers/workspaces.js b/server/www/scripts/workspaces/controllers/workspaces.js index 9d9085cbfa3..ffd66edb8df 100644 --- a/server/www/scripts/workspaces/controllers/workspaces.js +++ b/server/www/scripts/workspaces/controllers/workspaces.js @@ -82,8 +82,8 @@ angular.module('faradayApp') $scope.workspaces[i].description = workspace.description; if ($scope.workspaces[i].duration === undefined) $scope.workspaces[i].duration = {}; - $scope.workspaces[i].duration.start = workspace.duration.start; - $scope.workspaces[i].duration.end = workspace.duration.end; + $scope.workspaces[i].duration.start_date = workspace.duration.start_date; + $scope.workspaces[i].duration.end_date = workspace.duration.end_date; $scope.workspaces[i].scope = workspace.scope; break; } @@ -118,15 +118,15 @@ angular.module('faradayApp') }; $scope.update = function(ws){ - if(typeof(ws.duration.start) == "number") { - start = ws.duration.start; - } else if(ws.duration.start) { - start = ws.duration.start.getTime(); + if(typeof(ws.duration.start_date) == "number") { + start = ws.duration.start_date; + } else if(ws.duration.start_date) { + start = ws.duration.start_date.getTime(); } else {start = "";} - if(typeof(ws.duration.end) == "number") { - end = ws.duration.end; - } else if(ws.duration.end) { - end = ws.duration.end.getTime(); + if(typeof(ws.duration.end_date) == "number") { + end = ws.duration.end_date; + } else if(ws.duration.end_date) { + end = ws.duration.end_date.getTime(); } else {end = "";} duration = {'start': start, 'end': end}; workspace = { @@ -262,7 +262,7 @@ angular.module('faradayApp') "type": "Workspace", "children": [ ], - "duration": {"start": start, "end": end}, + "duration": {"start_date": start, "end_date": end}, "scope": scope, "description": wdesc }; diff --git a/server/www/scripts/workspaces/partials/list.html b/server/www/scripts/workspaces/partials/list.html index 9a1cd87e12a..35507b3d754 100644 --- a/server/www/scripts/workspaces/partials/list.html +++ b/server/www/scripts/workspaces/partials/list.html @@ -60,8 +60,8 @@

    {{ws.name}} - - + + {{objects[ws.name]['total_vulns']}} {{objects[ws.name]['hosts']}} diff --git a/server/www/scripts/workspaces/partials/modalEdit.html b/server/www/scripts/workspaces/partials/modalEdit.html index 9778bf92baa..c43ffe8de75 100644 --- a/server/www/scripts/workspaces/partials/modalEdit.html +++ b/server/www/scripts/workspaces/partials/modalEdit.html @@ -25,7 +25,7 @@

    Edit Workspace: {{workspace.name}}

    Start Date

    - + @@ -35,7 +35,7 @@

    Start Date
    End Date

    - + diff --git a/server/www/scripts/workspaces/partials/modalNew.html b/server/www/scripts/workspaces/partials/modalNew.html index 99a14e0b17d..8196dd94adc 100644 --- a/server/www/scripts/workspaces/partials/modalNew.html +++ b/server/www/scripts/workspaces/partials/modalNew.html @@ -44,7 +44,7 @@

    New Workspace

    - + @@ -53,7 +53,7 @@

    New Workspace

    - + From 40fbb00e6f6d25c7d2042349645ce7ea51686d20 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Mon, 4 Dec 2017 18:12:40 -0300 Subject: [PATCH 0599/1506] Change date to american format --- bin/import_csv.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/import_csv.py b/bin/import_csv.py index 9bc65babc8d..c4965cf6987 100644 --- a/bin/import_csv.py +++ b/bin/import_csv.py @@ -120,7 +120,7 @@ def parse_host(register): date = register.get("host_metadata_create_time") if date is not None: - datetime_object = datetime.strptime(date, "%d/%m/%Y") + datetime_object = datetime.strptime(date, "%m/%d/%Y") host._metadata.create_time = mktime(datetime_object.timetuple()) except Exception: print "Invalid date", host.name @@ -196,7 +196,7 @@ def parse_vulnerability(register): date = register.get("vulnerability_metadata_create_time") if date is not None: - datetime_object = datetime.strptime(date, "%d/%m/%Y") + datetime_object = datetime.strptime(date, "%m/%d/%Y") vulnerability._metadata.create_time = mktime(datetime_object.timetuple()) except Exception: print "Invalid date", vulnerability.name @@ -234,7 +234,7 @@ def parse_vulnerability_web(register): try: date = register.get("vulnerability_web_metadata_create_time") if date is not None: - datetime_object = datetime.strptime(date, "%d/%m/%Y") + datetime_object = datetime.strptime(date, "%m/%d/%Y") vulnerability_web._metadata.create_time = mktime(datetime_object.timetuple()) except Exception: print "Invalid date", vulnerability_web.name From 99791222d2700fd8bba1cf5a7461e73a6e3421fd Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 4 Dec 2017 18:34:48 -0300 Subject: [PATCH 0600/1506] More delete programming. Removing interface code --- model/controller.py | 113 -------------------------------------------- 1 file changed, 113 deletions(-) diff --git a/model/controller.py b/model/controller.py index 64ab756a189..0e80b882d5c 100644 --- a/model/controller.py +++ b/model/controller.py @@ -46,7 +46,6 @@ class modelactions: ADDNOTEVULN = 2030 DELNOTEVULN = 2031 EDITHOST = 2032 - EDITINTERFACE = 2033 EDITSERVICE = 2035 ADDCREDSRV = 2036 DELCREDSRV = 2037 @@ -79,7 +78,6 @@ class modelactions: ADDNOTENOTE: "ADDNOTENOTE", DELNOTENOTE: "DELNOTENOTE", EDITHOST: "EDITHOST", - EDITINTERFACE: "EDITINTERFACE", ADDCREDSRV: "ADDCREDSRV", DELCREDSRV: "DELCREDSRV", ADDVULNWEBSRV: "ADDVULNSWEBRV", @@ -198,7 +196,6 @@ def _setupActionDispatcher(self): modelactions.ADDHOST: self.__add, modelactions.DELHOST: self.__del, modelactions.EDITHOST: self.__edit, - modelactions.EDITINTERFACE: self.__edit, modelactions.EDITSERVICE: self.__edit, # Vulnerability modelactions.ADDVULNHOST: self.__add, @@ -487,87 +484,6 @@ def editHostSYNC(self, host, name, description, os, owned): self._processAction(modelactions.EDITHOST, [ host, name, description, os, owned], sync=True) - def addInterfaceASYNC(self, hostid, interface, update=False): - """ - ASYNC API - Adds an action to the ModelController actions queue indicating a - new interface must be added to a specific host - """ - self.__addPendingAction(modelactions.ADDINTERFACE, interface, hostid) - - def addInterfaceSYNC(self, hostId, interface, update=False): - """ - SYNC API - Adds interface directly to the model - """ - self._processAction(modelactions.ADDINTERFACE, [ - interface, hostId], sync=True) - - def delInterfaceASYNC(self, hostId, interfaceId): - """ - ASYNC API - Adds an action to the ModelController actions queue indicating a - particular host must be removed from the model - """ - self.__addPendingAction(modelactions.DELINTERFACE, interfaceId, hostId) - - def delInterfaceSYNC(self, host, interface_id, *args): - """ - SYNC API - Deletes an interface from model - """ - self._processAction(modelactions.DELINTERFACE, - [interface_id], sync=True) - - def editInterfaceSYNC(self, interface, name, description, hostnames, - mac, ipv4, ipv6, network_segment, - amount_ports_opened, amount_ports_closed, - amount_ports_filtered, owned): - """ - SYNC API - Modifies an interface from model - """ - self._processAction(modelactions.EDITINTERFACE, - [interface, name, description, hostnames, - mac, ipv4, ipv6, network_segment, - amount_ports_opened, amount_ports_closed, - amount_ports_filtered, owned], sync=True) - - def addServiceToInterfaceASYNC(self, host, interfaceId, newService): - """ - ASYNC API - Adds an action to the ModelController actions queue indicating a - new services must be added to a specific host in a specific interface - """ - self.__addPendingAction( - modelactions.ADDSERVICEINT, newService, interfaceId) - - def addServiceToInterfaceSYNC(self, host_id, interface_id, newService): - """ - SYNC API - Adds a service to a specific host in a specific interface - directly to the model - """ - self._processAction(modelactions.ADDSERVICEINT, [ - newService, interface_id], sync=True) - - def delServiceFromInterfaceASYNC(self, host, interfaceId, serviceId): - """ - ASYNC API - Adds an action to the ModelController actions queue indicating a - particular service in a host and interface must be removed from the - model Interface parameter can be "ALL" - """ - self.__addPendingAction( - modelactions.DELSERVICEINT, serviceId, interfaceId) - - def delServiceFromInterfaceSYNC(self, host, interfaceId, serviceId): - """ - SYNC API - Delete a service in a host and interface from the model - """ - self._processAction(modelactions.DELSERVICEINT, [serviceId], sync=True) - def editServiceSYNC(self, service, name, description, protocol, ports, status, version, owned): """ SYNC API @@ -615,13 +531,6 @@ def _pluginEnd(self, name): self.active_plugins_count_lock.release() return True - def addVulnToInterfaceASYNC(self, host, intId, newVuln): - self.__addPendingAction(modelactions.ADDVULNINT, newVuln, intId) - - def addVulnToInterfaceSYNC(self, host, intId, newVuln): - self._processAction(modelactions.ADDVULNINT, [ - newVuln, intId], sync=True) - def addVulnToHostASYNC(self, hostId, newVuln): self.__addPendingAction(modelactions.ADDVULNHOST, newVuln, hostId) @@ -647,14 +556,6 @@ def addVulnWebToServiceSYNC(self, host, srvId, newVuln): self._processAction(modelactions.ADDVULNWEBSRV, [newVuln, srvId], sync=True) - def delVulnFromInterfaceASYNC(self, hostname, intname, vuln): - self.__addPendingAction(modelactions.DELVULNINT, - hostname, intname, vuln) - - def delVulnFromInterfaceSYNC(self, hostname, intname, vuln): - self._processAction(modelactions.DELVULNINT, [ - hostname, intname, vuln], sync=True) - def delVulnFromHostASYNC(self, hostId, vulnId): self.__addPendingAction(modelactions.DELVULNHOST, vulnId) @@ -693,14 +594,6 @@ def editVulnWebASYNC(self, vuln, name, desc, website, path, refs, severity, resolution, request, response, method, pname, params, query, category) - # Note - def addNoteToInterfaceASYNC(self, host, intId, newNote): - self.__addPendingAction(modelactions.ADDNOTEINT, newNote, intId) - - def addNoteToInterfaceSYNC(self, host, intId, newNote): - self._processAction(modelactions.ADDNOTEINT, [ - newNote, intId], sync=True) - def addNoteToHostASYNC(self, hostId, newNote): self.__addPendingAction(modelactions.ADDNOTEHOST, newNote, hostId) @@ -726,12 +619,6 @@ def addNoteSYNC(self, model_object, newNote): self._processAction(modelactions.ADDNOTE, [ newNote, model_object], sync=True) - def delNoteFromInterfaceASYNC(self, hostname, intname, noteId): - self.__addPendingAction(modelactions.DELNOTEINT, noteId) - - def delNoteFromInterfaceSYNC(self, hostname, intname, noteId): - self._processAction(modelactions.DELNOTEINT, [noteId], sync=True) - def delNoteFromHostASYNC(self, hostId, noteId): self.__addPendingAction(modelactions.DELNOTEHOST, noteId) From b45ad4750fde514cd6caf8286ecb6de212e54db5 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 4 Dec 2017 18:35:58 -0300 Subject: [PATCH 0601/1506] PEP8 fixes --- persistence/server/models.py | 100 ++++++++++++++++++++++++++--------- 1 file changed, 75 insertions(+), 25 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index 4bbd66ae935..be56da390be 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -821,13 +821,16 @@ def updateAttributes(self, name=None, description=None, os=None, owned=None): def __str__(self): return "{0} ({1})".format(self.name, self.vuln_amount) + def getOS(self): return self.os + def getVulnAmount(self): return self.vuln_amount def getDefaultGateway(self): return self.default_gateway + def getVulns(self): return get_all_vulns(self._workspace_name, hostid=self._server_id) # def getInterface(self, interface_couch_id): @@ -850,7 +853,7 @@ class Service(ModelBase): def __init__(self, service, workspace_name): ModelBase.__init__(self, service, workspace_name) self.protocol = service['protocol'] - self.ports = [int(port) for port in service['ports']] + self.ports = [int(port) for port in service['ports']] self.version = service['version'] self.status = service['status'] self.vuln_amount = int(service.get('vulns', 0)) @@ -858,10 +861,10 @@ def __init__(self, service, workspace_name): @staticmethod def publicattrsrefs(): publicattrs = dict(ModelBase.publicattrsrefs(), **{ - 'Ports' : 'ports', - 'Protocol' : 'protocol', - 'Status' : 'status', - 'Version' : 'version' + 'Ports': 'ports', + 'Protocol': 'protocol', + 'Status': 'status', + 'Version': 'version' }) return publicattrs @@ -884,16 +887,22 @@ def updateAttributes(self, name=None, description=None, protocol=None, ports=Non def __str__(self): return "{0} ({1})".format(self.name, self.vuln_amount) + def getStatus(self): return self.status + def getPorts(self): return self.ports # this is a list of one element in faraday + def getVersion(self): return self.version + def getProtocol(self): return self.protocol + def isOwned(self): return self.owned + def getVulns(self): return get_all_vulns(self._workspace_name, serviceid=self._server_id) @@ -922,9 +931,9 @@ def __init__(self, vuln, workspace_name): @staticmethod def publicattrsrefs(): publicattrs = dict(ModelBase.publicattrsrefs(), **{ - 'Data' : 'data', - 'Severity' : 'severity', - 'Refs' : 'refs', + 'Data': 'data', + 'Severity': 'severity', + 'Refs': 'refs', 'Resolution': 'resolution', 'Status': 'status' }) @@ -1006,18 +1015,25 @@ def updateAttributes(self, name=None, desc=None, data=None, def getDesc(self): return self.desc + def getData(self): return self.data + def getSeverity(self): return self.severity + def getRefs(self): return self.refs + def getConfirmed(self): return self.confirmed + def getResolution(self): return self.resolution + def getStatus(self): return self.status + def getPolicyViolations(self): return self.policyviolations @@ -1050,23 +1066,22 @@ def __init__(self, vuln_web, workspace_name): self.service = vuln_web.get('service') self.tags = vuln_web.get('tags', list()) self.target = vuln_web.get('target') - self.parent = vuln_web.get('parent') self.policyviolations = vuln_web.get('policyviolations', list()) @staticmethod def publicattrsrefs(): publicattrs = dict(ModelBase.publicattrsrefs(), **{ - 'Data' : 'data', - 'Severity' : 'severity', - 'Refs' : 'refs', - 'Path' : 'path', - 'Website' : 'website', - 'Request' : 'request', - 'Response' : 'response', - 'Method' : 'method', - 'Pname' : 'pname', - 'Params' : 'params', - 'Query' : 'query', + 'Data': 'data', + 'Severity': 'severity', + 'Refs': 'refs', + 'Path': 'path', + 'Website': 'website', + 'Request': 'request', + 'Response': 'response', + 'Method': 'method', + 'Pname': 'pname', + 'Params': 'params', + 'Query': 'query', 'Status': 'status'}) return publicattrs @@ -1099,38 +1114,55 @@ def updateAttributes(self, name=None, desc=None, data=None, website=None, path=N def getDescription(self): return self.description + def getPath(self): return self.path + def getWebsite(self): return self.website + def getRequest(self): return self.request + def getResponse(self): return self.response + def getMethod(self): return self.method + def getPname(self): return self.pname + def getParams(self): return self.params + def getQuery(self): return self.query + def getResolution(self): return self.resolution + def getAttachments(self): return self.attachments + def getEaseOfResolution(self): return self.easeofresolution + def getHostnames(self): return self.hostnames + def getImpact(self): return self.impact + def getService(self): return self.service + def getStatus(self): return self.status + def getTags(self): return self.tags + def getTarget(self): return self.target def getParent(self): @@ -1202,8 +1234,12 @@ def updateAttributes(self, name=None, text=None): if text is not None: self.text = text - def getDescription(self): return self.description - def getText(self): return self.text + def getDescription(self): + return self.description + + def getText(self): + return self.text + class Credential(ModelBase): class_signature = "Cred" @@ -1225,9 +1261,11 @@ def updateAttributes(self, username=None, password=None): def getUsername(self): return self.username + def getPassword(self): return self.password + class Command: class_signature = 'CommandRunInformation' def __init__(self, command, workspace_name): @@ -1244,21 +1282,29 @@ def __init__(self, command, workspace_name): def getCommand(self): return self.command + def getDuration(self): return self.duration + def getHostname(self): return self.hostname + def getIP(self): return self.ip + def getItime(self): return self.itime + def getParams(self): return self.params + def getUser(self): return self.user + def getWorkspace(self): return self.workspace + class _Workspace: class_signature = 'Workspace' @@ -1272,12 +1318,16 @@ def __init__(self, workspace, workspace_name): def getName(self): return self.name + def getDescription(self): return self.description + def getCustomer(self): return self.customer + def getStartDate(self): return self.start_date + def getFinishDate(self): return self.finish_date @@ -1326,9 +1376,9 @@ def __getUpdateAction(self): """This private method grabs the stackframes in look for the controller call that generated the update""" - l_strace = traceback.extract_stack(limit = 10) - controller_funcallnames = [ x[2] for x in l_strace if "controller" in x[0] ] + l_strace = traceback.extract_stack(limit=10) + controller_funcallnames = [x[2] for x in l_strace if "controller" in x[0]] if controller_funcallnames: - return "ModelControler." + " ModelControler.".join(controller_funcallnames) + return "ModelControler." + " ModelControler.".join(controller_funcallnames) return "No model controller call" \ No newline at end of file From d46fddbd6d36b195fdd3af6efdbaeaf5780f56d6 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 4 Dec 2017 19:03:16 -0300 Subject: [PATCH 0602/1506] Remove some code from plugins engine. * code simplification to use less queues. * adapt code to use the new api endpoints --- managers/reports_managers.py | 6 +-- model/common.py | 3 +- model/controller.py | 14 +++++-- persistence/server/models.py | 16 +++++--- persistence/server/server.py | 17 ++++---- persistence/server/utils.py | 12 +++--- plugins/controller.py | 67 ++++++++++--------------------- plugins/manager.py | 3 +- plugins/plugin.py | 76 ++++++++++++++++-------------------- server/api/base.py | 14 +++++++ server/api/modules/vulns.py | 2 +- server/commands/reports.py | 36 +++++++++++------ server/models.py | 8 ++-- 13 files changed, 141 insertions(+), 133 deletions(-) diff --git a/managers/reports_managers.py b/managers/reports_managers.py index 9140bd25c58..cd855bd9ae8 100755 --- a/managers/reports_managers.py +++ b/managers/reports_managers.py @@ -5,6 +5,7 @@ See the file 'doc/LICENSE' for the license information ''' +from threading import Thread import os import re @@ -58,14 +59,13 @@ def sendReport(self, plugin_id, filename): return True def onlinePlugin(self, cmd): - _, new_cmd = self.plugin_controller.processCommandInput('0', cmd, './') self.plugin_controller.onCommandFinished('0', 0, cmd) -class ReportManager(Process): +class ReportManager(Thread): def __init__(self, timer, ws_name, plugin_controller, polling=True): - Process.__init__(self) + Thread.__init__(self) self.polling = polling self.ws_name = ws_name self.daemon = False diff --git a/model/common.py b/model/common.py index a7fa6ad5bf4..e39f6962f8b 100644 --- a/model/common.py +++ b/model/common.py @@ -100,8 +100,7 @@ def createModelObject(self, classname, object_name, workspace_name=None, parent_ if classname in self._registered_objects: if object_name is not None: objargs['name'] = object_name - objargs['_id'] = -1 # they still don't have a server id - objargs['id'] = -1 # we'll generate it after making sure the objects are okey + objargs['parent'] = parent_id tmpObj = self._registered_objects[classname](objargs, workspace_name) return tmpObj else: diff --git a/model/controller.py b/model/controller.py index 0e80b882d5c..9c07263731c 100644 --- a/model/controller.py +++ b/model/controller.py @@ -102,7 +102,7 @@ def getDescription(action): class ModelController(Thread): - def __init__(self, mappers_manager): + def __init__(self, mappers_manager, pending_actions): Thread.__init__(self) self.mappers_manager = mappers_manager @@ -124,7 +124,7 @@ def __init__(self, mappers_manager): # http://docs.python.org/library/collections.html#collections.deque # the actions queue - self._pending_actions = Queue() + self._pending_actions = pending_actions # a reference to the ModelObjectFactory self._object_factory = model.common.factory @@ -143,6 +143,7 @@ def __init__(self, mappers_manager): self._setupActionDispatcher() self.objects_with_updates = [] + self._stop = False def __getattr__(self, name): getLogger(self).debug("ModelObject attribute to refactor: %s" % name) @@ -206,6 +207,8 @@ def _setupActionDispatcher(self): modelactions.DELVULN: self.__del, modelactions.ADDVULNWEBSRV: self.__add, modelactions.EDITVULN: self.__edit, + #Service + modelactions.ADDSERVICEHOST: self.__add, # Note modelactions.ADDNOTEHOST: self.__add, modelactions.DELNOTEHOST: self.__del, @@ -311,6 +314,9 @@ def setSavingModel(self, value): except RuntimeError: pass + def stop(self): + self._stop = True + def _main(self): """ The main method for the thread. @@ -319,7 +325,7 @@ def _main(self): This will make host addition and removal "thread-safe" and will avoid locking components that need to interact with the model """ - while True: + while not self._stop: # check if thread must finish # no plugin should be active to stop the controller if self._stop and self.active_plugins_count == 0: @@ -329,7 +335,6 @@ def _main(self): # or if we have pending duplicated hosts that need to be # merged by the userget if not self._sync_api_request and not self._saving_model_flag: - self.processAction() else: # there is some object requesting for a sync api so we @@ -435,6 +440,7 @@ def __add(self, new_obj, parent_id=None, *args): self._save_new_object(new_obj) except ConflictInDatabase as conflict: old_obj = new_obj.__class__(conflict.answer.json()['object'], new_obj._workspace_name) + new_obj.setID(old_obj.getID()) return self._handle_conflict(old_obj, new_obj) def __edit(self, obj, *args, **kwargs): diff --git a/persistence/server/models.py b/persistence/server/models.py index be56da390be..ebc9f4fdfce 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -9,9 +9,9 @@ import glob import os import sys -from time import time +from time import time, sleep import traceback -from threading import Lock +from threading import Lock, Condition from persistence.server import server from persistence.server.server_io_exceptions import (WrongObjectSignature, CantAccessConfigurationWithoutTheClient) @@ -697,18 +697,25 @@ class ModelBase(object): def __init__(self, obj, workspace_name): self._workspace_name = workspace_name self._server_id = obj.get('_id', '') - self.id = obj.get('id', '') + self.id = obj.get('id', None) self.name = obj.get('name') self.description = obj.get('description', "") self.owned = obj.get('owned', False) self.owner = obj.get('owner', '') self._metadata = obj.get('metadata', Metadata(self.owner)) + self.parent_id = obj.get('parent') self.updates = [] + def getParent(self): + return self.parent_id + def setID(self, id): self.id = id def getID(self): + while self.id is None: + sleep(0.5) + return self.id @staticmethod @@ -1165,8 +1172,7 @@ def getTags(self): def getTarget(self): return self.target - def getParent(self): - return self.parent + def getPolicyViolations(self): return self.policyviolations diff --git a/persistence/server/server.py b/persistence/server/server.py index f0a451bf80e..4e2b55e095c 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -45,6 +45,10 @@ OBJECT_TYPE_END_POINT_MAPPER = { 'CommandRunInformation': 'commands', 'Host': 'hosts', + 'Vulnerability': 'vulns', + 'VulnerabilityWeb': 'vulns', + 'Service': 'services', + 'Note': 'comments', } @@ -982,7 +986,7 @@ def update_interface(workspace_name, id, name, description, mac, type="Interface", metadata=metadata) -def create_service(workspace_name, id, name, description, ports, +def create_service(workspace_name, name, description, ports, parent, owned=False, owner="", protocol="", status="", version="", metadata=None): """Creates a service. @@ -1005,8 +1009,8 @@ def create_service(workspace_name, id, name, description, ports, A dictionary with the server's response. """ return _save_to_server(workspace_name, - id, name=name, + parent=parent, description=description, ports=ports, owned=owned, @@ -1053,7 +1057,7 @@ def update_service(workspace_name, id, name, description, ports, metadata=metadata) -def create_vuln(workspace_name, id, name, description, owned=None, owner="", +def create_vuln(workspace_name, name, description, owned=None, owner="", confirmed=False, data="", refs=None, severity="info", resolution="", desc="", metadata=None, status=None, policyviolations=[]): """Creates a vuln. @@ -1081,7 +1085,6 @@ def create_vuln(workspace_name, id, name, description, owned=None, owner="", A dictionary with the server's response. """ return _save_to_server(workspace_name, - id, name=name, description=description, owned=owned, @@ -1141,7 +1144,7 @@ def update_vuln(workspace_name, id, name, description, owned=None, owner="", metadata=metadata, policyviolations=policyviolations) -def create_vuln_web(workspace_name, id, name, description, owned=None, owner="", +def create_vuln_web(workspace_name, name, description, owned=None, owner="", confirmed=False, data="", refs=None, severity="info", resolution="", desc="", metadata=None, method=None, params="", path=None, pname=None, query=None, request=None, response=None, category="", website=None, @@ -1178,7 +1181,6 @@ def create_vuln_web(workspace_name, id, name, description, owned=None, owner="", A dictionary with the server's response. """ return _save_to_server(workspace_name, - id, name=name, description=description, owned=owned, @@ -1265,7 +1267,7 @@ def update_vuln_web(workspace_name, id, name, description, owned=None, owner="", type='VulnerabilityWeb', policyviolations=policyviolations) -def create_note(workspace_name, id, name, text, owned=None, owner="", +def create_note(workspace_name, name, text, owned=None, owner="", description="", metadata=None): """Creates a note. @@ -1284,7 +1286,6 @@ def create_note(workspace_name, id, name, text, owned=None, owner="", A dictionary with the server's response. """ return _save_to_server(workspace_name, - id, name=name, description=description, owned=owned, diff --git a/persistence/server/utils.py b/persistence/server/utils.py index 7dbac15c95d..5845d552850 100644 --- a/persistence/server/utils.py +++ b/persistence/server/utils.py @@ -53,11 +53,13 @@ def get_host_properties(host): def get_service_properties(service): - service_dict = {'ports': service.getPorts(), - 'protocol': service.getProtocol(), - 'status': service.getStatus(), - 'version': service.getVersion() - } + service_dict = { + 'ports': service.getPorts(), + 'protocol': service.getProtocol(), + 'status': service.getStatus(), + 'version': service.getVersion(), + 'parent': service.getParent() + } service_dict.update(get_object_properties(service)) return service_dict diff --git a/plugins/controller.py b/plugins/controller.py index 3eb825a539e..c1664151d50 100644 --- a/plugins/controller.py +++ b/plugins/controller.py @@ -7,13 +7,16 @@ See the file 'doc/LICENSE' for the license information ''' +import threading + import os import time import Queue import shlex import errno import logging -import multiprocessing +from multiprocessing import JoinableQueue +from Queue import Queue, Empty from plugins.plugin import PluginProcess import model.api @@ -28,11 +31,12 @@ logger = logging.getLogger(__name__) -class PluginController(object): +class PluginController(threading.Thread): """ TODO: Doc string. """ - def __init__(self, id, plugin_manager, mapper_manager): + def __init__(self, id, plugin_manager, mapper_manager, pending_actions): + super(PluginController, self).__init__() self.plugin_manager = plugin_manager self._plugins = plugin_manager.getPlugins() self.id = id @@ -45,6 +49,8 @@ def __init__(self, id, plugin_manager, mapper_manager): self._active_plugins = {} self.plugin_sets = {} self.plugin_manager.addController(self, self.id) + self.stop = False + self.pending_actions = pending_actions def _find_plugin(self, plugin_id): return self._plugins.get(plugin_id, None) @@ -93,60 +99,30 @@ def getAvailablePlugins(self): """ return self._plugins + def stop(self): + self.stop = True + def processOutput(self, plugin, output, command_id, isReport=False): - output_queue = multiprocessing.JoinableQueue() - new_elem_queue = multiprocessing.Queue() + output_queue = JoinableQueue() + plugin.set_actions_queue(self.pending_actions) plugin_process = PluginProcess( - plugin, output_queue, new_elem_queue, isReport) + plugin, output_queue, isReport) getLogger(self).debug( "Created plugin_process (%d) for plugin instance (%d)" % (id(plugin_process), id(plugin))) + # TODO: stop this processes plugin_process.start() + print('Plugin controller ', self.pending_actions) + self.pending_actions.put((modelactions.PLUGINSTART, plugin.id)) + output_queue.put(output) - output_queue.put(None) output_queue.join() - self._processAction(modelactions.PLUGINSTART, [plugin.id]) - - while True: - try: - current_action = new_elem_queue.get(block=False) - if current_action is None: - break - action = current_action[0] - parameters = current_action[1:] - - if hasattr(parameters[-1], '_metadata'): - parameters[-1]._metadata.command_id = command_id - - getLogger(self).debug( - "Core: Processing a new '%s', parameters (%s)\n" % - (action, str(parameters))) - self._processAction(action, parameters) - - except Queue.Empty: - continue - except IOError, e: - if e.errno == errno.EINTR: - continue - else: - getLogger(self).debug( - "new_elem_queue Exception - " - "something strange happened... " - "unhandled exception?") - break - except Exception as ex: - logger.exception(ex) - getLogger(self).debug( - "new_elem_queue Exception - " - "something strange happened... " - "unhandled exception?") - break - self._processAction(modelactions.PLUGINEND, [plugin.id]) + self.pending_actions.put((modelactions.PLUGINEND, plugin.id)) def _processAction(self, action, parameters): """ @@ -230,8 +206,7 @@ def onCommandFinished(self, pid, exit_code, term_output): cmd_info.duration = time.time() - cmd_info.itime self._mapper_manager.update(cmd_info) - self.processOutput(plugin, term_output, cmd_info.getID() -) + self.processOutput(plugin, term_output, cmd_info.getID()) del self._active_plugins[pid] return True diff --git a/plugins/manager.py b/plugins/manager.py index e4a3e0e8eb5..65b64d3d818 100644 --- a/plugins/manager.py +++ b/plugins/manager.py @@ -22,6 +22,7 @@ class PluginManager(object): + def __init__(self, plugin_repo_path): self._controllers = {} self._plugin_modules = {} @@ -74,7 +75,7 @@ def updateSettings(self, settings): def _instancePlugins(self): plugins = {} - for module in self._plugin_modules.itervalues(): + for module in self._plugin_modules.values(): new_plugin = module.createPlugin() self._verifyPlugin(new_plugin) plugins[new_plugin.id] = new_plugin diff --git a/plugins/plugin.py b/plugins/plugin.py index 62280177180..3cb12fc84ff 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -7,11 +7,10 @@ See the file 'doc/LICENSE' for the license information ''' +from threading import Thread import os import re -import time -import Queue import traceback import multiprocessing import deprecation @@ -56,7 +55,6 @@ def __init__(self): self.framework_version = None self._completition = {} self._new_elems = [] - self._pending_actions = Queue.Queue() self._settings = {} def has_custom_output(self): @@ -65,6 +63,16 @@ def has_custom_output(self): def get_custom_file_path(self): return self._output_file_path + def set_actions_queue(self, _pending_actions): + """ + We use plugin controller queue to add actions created by plugins. + Plugin controller will consume this actions. + + :param controller: plugin controller + :return: None + """ + self._pending_actions = _pending_actions + def getSettings(self): for param, (param_type, value) in self._settings.iteritems(): yield param, value @@ -150,7 +158,6 @@ def createAndAddHost(self, name, os="unknown"): host_obj = factory.createModelObject( Host.class_signature, name, os=os, parent_id=None) - host_obj._metadata.creatoserverr = self.id self.__addPendingAction(modelactions.ADDHOST, host_obj) return host_obj.getID() @@ -184,7 +191,7 @@ def createAndAddServiceToInterface(self, host_id, interface_id, name, version=version, description=description, parent_id=host_id) serv_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDSERVICEHOST, host_id, serv_obj) + self.__addPendingAction(modelactions.ADDSERVICEHOST, serv_obj) return serv_obj.getID() def createAndAddVulnToHost(self, host_id, name, desc="", ref=[], @@ -196,7 +203,7 @@ def createAndAddVulnToHost(self, host_id, name, desc="", ref=[], confirmed=False, parent_id=host_id) vuln_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDVULNHOST, host_id, vuln_obj) + self.__addPendingAction(modelactions.ADDVULNHOST, vuln_obj) return vuln_obj.getID() @deprecation.deprecated(deprecated_in="3.0", removed_in="3.5", @@ -212,7 +219,7 @@ def createAndAddVulnToInterface(self, host_id, interface_id, name, confirmed=False, parent_id=interface_id) vuln_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDVULNHOST, host_id, vuln_obj) + self.__addPendingAction(modelactions.ADDVULNHOST, vuln_obj) return vuln_obj.getID() def createAndAddVulnToService(self, host_id, service_id, name, desc="", @@ -224,7 +231,7 @@ def createAndAddVulnToService(self, host_id, service_id, name, desc="", confirmed=False, parent_id=service_id) vuln_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDVULNSRV, host_id, service_id, vuln_obj) + self.__addPendingAction(modelactions.ADDVULNSRV, vuln_obj) return vuln_obj.getID() def createAndAddVulnWebToService(self, host_id, service_id, name, desc="", @@ -240,7 +247,7 @@ def createAndAddVulnWebToService(self, host_id, service_id, name, desc="", category=category, confirmed=False, parent_id=service_id) vulnweb_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDVULNWEBSRV, host_id, service_id, vulnweb_obj) + self.__addPendingAction(modelactions.ADDVULNWEBSRV, vulnweb_obj) return vulnweb_obj.getID() def createAndAddNoteToHost(self, host_id, name, text): @@ -250,7 +257,7 @@ def createAndAddNoteToHost(self, host_id, name, text): name, text=text, parent_id=host_id) note_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDNOTEHOST, host_id, note_obj) + self.__addPendingAction(modelactions.ADDNOTEHOST, note_obj) return note_obj.getID() @deprecation.deprecated(deprecated_in="3.0", removed_in="3.5", @@ -263,7 +270,7 @@ def createAndAddNoteToInterface(self, host_id, interface_id, name, text): name, text=text, parent_id=interface_id) note_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDNOTEHOST, host_id, note_obj) + self.__addPendingAction(modelactions.ADDNOTEHOST, note_obj) return note_obj.getID() def createAndAddNoteToService(self, host_id, service_id, name, text): @@ -273,17 +280,17 @@ def createAndAddNoteToService(self, host_id, service_id, name, text): name, text=text, parent_id=service_id) note_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDNOTESRV, host_id, service_id, note_obj) + self.__addPendingAction(modelactions.ADDNOTESRV, note_obj) return note_obj.getID() - def createAndAddNoteToNote(self, host_id, service_id, note_id, name, text): + def createAndAddNoteToNote(self, host_id, name, text): note_obj = model.common.factory.createModelObject( Note.class_signature, name, text=text, parent_id=note_id) note_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDNOTENOTE, host_id, service_id, note_id, note_obj) + self.__addPendingAction(modelactions.ADDNOTENOTE, host_id, note_obj) return note_obj.getID() def createAndAddCredToService(self, host_id, service_id, username, @@ -294,7 +301,7 @@ def createAndAddCredToService(self, host_id, service_id, username, username, password=password, parent_id=service_id) cred_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDCREDSRV, host_id, service_id, cred_obj) + self.__addPendingAction(modelactions.ADDCREDSRV, cred_obj) return cred_obj.getID() def log(self, msg, level='INFO'): @@ -322,11 +329,17 @@ def processOutput(self, term_output): self.processReport(self._output_file_path) -class PluginProcess(multiprocessing.Process): - def __init__(self, plugin_instance, output_queue, new_elem_queue, isReport=False): - multiprocessing.Process.__init__(self) +class PluginProcess(Thread): + def __init__(self, plugin_instance, output_queue, isReport=False): + """ + Executes one plugin. + + :param plugin_instance: current plugin in execution. + :param output_queue: queue with raw ouput of that the plugin needs. + :param isReport: output data was read from file. + """ + Thread.__init__(self) self.output_queue = output_queue - self.new_elem_queue = new_elem_queue self.plugin = plugin_instance self.isReport = isReport @@ -348,34 +361,13 @@ def run(self): self.plugin.processReport(output) else: self.plugin.processOutput(output) - except Exception: + except Exception as ex: model.api.devlog("Plugin raised an exception:") model.api.devlog(traceback.format_exc()) - else: - while True: - try: - self.new_elem_queue.put( - self.plugin._pending_actions.get(block=False)) - time.sleep(0) # potentially allowing another process to run - except Queue.Empty: - model.api.devlog( - ("PluginProcess run _pending_actions" - " queue Empty. Breaking loop")) - break - except Exception as ex: - model.api.devlog( - ("PluginProcess run getting from " - "_pending_action queue - something strange " - "happened... unhandled exception?")) - model.api.devlog(traceback.format_exc()) - break - - else: - done = True model.api.devlog('%s: Exiting' % proc_name) self.output_queue.task_done() - self.new_elem_queue.put(None) + return diff --git a/server/api/base.py b/server/api/base.py index df48a3c1f9b..ef5404f5596 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -159,6 +159,20 @@ def register(cls, app, *args, **kwargs): """Register and add JSON error handler. Use error code 400 instead of 409""" super(GenericView, cls).register(app, *args, **kwargs) + + @app.errorhandler(422) + def handle_conflict(err): + # webargs attaches additional metadata to the `data` attribute + exc = getattr(err, 'exc') + if exc: + # Get validations from the ValidationError object + messages = exc.messages + else: + messages = ['Invalid request'] + return flask.jsonify({ + 'messages': messages, + }), 400 + @app.errorhandler(409) def handle_conflict(err): # webargs attaches additional metadata to the `data` attribute diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 362bc984899..440bb9348a0 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -101,7 +101,7 @@ class VulnerabilitySchema(AutoSchema): host = fields.Integer(dump_only=True, attribute='host_id') severity = fields.Method(serialize='get_severity', deserialize='load_severity') status = fields.Method(serialize='get_status', deserialize='load_status') # TODO: this breaks enum validation. - type = fields.Method(serialize='get_type', deserialize='load_type') + type = fields.Method(serialize='get_type', deserialize='load_type', required=True) obj_id = fields.String(dump_only=True, attribute='id') target = fields.Method('get_target') metadata = SelfNestedField(MetadataSchema()) diff --git a/server/commands/reports.py b/server/commands/reports.py index ac3bc6a40fd..25fad24dc9c 100644 --- a/server/commands/reports.py +++ b/server/commands/reports.py @@ -1,5 +1,6 @@ import os import logging +from Queue import Queue from tqdm import tqdm @@ -22,37 +23,48 @@ def import_external_reports(workspace_name=None): os.path.join(CONF.getConfigPath(), "plugins")) mappers_manager = MapperManager() - plugin_controller = PluginController( - 'PluginController', - plugin_manager, - mappers_manager - ) + if workspace_name: query = Workspace.query.filter_by(name=workspace_name) else: query = Workspace.query - process_workspaces(mappers_manager, plugin_controller, query) + process_workspaces(mappers_manager, plugin_manager, query) #controller._pending_actions.join() -def process_workspaces(mappers_manager, plugin_controller, query): - processes = [] +def process_workspaces(mappers_manager, plugin_manager, query): + report_managers = [] + controllers = [] for workspace in query.all(): + if workspace.name != 'airbnb': + continue + print(workspace.name) + pending_actions = Queue() + plugin_controller = PluginController( + 'PluginController', + plugin_manager, + mappers_manager, + pending_actions + ) mappers_manager.createMappers(workspace.name) - controller = ModelController(mappers_manager) + controller = ModelController(mappers_manager, pending_actions) workspace_manager = WorkspaceManager(mappers_manager) setUpAPIs(controller, workspace_manager, hostname=None, port=None) controller.start() + controllers.append(controller) report_manager = ReportManager( 0.1, workspace.name, plugin_controller, polling=False ) - processes.append(report_manager) + report_managers.append(report_manager) report_manager.start() - #for thread in tqdm(processes): - # thread.join() \ No newline at end of file + #for report_manager in report_managers: + # report_manager.join() + + #for controller in controllers: + # controller.join() \ No newline at end of file diff --git a/server/models.py b/server/models.py index e208faeaeaa..acff656829d 100644 --- a/server/models.py +++ b/server/models.py @@ -1247,10 +1247,10 @@ class Comment(Metadata): reply_to_id = Column(Integer, ForeignKey('comment.id')) reply_to = relationship( - 'Comment', - remote_side=[id], - foreign_keys=[reply_to_id] - ) + 'Comment', + remote_side=[id], + foreign_keys=[reply_to_id] + ) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) From b0409fd922403c544ebbc87d44db88a548bf8c47 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 4 Dec 2017 19:08:41 -0300 Subject: [PATCH 0603/1506] Add validation error si type is missing --- server/api/modules/vulns.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 440bb9348a0..774a3546e37 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -377,7 +377,9 @@ def model_class(self): def _get_schema_class(self): assert self.schema_class_dict is not None, "You must define schema_class" if request.method == 'POST': - requested_type = request.json['type'] + requested_type = request.json.get('type', None) + if not requested_type: + raise ValidationError('Type is required.') if requested_type not in self.schema_class_dict: raise InvalidUsage('Invalid vulnerability type.') return self.schema_class_dict[requested_type] From 07e1a7c31a0d4edd7cdb95100af14674055f5f22 Mon Sep 17 00:00:00 2001 From: Mike Zhong Date: Tue, 5 Dec 2017 11:10:55 -0300 Subject: [PATCH 0604/1506] corrected get_services_number before, it would (incorrectly) return the number of interfaces, not services --- persistence/server/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence/server/server.py b/persistence/server/server.py index c9f7e9b95b6..9b06e34f5b8 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -730,7 +730,7 @@ def get_services_number(workspace_name, **params): Returns: The amount of services in the workspace as an integer. """ - return int(get_workspace_summary(workspace_name)['interfaces']) + return int(get_workspace_summary(workspace_name)['services']) def get_interfaces_number(workspace_name, **params): """ From 04514eccd749a9f4b7915b3fda4fc4e3b0d27b0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 5 Dec 2017 12:17:55 -0300 Subject: [PATCH 0605/1506] Add ugly hack to make flask shell work --- server/web.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/web.py b/server/web.py index 5d33f46ae1e..9ee98cb5c69 100644 --- a/server/web.py +++ b/server/web.py @@ -8,6 +8,12 @@ import twisted.web from twisted.web.resource import Resource +# Ugly hack to make "flask shell" work. It works because when running via flask +# shell, __file__ will be server/web.py instead of faraday-server.py +if os.path.split(os.path.dirname(__file__))[-1] == 'server': + path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) + sys.path.append(path) + import server.config from twisted.internet import ssl, reactor, error From 45d9dd7a1fe96544623a06d976e8fb6f5636060d Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 5 Dec 2017 16:17:58 -0300 Subject: [PATCH 0606/1506] Fix field names Start and End dates for the Workspace object are now start_date and end_date. Renamed across all components to avoid bugs. --- server/api/modules/workspaces.py | 4 ++-- .../workspaces/controllers/workspaces.js | 24 +++++++++---------- .../workspaces/providers/workspaces.js | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index 7659a57e521..be32e6baf05 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -49,8 +49,8 @@ class WorkspaceSummarySchema(Schema): class WorkspaceDurationSchema(Schema): - start = JSTimestampField(attribute='start_date') - end = JSTimestampField(attribute='end_date') + start_date = JSTimestampField(attribute='start_date') + end_date = JSTimestampField(attribute='end_date') class WorkspaceSchema(AutoSchema): diff --git a/server/www/scripts/workspaces/controllers/workspaces.js b/server/www/scripts/workspaces/controllers/workspaces.js index ffd66edb8df..5bd0cfa907b 100644 --- a/server/www/scripts/workspaces/controllers/workspaces.js +++ b/server/www/scripts/workspaces/controllers/workspaces.js @@ -119,16 +119,16 @@ angular.module('faradayApp') $scope.update = function(ws){ if(typeof(ws.duration.start_date) == "number") { - start = ws.duration.start_date; + start_date = ws.duration.start_date; } else if(ws.duration.start_date) { - start = ws.duration.start_date.getTime(); - } else {start = "";} + start_date = ws.duration.start_date.getTime(); + } else {start_date = "";} if(typeof(ws.duration.end_date) == "number") { - end = ws.duration.end_date; + end_date = ws.duration.end_date; } else if(ws.duration.end_date) { - end = ws.duration.end_date.getTime(); - } else {end = "";} - duration = {'start': start, 'end': end}; + end_date = ws.duration.end_date.getTime(); + } else {end_date = "";} + duration = {'start_date': start_date, 'end_date': end_date}; workspace = { "_id": ws._id, "_rev": ws._rev, @@ -160,7 +160,7 @@ angular.module('faradayApp') }); $scope.modal.result.then(function(workspace) { - workspace = $scope.create(workspace.name, workspace.description, workspace.start, workspace.end, workspace.scope); + workspace = $scope.create(workspace.name, workspace.description, workspace.start_date, workspace.end_date, workspace.scope); $scope.insert(workspace); }); @@ -252,9 +252,9 @@ angular.module('faradayApp') }; // end of modal context - $scope.create = function(wname, wdesc, start, end, scope){ - if(end) end = end.getTime(); else end = ""; - if(start) start = start.getTime(); else start = ""; + $scope.create = function(wname, wdesc, start_date, end_date, scope){ + if(end_date) end_date = end_date.getTime(); else end_date = ""; + if(start_date) start_date = start_date.getTime(); else start_date = ""; workspace = { "_id": wname, "customer": "", @@ -262,7 +262,7 @@ angular.module('faradayApp') "type": "Workspace", "children": [ ], - "duration": {"start_date": start, "end_date": end}, + "duration": {"start_date": start_date, "end_date": end_date}, "scope": scope, "description": wdesc }; diff --git a/server/www/scripts/workspaces/providers/workspaces.js b/server/www/scripts/workspaces/providers/workspaces.js index 17e77585c06..c8c2e028cd2 100644 --- a/server/www/scripts/workspaces/providers/workspaces.js +++ b/server/www/scripts/workspaces/providers/workspaces.js @@ -46,8 +46,8 @@ angular.module('faradayApp') var deferred = $q.defer(); ServerAPI.getWorkspace(workspace_name).then(function(workspace) { deferred.resolve({ - "start": workspace.data.duration.start, - "end": workspace.data.duration.end + "start_date": workspace.data.duration.start_date, + "end_date": workspace.data.duration.end_date }); }); return deferred.promise; From 5528b7e794657323897f255fff0a3d4aebdc3b2d Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 5 Dec 2017 16:22:50 -0300 Subject: [PATCH 0607/1506] Add deserialize method for JS Timestamp schema --- server/schemas.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/schemas.py b/server/schemas.py index 002d1f1417d..0ee37921b77 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -1,4 +1,5 @@ import time +import datetime from marshmallow import fields, Schema from marshmallow.exceptions import ValidationError @@ -15,7 +16,8 @@ def _serialize(self, value, attr, obj): return int(time.mktime(value.timetuple()) * 1000) def _deserialize(self, value, attr, data): - raise NotImplementedError("Only dump is implemented for now") + if value is not None and value: + return datetime.datetime.fromtimestamp(value/1e3) class PrimaryKeyRelatedField(fields.Field): From 2b8d354c69434360d2c831968da8654f6b58da07 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 5 Dec 2017 16:30:00 -0300 Subject: [PATCH 0608/1506] Fix URL for Workspace creation in the Web UI --- server/www/scripts/commons/providers/server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index df921c70055..70fb560163d 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -56,7 +56,7 @@ angular.module("faradayApp") return APIURL + objectType + "/" + objectId + "/"; }; - var createDbUrl = function(wsName) { + var createDbUrl = function(wsName = "") { return APIURL + "ws/" + wsName; } @@ -483,7 +483,7 @@ angular.module("faradayApp") } ServerAPI.createWorkspace = function(wsName, data) { - var dbUrl = createDbUrl(wsName); + var dbUrl = createDbUrl(); return send_data(dbUrl, data, true, "POST"); } From 074a907ff711145868920fc853d2e0f85b80cd72 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 5 Dec 2017 16:32:55 -0300 Subject: [PATCH 0609/1506] Fix Workspace duration object deserialization Added a post_load method to flatten the duration dict into the two corresponding fields - start_date and end_date. --- server/api/modules/workspaces.py | 10 +++++++++- server/schemas.py | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index be32e6baf05..7b96360ff54 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -5,7 +5,7 @@ import flask from flask import Blueprint -from marshmallow import Schema, fields +from marshmallow import Schema, fields, post_load from sqlalchemy.orm import undefer from server.models import db, Workspace @@ -64,6 +64,14 @@ class Meta: fields = ('_id', 'id', 'customer', 'description', 'active', 'duration', 'name', 'public', 'scope', 'stats') + @post_load + def post_load_duration(self, data): + # Unflatten duration (move data[duration][*] to data[*]) + duration = data.pop('duration', None) + if duration: + data.update(duration) + return data + class WorkspaceView(ReadWriteView): route_base = 'ws' diff --git a/server/schemas.py b/server/schemas.py index 0ee37921b77..17334e180ca 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -60,9 +60,13 @@ def _serialize(self, value, attr, obj): return ret def _deserialize(self, value, attr, data): + """ + It would be awesome if this method could also flatten the dict keys into the parent + """ load = self.target_schema.load(value) if load.errors: raise ValidationError(load.errors) + return load.data From 45abdb514cd47c14133c222dee4e7afd27cc86d6 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 5 Dec 2017 19:14:46 -0300 Subject: [PATCH 0610/1506] Add type validation to JSTimestamp Field --- server/schemas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/schemas.py b/server/schemas.py index 17334e180ca..f7440dedb29 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -7,7 +7,7 @@ from server.models import CommandObject -class JSTimestampField(fields.Field): +class JSTimestampField(fields.Integer): """A field to serialize datetime objects into javascript compatible timestamps (like time.time()) * 1000""" @@ -17,7 +17,7 @@ def _serialize(self, value, attr, obj): def _deserialize(self, value, attr, data): if value is not None and value: - return datetime.datetime.fromtimestamp(value/1e3) + return datetime.datetime.fromtimestamp(self._validated(value)/1e3) class PrimaryKeyRelatedField(fields.Field): From e40871f885d4c2752c9d51f235c91b4570ede66d Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 5 Dec 2017 19:15:52 -0300 Subject: [PATCH 0611/1506] Add test cases for Workspace obj --- test_cases/test_api_workspace.py | 70 +++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 5 deletions(-) diff --git a/test_cases/test_api_workspace.py b/test_cases/test_api_workspace.py index 28edddbe4ed..ed20a41cebc 100644 --- a/test_cases/test_api_workspace.py +++ b/test_cases/test_api_workspace.py @@ -1,8 +1,9 @@ +import time import pytest from server.models import Workspace from server.api.modules.workspaces import WorkspaceView -from test_cases.test_api_non_workspaced_base import ReadWriteAPITests, API_PREFIX +from test_cases.test_api_non_workspaced_base import ReadWriteAPITests from test_cases import factories class TestWorkspaceAPI(ReadWriteAPITests): @@ -25,7 +26,10 @@ def test_host_count(self, host_factory, test_client, session): '?confirmed=false' ]) - def test_vuln_count(self, vulnerability_factory, test_client, session, + def test_vuln_count(self, + vulnerability_factory, + test_client, + session, querystring): vulnerability_factory.create_batch(8, workspace=self.first_object, confirmed=False) @@ -41,8 +45,11 @@ def test_vuln_count(self, vulnerability_factory, test_client, session, '?confirmed=true' ]) - def test_vuln_count_confirmed(self, vulnerability_factory, test_client, - session, querystring): + def test_vuln_count_confirmed(self, + vulnerability_factory, + test_client, + session, + querystring): vulnerability_factory.create_batch(8, workspace=self.first_object, confirmed=False) vulnerability_factory.create_batch(5, workspace=self.first_object, @@ -50,4 +57,57 @@ def test_vuln_count_confirmed(self, vulnerability_factory, test_client, session.commit() res = test_client.get(self.url(self.first_object) + querystring) assert res.status_code == 200 - assert res.json['stats']['total_vulns'] == 5 \ No newline at end of file + assert res.json['stats']['total_vulns'] == 5 + + def test_create_fails_with_valid_duration(self, session, test_client): + workspace_count_previous = session.query(Workspace).count() + start_date = int(time.time())*1000 + end_date = start_date+86400000 + duration = {'start_date': start_date, 'end_date': end_date} + raw_data = {'name': 'somethingdarkside', 'duration': duration} + res = test_client.post('/v2/ws/', data=raw_data) + assert res.status_code == 201 + assert workspace_count_previous + 1 == session.query(Workspace).count() + assert res.json['duration']['start_date'] == start_date + assert res.json['duration']['end_date'] == end_date + + def test_create_fails_with_invalid_duration_start_type(self, + session, + test_client): + workspace_count_previous = session.query(Workspace).count() + start_date = 'this should clearly fail' + duration = {'start_date': start_date, 'end_date': 86400000} + raw_data = {'name': 'somethingdarkside', 'duration': duration} + res = test_client.post('/v2/ws/', data=raw_data) + assert res.status_code == 400 + assert workspace_count_previous == session.query(Workspace).count() + + @pytest.mark.xfail(reason="Filter not implemented yet") + def test_create_fails_with_invalid_duration_start_after_end(self, + session, + test_client): + workspace_count_previous = session.query(Workspace).count() + start_date = int(time.time())*1000 + duration = {'start_date': start_date, 'end_date': start_date-86400000} + raw_data = {'name': 'somethingdarkside', 'duration': duration} + res = test_client.post('/v2/ws/', data=raw_data) + assert res.status_code == 400 + assert workspace_count_previous == session.query(Workspace).count() + + def test_create_with_description(self, session, test_client): + description = 'darkside' + raw_data = {'name': 'something', 'description': description} + workspace_count_previous = session.query(Workspace).count() + res = test_client.post('/v2/ws/', data=raw_data) + assert res.status_code == 201 + assert workspace_count_previous + 1 == session.query(Workspace).count() + assert res.json['description'] == description + + def test_create_with_scope(self, session, test_client): + description = 'darkside' + raw_data = {'name': 'something', 'description': description} + workspace_count_previous = session.query(Workspace).count() + res = test_client.post('/v2/ws/', data=raw_data) + assert res.status_code == 201 + assert workspace_count_previous + 1 == session.query(Workspace).count() + assert res.json['description'] == description \ No newline at end of file From 251de1108fc4198f5967805c961993e43d01873f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 5 Dec 2017 19:18:58 -0300 Subject: [PATCH 0612/1506] Update code to check multiple fields on uniques * For uniques in views use a list of tuples instead of a list of strings. * Update tests to assert status code 409 --- server/api/base.py | 27 ++++++++++++++------------ server/api/modules/hosts.py | 2 +- server/api/modules/services.py | 1 + test_cases/test_api_hosts.py | 4 ++-- test_cases/test_api_services.py | 18 +++++++++++++++++ test_cases/test_api_vulnerability.py | 15 +++++++++++--- test_cases/test_api_workspaced_base.py | 4 ++-- 7 files changed, 51 insertions(+), 20 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index ef5404f5596..41315edce17 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -234,18 +234,21 @@ def _validate_uniqueness(self, obj, object_id=None): assert obj.workspace is not None, "Object must have a " \ "workspace attribute set to call _validate_uniqueness" primary_key_field = inspect(self.model_class).primary_key[0] - for field_name in self.unique_fields: - field = getattr(self.model_class, field_name) - value = getattr(obj, field_name) - query = self._get_base_query(obj.workspace.name).filter( - field == value) - if object_id is not None: - # The object already exists in DB, we want to fetch an object - # different to this one but with the same unique field - query = query.filter(primary_key_field != object_id) - obj = query.one_or_none() - conflict_data = self._get_schema_class()().dump(obj).data - if obj: + query = self._get_base_query(obj.workspace.name) + if object_id is not None: + # The object already exists in DB, we want to fetch an object + # different to this one but with the same unique field + query = query.filter(primary_key_field != object_id) + for field_names in self.unique_fields: + for field_name in field_names: + field = getattr(self.model_class, field_name) + value = getattr(obj, field_name) + query = query.filter( + field == value) + + existing_obj = query.one_or_none() + conflict_data = self._get_schema_class()().dump(existing_obj).data + if existing_obj: db.session.rollback() abort(409, ValidationError( { diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index a86a921e178..25cb0b33637 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -79,7 +79,7 @@ class HostsView(PaginatedMixin, model_class = Host order_field = Host.ip.asc() schema_class = HostSchema - unique_fields = ['ip'] + unique_fields = [('ip', )] filterset_class = HostFilterSet get_undefer = [Host.open_service_count, Host.vulnerability_count] diff --git a/server/api/modules/services.py b/server/api/modules/services.py index d5daa2f4209..e09d3d84d9d 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -81,6 +81,7 @@ class ServiceView(ReadWriteWorkspacedView): schema_class = ServiceSchema count_extra_filters = [Service.status == 'open'] get_undefer = [Service.credentials_count, Service.vulnerability_count] + unique_fields = [('port', 'protocol', 'host')] def _envelope_list(self, objects, pagination_metadata=None): services = [] diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 1fc58719251..8e83936fdb4 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -118,7 +118,7 @@ def test_create_a_host_fails_with_existing_ip(self, session, "ip": host.ip, "description": "aaaaa", }) - assert res.status_code == 400 + assert res.status_code == 409 assert Host.query.count() == HOSTS_COUNT + 1 def test_create_a_host_with_ip_of_other_workspace(self, test_client, @@ -154,7 +154,7 @@ def test_update_a_host_fails_with_existing_ip(self, test_client, session): "ip": self.workspace.hosts[1].ip, # Existing IP "description": "bbbbb", }) - assert res.status_code == 400 + assert res.status_code == 409 session.refresh(host) assert host.ip == original_ip assert host.description == original_desc # It shouldn't do a partial update diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index c63a0eb1b97..417c7b37aff 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -2,6 +2,7 @@ """Tests for many API endpoints that do not depend on workspace_name""" import pytest +import json from server.api.modules.services import ServiceView from test_cases import factories @@ -101,3 +102,20 @@ def test_update_fails_with_host_of_other_workspace(self, test_client, res = test_client.put(self.url(self.first_object), data=data) assert res.status_code == 400 assert 'Host with id' in res.data + + def test_create_service_returns_conflict_if_already_exists(self, test_client, host, session): + session.commit() + service = self.first_object + data = { + "name": service.name, + "description": service.description, + "owned": service.owned, + "ports": [service.port], + "protocol": service.protocol, + "status": service.status, + "parent": service.host_id + } + res = test_client.post(self.url(workspace=service.workspace), data=data) + assert res.status_code == 409 + message = json.loads(res.data) + assert message['object']['_id'] == service.id \ No newline at end of file diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 020a4e39c3c..c907ead5e73 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -628,7 +628,10 @@ def test_handles_invalid_impact(self, host_with_hostnames, test_client, res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) assert res.status_code == 400 - def test_create_vuln_with_invalid_type(self, host_with_hostnames, test_client, session): + def test_create_vuln_with_invalid_type(self, + host_with_hostnames, + test_client, + session): session.commit() # flush host_with_hostnames raw_data = self._create_post_data_vulnerability( name='New vulns', @@ -640,12 +643,18 @@ def test_create_vuln_with_invalid_type(self, host_with_hostnames, test_client, s ) ws_name = host_with_hostnames.workspace.name vuln_count_previous = session.query(Vulnerability).count() - res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) + res = test_client.post( + '/v2/ws/{0}/vulns/'.format(ws_name), + data=raw_data, + ) assert res.status_code == 400 assert vuln_count_previous == session.query(Vulnerability).count() assert res.json['message'] == 'Invalid vulnerability type.' - def test_create_vuln_with_invalid_ease_of_resolution(self, host_with_hostnames, test_client, session): + def test_create_vuln_with_invalid_ease_of_resolution(self, + host_with_hostnames, + test_client, + session): session.commit() # flush host_with_hostnames raw_data = self._create_post_data_vulnerability( name='New vulns', diff --git a/test_cases/test_api_workspaced_base.py b/test_cases/test_api_workspaced_base.py index c95a2e72409..37c6f15215e 100644 --- a/test_cases/test_api_workspaced_base.py +++ b/test_cases/test_api_workspaced_base.py @@ -114,7 +114,7 @@ def test_create_fails_with_existing(self, session, test_client): data = self.factory.build_dict() data[unique_field] = getattr(self.first_object, unique_field) res = test_client.post(self.url(), data=data) - assert res.status_code == 400 + assert res.status_code == 409 assert self.model.query.count() == OBJECT_COUNT def test_create_with_existing_in_other_workspace(self, test_client, @@ -148,7 +148,7 @@ def test_update_fails_with_existing(self, test_client, session): data = self.factory.build_dict() data[unique_field] = getattr(self.objects[1], unique_field) res = test_client.put(self.url(self.first_object), data=data) - assert res.status_code == 400 + assert res.status_code == 409 assert self.model.query.count() == OBJECT_COUNT def test_update_an_object_fails_with_empty_dict(self, test_client): From ecae6d09426ca62de24875eb06057c43a2e98a68 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 5 Dec 2017 19:43:25 -0300 Subject: [PATCH 0613/1506] Use thread event to set and get the obj id --- persistence/server/models.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index ebc9f4fdfce..5347b3d5489 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -6,12 +6,10 @@ See the file 'doc/LICENSE' for the license information ''' -import glob -import os -import sys -from time import time, sleep +import logging +from time import time import traceback -from threading import Lock, Condition +from threading import Lock, Condition, RLock, Event from persistence.server import server from persistence.server.server_io_exceptions import (WrongObjectSignature, CantAccessConfigurationWithoutTheClient) @@ -28,7 +26,6 @@ from model.diff import ModelObjectDiff, MergeSolver from model.conflict import ConflictUpdate -from config.configuration import getInstanceConfiguration from functools import wraps from difflib import Differ @@ -705,17 +702,20 @@ def __init__(self, obj, workspace_name): self._metadata = obj.get('metadata', Metadata(self.owner)) self.parent_id = obj.get('parent') self.updates = [] + self.id_available = Event() + if self.id is not None: + self.id_available.set() def getParent(self): return self.parent_id def setID(self, id): self.id = id + self.id_available.set() def getID(self): - while self.id is None: - sleep(0.5) - + if self.id is None: + self.id_available.wait(timeout=10) return self.id @staticmethod From db4f9913f25595abea320e35516bb87d16f12cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 5 Dec 2017 19:43:59 -0300 Subject: [PATCH 0614/1506] Fix hostnames field in the hosts API Also make default_gateway field optional because the API isn't using it and it is not required --- server/api/modules/hosts.py | 37 +++++++++++++++++----- server/models.py | 51 +++++++++++++++++++++++++++++++ test_cases/models/test_host.py | 56 ++++++++++++++++++++++++++++++++++ test_cases/test_api_hosts.py | 24 ++++++++++++++- 4 files changed, 159 insertions(+), 9 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 84358d6365e..50c05702c94 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -17,7 +17,12 @@ FilterAlchemyMixin, FilterSetMeta, ) -from server.schemas import PrimaryKeyRelatedField, MetadataSchema, SelfNestedField +from server.schemas import ( + MetadataSchema, + MutableField, + PrimaryKeyRelatedField, + SelfNestedField +) from server.models import Host, Service, db, Hostname host_api = Blueprint('host_api', __name__) @@ -29,7 +34,8 @@ class HostSchema(AutoSchema): _rev = fields.String(default='') ip = fields.String(default='') description = fields.String(required=True) # Explicitly set required=True - default_gateway = fields.String(attribute="default_gateway_ip", allow_none=True) + default_gateway = fields.String(attribute="default_gateway_ip", + required=False, allow_none=True) name = fields.String(dump_only=True, attribute='ip', default='') os = fields.String(default='') owned = fields.Boolean(default=False) @@ -37,11 +43,12 @@ class HostSchema(AutoSchema): services = fields.Integer(attribute='open_service_count', dump_only=True) vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) credentials = fields.Integer(attribute='credentials_count', dump_only=True) - hostnames = PrimaryKeyRelatedField('name', many=True, - attribute="hostnames", - # TODO migration: make it writable - dump_only=True, # Only for now - default=[]) + hostnames = MutableField( + PrimaryKeyRelatedField('name', many=True, + attribute="hostnames", + dump_only=True, + default=[]), + fields.List(fields.String)) metadata = SelfNestedField(MetadataSchema()) class Meta: @@ -102,11 +109,25 @@ def _perform_create(self, data, **kwargs): hostnames = data.pop('hostnames', []) host = super(HostsView, self)._perform_create(data, **kwargs) for name in hostnames: - get_or_create(db.session, Hostname, name=name['key'], host=host, + get_or_create(db.session, Hostname, name=name, host=host, workspace=host.workspace) db.session.commit() return host + def _update_object(self, obj, data): + try: + hostnames = data.pop('hostnames') + except KeyError: + pass + else: + obj.set_hostnames(hostnames) + + # Required to make the assert pass. This actually makes two requests + # to the DB + db.session.commit() + + return super(HostsView, self)._update_object(obj, data) + def _envelope_list(self, objects, pagination_metadata=None): hosts = [] for host in objects: diff --git a/server/models.py b/server/models.py index e208faeaeaa..9c0d3d04308 100644 --- a/server/models.py +++ b/server/models.py @@ -1,6 +1,7 @@ # Faraday Penetration Test IDE # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information +import operator from datetime import datetime from sqlalchemy import ( @@ -220,6 +221,56 @@ class Host(Metadata): def parent(self): return + def set_hostnames(self, new_hostnames): + """Override the host's hostnames. Take care of deleting old not + used hostnames and to leave the sames the ones that weren't + modified + + This function was thought to update existing objects, it shouldn't + be used when creating! + """ + return set_children_objects(self, new_hostnames, + parent_field='hostnames', + child_field='name') + + +def set_children_objects(instance, value, parent_field, child_field='id'): + """ + Override some kind of children of instance. This is useful in one + to many relationships. It takes care of deleting not used children, + adding new objects, and keeping the not modified ones the same. + + :param instance: instance of the parent object + :param value: list of childs (values of the child_field) + :param parent_field: the parent field's relationship to the children name + :param child_field: the "lookup field" of the children model + """ + # Get the class of the children. Inspired in + # https://stackoverflow.com/questions/6843144/how-to-find-sqlalchemy-remote-side-objects-class-or-class-name-without-db-queri + children_model = getattr( + type(instance), parent_field).property.mapper.class_ + + value = set(value) + current_value = getattr(instance, parent_field) + current_value_fields = set(map(operator.attrgetter(child_field), + current_value)) + + for existing_child in current_value_fields: + if existing_child not in value: + # It was removed + removed_instance = next( + inst for inst in current_value + if getattr(inst, child_field) == existing_child) + db.session.delete(removed_instance) + + for new_child in value: + if new_child in current_value_fields: + # it already exists + continue + kwargs = {child_field: new_child} + kwargs['workspace'] = instance.workspace + current_value.append(children_model(**kwargs)) + class Hostname(Metadata): __tablename__ = 'hostname' diff --git a/test_cases/models/test_host.py b/test_cases/models/test_host.py index d008484948d..dd3535abb8d 100644 --- a/test_cases/models/test_host.py +++ b/test_cases/models/test_host.py @@ -1,6 +1,7 @@ import random import pytest from functools import partial +from server.models import Hostname @pytest.mark.parametrize( @@ -39,3 +40,58 @@ def test_vulnerability_count(with_host_vulns, with_service_vulns, host, session.commit() assert host.vulnerability_count == expected_count + + +class TestUpdateHostnames: + """This class tests the generic set_children_objects function and + not only the Host model logic. Think twice if you are going to + remove this""" + # set_children = partial(set_children_objects, attr='hostnames', key='name') + + def test_set_from_empty_host(self, host, session): + session.commit() + assert len(host.hostnames) == 0 + host.set_hostnames(['test.com', 'other.com']) + assert len(session.new) == 2 + session.commit() + assert len(host.hostnames) == 2 + assert {hn.name for hn in host.hostnames} == {'test.com', 'other.com'} + + def test_set_to_empty_host(self, host_with_hostnames, session): + session.commit() + n_hostnames = len(host_with_hostnames.hostnames) + host_with_hostnames.set_hostnames([]) + assert len(session.deleted) == n_hostnames + session.commit() + assert host_with_hostnames.hostnames == [] + + def test_stays_equal(self, host_with_hostnames, session): + new_value = [hn.name for hn in host_with_hostnames.hostnames] + random.shuffle(new_value) + host_with_hostnames.set_hostnames(new_value) + session.commit() + assert len(session.new) == len(session.deleted) == len( + session.dirty) == 0 + assert set(new_value) == set(hn.name for hn in + host_with_hostnames.hostnames) + + def test_all(self, host, session): + a = Hostname(workspace=host.workspace, host=host, name='a') + b = Hostname(workspace=host.workspace, host=host, name='b') + session.add(a) + session.add(b) + session.commit() + + host.set_hostnames(['b', 'c']) + + # a should be deleted + assert len(session.deleted) == 1 + assert session.deleted.pop() is a + + # c should be created + assert len(session.new) == 1 + c = session.new.pop() + assert c.name == 'c' + + session.commit() + assert set(hn.name for hn in host.hostnames) == {'b', 'c'} diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index a4db68f599a..489ed7336dc 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -321,7 +321,7 @@ def test_host_services_vuln_count_verification(self, test_client, session, def test_create_host_with_hostnames(self, test_client): raw_data = { "ip": "192.168.0.21", - "hostnames": [{"key": "google.com"}], + "hostnames": ["google.com"], "mac": "00:00:00:00:00:00", "description": "", "os": "", @@ -330,6 +330,28 @@ def test_create_host_with_hostnames(self, test_client): } res = test_client.post(self.url(), data=raw_data) assert res.status_code == 201 + assert res.json['hostnames'] == ['google.com'] + host = Host.query.get(res.json['id']) + assert len(host.hostnames) == 1 + assert host.hostnames[0].name == 'google.com' + + def test_update_host_with_hostnames(self, test_client, session, + host_with_hostnames): + session.commit() + data = { + "ip": "192.168.0.21", + "hostnames": ["other.com", "test.com"], + "mac": "00:00:00:00:00:00", + "description": "", + "os": "", + "owned": False, + "owner": "" + } + res = test_client.put(self.url(host_with_hostnames), data=data) + assert res.status_code == 200 + expected = set(["other.com", "test.com"]) + assert set(res.json['hostnames']) == expected + assert set(hn.name for hn in host_with_hostnames.hostnames) == expected def test_create_host_with_default_gateway(self, test_client): raw_data = { From c636a27211186f9e382ea23c98107fa791b8162b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 5 Dec 2017 19:44:29 -0300 Subject: [PATCH 0615/1506] Fix createAndAddNoteToNote (deleted params by mistake). Some PEP8 fixes --- persistence/server/server.py | 5 ++++- plugins/plugin.py | 26 ++++++++++++++------------ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/persistence/server/server.py b/persistence/server/server.py index 4e2b55e095c..0b8d8ad306c 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -24,8 +24,11 @@ """ import os -import requests import json +import logging + +import requests + from persistence.server.utils import force_unique from persistence.server.server_io_exceptions import (WrongObjectSignature, CantCommunicateWithServerError, diff --git a/plugins/plugin.py b/plugins/plugin.py index 3cb12fc84ff..342e1219958 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -12,21 +12,20 @@ import os import re import traceback -import multiprocessing import deprecation import server.config import model.api import model.common from model.common import factory -from persistence.server.models import (Host, - Service, - Vuln, - VulnWeb, - Credential, - Note, - Command - ) +from persistence.server.models import ( + Host, + Service, + Vuln, + VulnWeb, + Credential, + Note, +) from plugins.modelactions import modelactions from config.configuration import getInstanceConfiguration @@ -157,7 +156,9 @@ def createAndAddHost(self, name, os="unknown"): host_obj = factory.createModelObject( Host.class_signature, - name, os=os, parent_id=None) + name, + os=os, + parent_id=None) host_obj._metadata.creatoserverr = self.id self.__addPendingAction(modelactions.ADDHOST, host_obj) return host_obj.getID() @@ -283,14 +284,15 @@ def createAndAddNoteToService(self, host_id, service_id, name, text): self.__addPendingAction(modelactions.ADDNOTESRV, note_obj) return note_obj.getID() - def createAndAddNoteToNote(self, host_id, name, text): + def createAndAddNoteToNote(self, host_id, service_id, note_id, name, text): note_obj = model.common.factory.createModelObject( Note.class_signature, name, text=text, parent_id=note_id) note_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDNOTENOTE, host_id, note_obj) + + self.__addPendingAction(modelactions.ADDNOTENOTE, note_obj) return note_obj.getID() def createAndAddCredToService(self, host_id, service_id, username, From 075e984fa99a07314ca142cd21864f9202a4c46b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 6 Dec 2017 12:06:00 -0300 Subject: [PATCH 0616/1506] Add support to create vulns from plugins * Added required parent and parent type to vuln jsons. * Fix a bug on vuln endpoint when no attachment was sent in the json --- persistence/server/models.py | 4 ++++ persistence/server/server.py | 7 ++++-- persistence/server/utils.py | 41 ++++++++++++++++++++---------------- plugins/plugin.py | 30 ++++++++++++++++---------- server/api/modules/vulns.py | 2 +- 5 files changed, 52 insertions(+), 32 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index 5347b3d5489..0c2f4a8f6c6 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -934,6 +934,10 @@ def __init__(self, vuln, workspace_name): self.resolution = vuln.get('resolution') self.status = vuln.get('status', "opened") self.policyviolations = vuln.get('policyviolations', list()) + self.parent_type = vuln.get('parent_type') + + def getParentType(self): + return self.parent_type @staticmethod def publicattrsrefs(): diff --git a/persistence/server/server.py b/persistence/server/server.py index 0b8d8ad306c..e2f597c7fce 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -1060,8 +1060,9 @@ def update_service(workspace_name, id, name, description, ports, metadata=metadata) -def create_vuln(workspace_name, name, description, owned=None, owner="", - confirmed=False, data="", refs=None, severity="info", resolution="", +def create_vuln(workspace_name, name, description, parent, parent_type, + owned=None, owner="", confirmed=False, + resolution="", data="", refs=None, severity="info", desc="", metadata=None, status=None, policyviolations=[]): """Creates a vuln. @@ -1090,6 +1091,8 @@ def create_vuln(workspace_name, name, description, owned=None, owner="", return _save_to_server(workspace_name, name=name, description=description, + parent=parent, + parent_type=parent_type, owned=owned, owner=owner, confirmed=confirmed, diff --git a/persistence/server/utils.py b/persistence/server/utils.py index 5845d552850..d85653f3e84 100644 --- a/persistence/server/utils.py +++ b/persistence/server/utils.py @@ -65,28 +65,33 @@ def get_service_properties(service): def get_vuln_properties(vuln): - vuln_dict = {'confirmed': vuln.getConfirmed(), - 'data': vuln.getData(), - 'refs': vuln.getRefs(), - 'severity': vuln.getSeverity(), - 'resolution': vuln.getResolution(), - 'desc': vuln.getDesc(), - 'status': vuln.getStatus(), - 'policyviolations': vuln.getPolicyViolations()} + vuln_dict = { + 'confirmed': vuln.getConfirmed(), + 'data': vuln.getData(), + 'refs': vuln.getRefs(), + 'severity': vuln.getSeverity(), + 'resolution': vuln.getResolution(), + 'desc': vuln.getDesc(), + 'status': vuln.getStatus(), + 'policyviolations': vuln.getPolicyViolations(), + 'parent': vuln.getParent(), + 'parent_type': vuln.getParentType(), + } vuln_dict.update(get_object_properties(vuln)) return vuln_dict def get_vuln_web_properties(vuln_web): - vuln_web_dict = {'method': vuln_web.getMethod(), - 'params': vuln_web.getParams(), - 'request': vuln_web.getRequest(), - 'response': vuln_web.getResponse(), - 'website': vuln_web.getWebsite(), - 'path': vuln_web.getPath(), - 'pname': vuln_web.getPname(), - 'query': vuln_web.getQuery(), - 'status': vuln_web.getStatus() - } + vuln_web_dict = { + 'method': vuln_web.getMethod(), + 'params': vuln_web.getParams(), + 'request': vuln_web.getRequest(), + 'response': vuln_web.getResponse(), + 'website': vuln_web.getWebsite(), + 'path': vuln_web.getPath(), + 'pname': vuln_web.getPname(), + 'query': vuln_web.getQuery(), + 'status': vuln_web.getStatus() + } vuln_web_dict.update(get_object_properties(vuln_web)) vuln_web_dict.update(get_vuln_properties(vuln_web)) return vuln_web_dict diff --git a/plugins/plugin.py b/plugins/plugin.py index 342e1219958..990e9665009 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -189,7 +189,8 @@ def createAndAddServiceToInterface(self, host_id, interface_id, name, serv_obj = model.common.factory.createModelObject( Service.class_signature, name, protocol=protocol, ports=ports, status=status, - version=version, description=description, parent_id=host_id) + version=version, description=description, + parent_type='Service', parent_id=host_id) serv_obj._metadata.creator = self.id self.__addPendingAction(modelactions.ADDSERVICEHOST, serv_obj) @@ -200,8 +201,9 @@ def createAndAddVulnToHost(self, host_id, name, desc="", ref=[], vuln_obj = model.common.factory.createModelObject( Vuln.class_signature, - name, desc=desc, refs=ref, severity=severity, resolution=resolution, - confirmed=False, parent_id=host_id) + name, desc=desc, refs=ref, severity=severity, + resolution=resolution, confirmed=False, + parent_id=host_id, parent_type='Host') vuln_obj._metadata.creator = self.id self.__addPendingAction(modelactions.ADDVULNHOST, vuln_obj) @@ -216,8 +218,9 @@ def createAndAddVulnToInterface(self, host_id, interface_id, name, vuln_obj = model.common.factory.createModelObject( Vuln.class_signature, - name, desc=desc, refs=ref, severity=severity, resolution=resolution, - confirmed=False, parent_id=interface_id) + name, desc=desc, refs=ref, severity=severity, + resolution=resolution, confirmed=False, + parent_type='Host', parent_id=host_id) vuln_obj._metadata.creator = self.id self.__addPendingAction(modelactions.ADDVULNHOST, vuln_obj) @@ -228,8 +231,10 @@ def createAndAddVulnToService(self, host_id, service_id, name, desc="", vuln_obj = model.common.factory.createModelObject( Vuln.class_signature, - name, desc=desc, refs=ref, severity=severity, resolution=resolution, - confirmed=False, parent_id=service_id) + name, desc=desc, refs=ref, severity=severity, + resolution=resolution, confirmed=False, + parent_type='Service', parent_id=service_id + ) vuln_obj._metadata.creator = self.id self.__addPendingAction(modelactions.ADDVULNSRV, vuln_obj) @@ -242,10 +247,13 @@ def createAndAddVulnWebToService(self, host_id, service_id, name, desc="", params="", query="", category=""): vulnweb_obj = model.common.factory.createModelObject( VulnWeb.class_signature, - name, desc=desc, refs=ref, severity=severity, resolution=resolution, - website=website, path=path, request=request, response=response, - method=method, pname=pname, params=params, query=query, - category=category, confirmed=False, parent_id=service_id) + name, desc=desc, refs=ref, severity=severity, + resolution=resolution, website=website, path=path, + request=request, response=response, method=method, + pname=pname, params=params, query=query, + category=category, confirmed=False, parent_id=service_id, + parent_type='Service', + ) vulnweb_obj._metadata.creator = self.id self.__addPendingAction(modelactions.ADDVULNWEBSRV, vulnweb_obj) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 774a3546e37..31a739fb9f6 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -314,7 +314,7 @@ def _perform_create(self, data, **kwargs): request) # TODO migration: use default values when popping and validate the # popped object has the expected type. - attachments = data.pop('_attachments') + attachments = data.pop('_attachments', {}) # This will be set after setting the workspace references = data.pop('references') From 75a981bf64b97cb70279e230260e25a1bf5cfd2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 6 Dec 2017 13:55:09 -0300 Subject: [PATCH 0617/1506] Fix hostnames in web ui --- server/www/scripts/hosts/controllers/host.js | 15 +++++++++++---- server/www/scripts/hosts/controllers/new.js | 8 ++++++++ server/www/scripts/services/partials/list.html | 4 ++-- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/server/www/scripts/hosts/controllers/host.js b/server/www/scripts/hosts/controllers/host.js index 2dd823da79b..7a9b02a4848 100644 --- a/server/www/scripts/hosts/controllers/host.js +++ b/server/www/scripts/hosts/controllers/host.js @@ -12,7 +12,10 @@ angular.module('faradayApp') loadHosts = function(){ hostsManager.getHost($routeParams.hidId, $scope.workspace, true) .then(function(host) { - $scope.host = host; + $scope.host = host; + $scope.host.hostnames = $scope.host.hostnames.map(function(hostname){ + return {key: hostname} + }); $scope.hostName = host.ip; // User can edit $scope.host.name but not $scope.hostName $scope.loadIcons(); }); @@ -103,10 +106,11 @@ angular.module('faradayApp') var date = new Date(), timestamp = date.getTime()/1000.0; - // The objectToArray transform is necessary to call updateHost correctly - // If I don't restore the object after the call hostnames won't be shown in the host + // The API expects list of strings in hostnames var old_hostnames = $scope.host.hostnames; - $scope.host.hostnames = commons.objectToArray($scope.host.hostnames.filter(Boolean)); + $scope.host.hostnames = $scope.host.hostnames.map(function(hostname){ + return hostname.key + }).filter(Boolean); $scope.hostdata = $scope.host; $scope.hostdata.metadata['update_time'] = timestamp; @@ -114,8 +118,11 @@ angular.module('faradayApp') hostsManager.updateHost($scope.host, $scope.hostdata, $scope.workspace).then(function(){ + $scope.host.hostnames = old_hostnames; $scope.hostnames = old_hostnames; $location.path('/host/ws/' + $scope.workspace + '/hid/' + $scope.host._id); + }, function(){ + $scope.host.hostnames = old_hostnames; }); }; diff --git a/server/www/scripts/hosts/controllers/new.js b/server/www/scripts/hosts/controllers/new.js index 08005b67c0b..26aa3071c77 100644 --- a/server/www/scripts/hosts/controllers/new.js +++ b/server/www/scripts/hosts/controllers/new.js @@ -37,9 +37,17 @@ angular.module('faradayApp') }; $scope.insert = function(hostdata) { + // The API expects list of strings in hostnames + var old_hostnames = $scope.host.hostnames; + $scope.host.hostnames = $scope.host.hostnames.map(function(hostname){ + return hostname.key + }).filter(Boolean); + hostsManager.createHost(hostdata, $scope.workspace).then(function(host) { + $scope.host.hostnames = old_hostnames; $location.path('/host/ws/' + $scope.workspace + '/hid/' + host.data.id); }, function(message) { + $scope.host.hostnames = old_hostnames; $uibModal.open({ templateUrl: 'scripts/commons/partials/modalKO.html', controller: 'commonsModalKoCtrl', diff --git a/server/www/scripts/services/partials/list.html b/server/www/scripts/services/partials/list.html index 6a53f19d105..19e6020e85b 100644 --- a/server/www/scripts/services/partials/list.html +++ b/server/www/scripts/services/partials/list.html @@ -187,7 +187,7 @@

    Hostnames
    - +
    @@ -195,7 +195,7 @@
    Hostnames
      -
    • {{hostname}}
      • +
    • {{hostname.key}}
     From f8ee86b7cd16de4b32542a4c49a0f3d1b0792da0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 6 Dec 2017 14:00:03 -0300 Subject: [PATCH 0618/1506] Add MAC field to the Hosts API --- server/api/modules/hosts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 50c05702c94..9101c5aeff7 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -53,7 +53,7 @@ class HostSchema(AutoSchema): class Meta: model = Host - fields = ('id', '_id', '_rev', 'ip', 'description', + fields = ('id', '_id', '_rev', 'ip', 'description', 'mac', 'credentials', 'default_gateway', 'metadata', 'name', 'os', 'owned', 'owner', 'services', 'vulns', 'hostnames' From a926a86a81a3a374ddf26ab94ff71fec4b5ed652 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 6 Dec 2017 14:18:22 -0300 Subject: [PATCH 0619/1506] Rename CouchDBConnectionProblem to DBConnectionProblem --- gui/gtk/server.py | 2 +- gui/notifier.py | 2 +- managers/workspace_manager.py | 2 +- model/application.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gui/gtk/server.py b/gui/gtk/server.py index a9a9314e6fc..f347dfdd43e 100644 --- a/gui/gtk/server.py +++ b/gui/gtk/server.py @@ -195,7 +195,7 @@ def test_server_connection(): else: tolerance += 1 if tolerance == 3: - notification_center.CouchDBConnectionProblem() + notification_center.DBConnectionProblem() test_server_thread = threading.Thread(target=test_server_connection) test_server_thread.daemon = True diff --git a/gui/notifier.py b/gui/notifier.py index 16a28bc881c..0b399fbb273 100644 --- a/gui/notifier.py +++ b/gui/notifier.py @@ -54,7 +54,7 @@ def showDialog(self, msg, level="INFORMATION"): def workspaceChanged(self, workspace): self._notifyWidgets(events.WorkspaceChangedCustomEvent(workspace)) - def CouchDBConnectionProblem(self, problem=None): + def DBConnectionProblem(self, problem=None): self._notifyWidgets(events.ShowExceptionConnectionRefusedCustomEvent(problem)) def WorkspaceProblem(self, problem=None): diff --git a/managers/workspace_manager.py b/managers/workspace_manager.py index e821a736ccd..ca4dbf76725 100644 --- a/managers/workspace_manager.py +++ b/managers/workspace_manager.py @@ -78,7 +78,7 @@ def openWorkspace(self, name): "For example: " "http://john:password@127.0.0.1:5984")) except Exception as e: - notification_center.CouchDBConnectionProblem(e) + notification_center.DBConnectionProblem(e) raise WorkspaceException(str(e)) self.mappersManager.createMappers(name) self.setActiveWorkspace(workspace) diff --git a/model/application.py b/model/application.py index 74aea8bfe1f..0aaabb1169b 100644 --- a/model/application.py +++ b/model/application.py @@ -130,7 +130,7 @@ def __init__(self, args): def on_connection_lost(self): """All it does is send a notification to the notification center""" - model.guiapi.notification_center.CouchDBConnectionProblem() + model.guiapi.notification_center.DBConnectionProblem() def enableExceptHook(self): sys.excepthook = exception_handler From 6eaf8451b4afb4ec26a351878381fccb848955cf Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 6 Dec 2017 14:18:48 -0300 Subject: [PATCH 0620/1506] Update code to use Queue as param --- model/application.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/model/application.py b/model/application.py index 0aaabb1169b..43add6cc519 100644 --- a/model/application.py +++ b/model/application.py @@ -10,16 +10,18 @@ import signal import json import threading -import requests - from json import loads from time import sleep +from Queue import Queue + +import requests from model.controller import ModelController from managers.workspace_manager import WorkspaceManager from plugins.controller import PluginController from persistence.server.server import login_user +from utils.logs import setUpLogger import model.api import model.guiapi import apis.rest.api as restapi @@ -76,6 +78,7 @@ def stop(self): class MainApplication(object): def __init__(self, args): + setUpLogger() self._original_excepthook = sys.excepthook self.args = args @@ -99,8 +102,8 @@ def __init__(self, args): logger.error("Credentials file couldn't be loaded") self._mappers_manager = MapperManager() - - self._model_controller = ModelController(self._mappers_manager) + pending_actions = Queue() + self._model_controller = ModelController(self._mappers_manager, pending_actions) self._plugin_manager = PluginManager( os.path.join(CONF.getConfigPath(), "plugins")) @@ -112,7 +115,8 @@ def __init__(self, args): self._plugin_controller = PluginController( 'PluginController', self._plugin_manager, - self._mappers_manager + self._mappers_manager, + pending_actions ) if self.args.cli: From 67947e7b4180bd5e14beaca4e14dc261cccce659 Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 6 Dec 2017 15:27:00 -0300 Subject: [PATCH 0621/1506] Fix test for feature implemented a few commits back --- test_cases/test_marshmallow_fields.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test_cases/test_marshmallow_fields.py b/test_cases/test_marshmallow_fields.py index eb3aaf3fb9c..a9cad9eaf0b 100644 --- a/test_cases/test_marshmallow_fields.py +++ b/test_cases/test_marshmallow_fields.py @@ -60,8 +60,13 @@ def test_parses_null_datetime(self): assert JSTimestampField()._serialize(None, None, None) is None def test_deserialization_fails(self): - with pytest.raises(NotImplementedError): - JSTimestampField()._deserialize(time.time() * 1000, None, None) + ts = time.time() + dt = datetime.datetime.fromtimestamp(ts) + loaded = JSTimestampField()._deserialize(ts * 1000, + None, + None) + assert isinstance(loaded, datetime.date) + assert abs(loaded - dt) < datetime.timedelta(seconds=60) User = namedtuple('User', ['username', 'blogposts']) From 4c5a94649b4de401cb308f914bfd95553a8de1d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 6 Dec 2017 16:11:29 -0300 Subject: [PATCH 0622/1506] Fix host sort tests The sort by description was failing in postgres due to case sentitiveness issues. The sort by services tests was badly written --- test_cases/test_api_hosts.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index a4db68f599a..eaff7bcde2f 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -411,6 +411,9 @@ class TestHostAPIGeneric(ReadWriteAPITests, PaginationTestsMixin): @pytest.mark.usefixtures("mock_envelope_list") def test_sort_by_description(self, test_client, session): + for host in Host.query.all(): + # I don't want to test case sensitive sorting + host.description = host.description.lower() session.commit() expected_ids = [host.id for host in sorted(Host.query.all(), @@ -430,8 +433,8 @@ def test_sort_by_services(self, test_client, session, second_workspace, expected_ids = [] for i in range(10): host = host_factory.create(workspace=second_workspace) - service_factory.create(host=host, workspace=second_workspace, - status='open') + service_factory.create_batch( + i, host=host, workspace=second_workspace, status='open') session.flush() expected_ids.append(host.id) session.commit() From 2daedeb69e3d511e6baf97c70d347eb8dd222c0b Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 6 Dec 2017 16:41:36 -0300 Subject: [PATCH 0623/1506] Remove Licenses DB check With CouchDB, we had to check whether the database was already created or not. That is no longer a problem. --- .../scripts/licenses/controllers/licenses.js | 29 ++++------------- .../licenses/controllers/modalCreateDB.js | 31 ------------------- .../www/scripts/licenses/partials/list.html | 2 +- .../licenses/partials/modalCreateDB.html | 12 ------- ...kspaced_various.py => test_api_license.py} | 0 5 files changed, 7 insertions(+), 67 deletions(-) delete mode 100644 server/www/scripts/licenses/controllers/modalCreateDB.js delete mode 100644 server/www/scripts/licenses/partials/modalCreateDB.html rename test_cases/{test_api_non_workspaced_various.py => test_api_license.py} (100%) diff --git a/server/www/scripts/licenses/controllers/licenses.js b/server/www/scripts/licenses/controllers/licenses.js index d979ea610c5..e6857e3cf23 100644 --- a/server/www/scripts/licenses/controllers/licenses.js +++ b/server/www/scripts/licenses/controllers/licenses.js @@ -7,7 +7,6 @@ angular.module('faradayApp') ['$scope', '$filter', '$q', '$uibModal', 'commonsFact', 'licensesManager', function($scope, $filter, $q, $uibModal, commonsFact, licensesManager) { - $scope.db_exists = false; $scope.expiration_month = false; $scope.licenses = []; $scope.loaded_licenses = false; @@ -25,28 +24,12 @@ angular.module('faradayApp') $scope.sort_field = "end"; $scope.reverse = true; - licensesManager.DBExists() - .then(function(exists) { - if(!exists) { - $uibModal.open({ - templateUrl: 'scripts/licenses/partials/modalCreateDB.html', - controller: 'licensesModalCreateDB', - size: 'lg' - }).result.then(function() { - $scope.db_exists = true; - }, function(message) { - // The user didn't create the DB, do nothing! - }); - } else { - $scope.db_exists = true; - licensesManager.get() - .then(function() { - $scope.licenses = licensesManager.licenses; - $scope.loaded_licenses = true; - - $scope.expiration_month = $scope.isExpirationMonth($scope.licenses); - }); - } + licensesManager.get() + .then(function() { + $scope.licenses = licensesManager.licenses; + $scope.loaded_licenses = true; + + $scope.expiration_month = $scope.isExpirationMonth($scope.licenses); }, function(message) { commonsFact.errorDialog(message); }); diff --git a/server/www/scripts/licenses/controllers/modalCreateDB.js b/server/www/scripts/licenses/controllers/modalCreateDB.js deleted file mode 100644 index 7bdc2b42df3..00000000000 --- a/server/www/scripts/licenses/controllers/modalCreateDB.js +++ /dev/null @@ -1,31 +0,0 @@ -// Faraday Penetration Test IDE -// Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) -// See the file 'doc/LICENSE' for the license information - -angular.module('faradayApp') - .controller('licensesModalCreateDB', - ['$scope', '$modalInstance', 'licensesManager', - function($scope, $modalInstance, licensesManager) { - - $scope.message; - - var init = function() { - $scope.message = "It looks like your Faraday installation is missing "+ - "the Licenses database. Would you like to create it now?"; - }; - - $scope.yes = function() { - licensesManager.createDB() - .then(function() { - $modalInstance.close($scope.data); - }, function() { - $scope.message = "There's been a problem creating the database."; - }); - }; - - $scope.no = function() { - $modalInstance.dismiss('cancel'); - }; - - init(); - }]); diff --git a/server/www/scripts/licenses/partials/list.html b/server/www/scripts/licenses/partials/list.html index fe303e4ab0f..39de606e741 100644 --- a/server/www/scripts/licenses/partials/list.html +++ b/server/www/scripts/licenses/partials/list.html @@ -3,7 +3,7 @@
    -
    +

    Licenses ({{licenses.length}}) - -

    \ No newline at end of file diff --git a/test_cases/test_api_non_workspaced_various.py b/test_cases/test_api_license.py similarity index 100% rename from test_cases/test_api_non_workspaced_various.py rename to test_cases/test_api_license.py From 21be0b701bf1e930f1b45f302e4af8062ac90b72 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 6 Dec 2017 17:22:51 -0300 Subject: [PATCH 0624/1506] Minor fixes to make gtk ui work with new faraday-server --- gui/gtk/application.py | 4 +++- gui/gtk/mainwidgets.py | 9 ++++----- persistence/server/models.py | 5 +++-- persistence/server/server.py | 4 ++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/gui/gtk/application.py b/gui/gtk/application.py index 3f4a246cf68..cfd3b9ff45f 100644 --- a/gui/gtk/application.py +++ b/gui/gtk/application.py @@ -388,7 +388,7 @@ def handle_connection_lost(self, button=None, dialog=None): def update_counts(self): """Returns the counts of hosts, services and vulns on the current workspace.""" - hosts, interfaces, services, vulns = self.serverIO.get_workspace_numbers() + hosts, services, vulns = self.serverIO.get_workspace_numbers() return hosts, services, vulns def show_host_info(self, host_id): @@ -886,6 +886,8 @@ def on_file_selected(plugin_id, report): self.show_normal_error("You are not authorized to write data " "to this workspace.") + plugin_response, plugin_id = select_plugin() + while plugin_response == Gtk.ResponseType.ACCEPT and plugin_id is None: # force user to select a plugin if he did not do it errorDialog(self.window, diff --git a/gui/gtk/mainwidgets.py b/gui/gtk/mainwidgets.py index 9ca430bd819..9734f1d2dc9 100644 --- a/gui/gtk/mainwidgets.py +++ b/gui/gtk/mainwidgets.py @@ -222,9 +222,8 @@ def __decide_icon(self, os): def _find_host_id(self, object_): """Return the ID of the object's parent host.""" - object_id = object_.getID() - host_id = object_id.split(".")[0] - return host_id + if object_.getParentType() == 'Host': + return object_.getParent() def _is_host_in_model_by_host_id(self, host_id): """Return a boolean indicating if host_id is in the model""" @@ -255,7 +254,7 @@ def _add_single_host_to_model(self, host): vuln_count = host.getVulnAmount() os_icon, os_str = self.__decide_icon(host.getOS()) display_str = str(host) - host_iter = self.model.append([host.id, os_icon, os_str, display_str, vuln_count]) + host_iter = self.model.append([str(host.id), os_icon, os_str, display_str, vuln_count]) self.host_id_to_iter[host.id] = host_iter self.host_amount_in_model += 1 @@ -346,7 +345,7 @@ def add_relevant_vulns_to_model(self, vulns): """Takes vulns, a list of vulnerability object, and adds them to the model by modifying their corresponding hosts in the model. Return None. """ - host_ids = map(self._find_host_id, vulns) + host_ids = [host_id for host_id in map(self._find_host_id, vulns) if host_id is not None] self._modify_vuln_amounts_of_hosts_in_model(host_ids, lambda x: x + 1) def remove_relevant_vulns_from_model(self, vulns_ids): diff --git a/persistence/server/models.py b/persistence/server/models.py index 2a393975011..cfee930adfc 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -1323,8 +1323,9 @@ def __init__(self, workspace, workspace_name): self.name = workspace['name'] self.description = workspace['description'] self.customer = workspace['customer'] - self.start_date = workspace['start_date'] - self.end_date = workspace['end_date'] + if 'duration' in workspace: + self.start_date = workspace['duration']['start_date'] + self.end_date = workspace['duration']['end_date'] def getID(self): return self._id diff --git a/persistence/server/server.py b/persistence/server/server.py index 3afd47b706f..5ee217a624f 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -757,7 +757,7 @@ def get_workspace_numbers(workspace_name): A tuple of 4 elements with the amounts of hosts, interfaces, services and vulns. """ stats = _get_raw_workspace_summary(workspace_name)['stats'] - return stats['hosts'], stats['interfaces'], stats['services'], stats['total_vulns'] + return stats['hosts'], stats['services'], stats['total_vulns'] def get_hosts_number(workspace_name, **params): """ @@ -1559,7 +1559,7 @@ def test_server_url(url_to_test): False otherwise. """ try: - _get("{0}/_api/info".format(url_to_test)) + _get("{0}/v2/_api/info".format(url_to_test)) test_okey = True except: test_okey = False From 2f142e29f335a918a109cd8236a70643a359b368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 6 Dec 2017 18:24:13 -0300 Subject: [PATCH 0625/1506] Remove CWE field from vuln template modal It isn't used anymore after the migration. Also improved update error messages on the Web UI --- RELEASE.md | 1 + server/www/scripts/vulndb/partials/modalEdit.html | 6 ------ server/www/scripts/vulndb/partials/modalNew.html | 6 ------ server/www/scripts/vulndb/providers/vulnModel.js | 2 +- 4 files changed, 2 insertions(+), 13 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index d3b273b41bb..c23a478c926 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -16,6 +16,7 @@ TBA: * Note object was removed and replaced with Comment * Communication object was removed and replaced with Comment * Show credentials count in summarized report on the dashboard +* Remove vuln template CWE fields, join it with references November 17, 2017: --- diff --git a/server/www/scripts/vulndb/partials/modalEdit.html b/server/www/scripts/vulndb/partials/modalEdit.html index 33b0a12f649..ea3e0214e2d 100644 --- a/server/www/scripts/vulndb/partials/modalEdit.html +++ b/server/www/scripts/vulndb/partials/modalEdit.html @@ -23,12 +23,6 @@

    Edit {{model.name}} Vulnerability Model

    -
    -
    - - -
    -
    diff --git a/server/www/scripts/vulndb/partials/modalNew.html b/server/www/scripts/vulndb/partials/modalNew.html index e7d053513ea..1bd3c546d2f 100644 --- a/server/www/scripts/vulndb/partials/modalNew.html +++ b/server/www/scripts/vulndb/partials/modalNew.html @@ -23,12 +23,6 @@

    New Vulnerability Model

    -
    -
    - - -
    -
    diff --git a/server/www/scripts/vulndb/providers/vulnModel.js b/server/www/scripts/vulndb/providers/vulnModel.js index dbdaae6b015..baa471b28cd 100644 --- a/server/www/scripts/vulndb/providers/vulnModel.js +++ b/server/www/scripts/vulndb/providers/vulnModel.js @@ -66,7 +66,7 @@ angular.module('faradayApp'). self.set(res.data); deferred.resolve(self); }, function(res) { - deferred.reject("Unable to update the Vuln Model. " + res.data.reason); + deferred.reject("Unable to update the Vuln Model. " + JSON.stringify(res.data)); }); return deferred.promise; }, From 766ceb2d30df9655b1b796eafbf20e9484546b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 6 Dec 2017 18:25:05 -0300 Subject: [PATCH 0626/1506] Avoid creating empty references when importing vuln templates --- server/importer.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index ff46cb36c8c..5ff2465e155 100644 --- a/server/importer.py +++ b/server/importer.py @@ -954,7 +954,18 @@ def run(self): vuln_template.resolution = document.get('resolution') vuln_template.severity = new_severity - references = document['references'] if isinstance(document['references'], list) else [x.strip() for x in document['references'].split(',')] + if isinstance(document['references'], list): + references = document['references'] + elif isinstance(document['references'], (str, unicode)): + references = [x.strip() + for x in document['references'].split(',') + if x.strip()] + else: + logger.warn("Unknown type of vuln template references: {}. " + "Reference data: {}".format( + type(document['references']), + document)) + continue cwe_field = document.get('cwe') if cwe_field not in references: references.append(cwe_field) From d43d96f92dfc467bb1b6df18f7720314353ed55b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 7 Dec 2017 12:43:20 -0300 Subject: [PATCH 0627/1506] Fix create workspace and code cleanup --- persistence/server/server.py | 14 +++----------- server/commands/reports.py | 2 -- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/persistence/server/server.py b/persistence/server/server.py index 5ee217a624f..349b03818aa 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -136,7 +136,7 @@ def _create_couch_db_url(workspace_name): def _create_server_db_url(workspace_name): server_api_url = _create_server_api_url() - db_url = '{0}/ws/{1}'.format(server_api_url, workspace_name) + db_url = '{0}/ws/'.format(server_api_url) return db_url def _add_session_cookies(func): @@ -284,17 +284,9 @@ def _get_raw_commands(workspace_name, **params): def _get_raw_workspace_summary(workspace_name): - request_url = _create_server_get_url(workspace_name, 'summary') + request_url = _create_server_get_url(workspace_name) return _get(request_url) -def _save_to_couch(workspace_name, faraday_object_id, **params): - post_url = _create_couch_post_url(workspace_name, faraday_object_id) - return _post(post_url, update=False, **params) - -def _update_in_couch(workspace_name, faraday_object_id, **params): - post_url = _create_server_put_url(workspace_name, faraday_object_id) - return _put(post_url, **params) - def _save_to_server(workspace_name, **params): post_url = _create_server_post_url(workspace_name, params['type']) return _post(post_url, update=False, expected_response=201, **params) @@ -305,7 +297,7 @@ def _update_in_server(workspace_name, faraday_object_id, **params): def _save_db_to_server(db_name, **params): post_url = _create_server_db_url(db_name) - return _put(post_url, expected_response=200, **params) + return _post(post_url, expected_response=201, **params) # XXX: SEMI COUCH IT! def _delete_from_couch(workspace_name, faraday_object_id): diff --git a/server/commands/reports.py b/server/commands/reports.py index 25fad24dc9c..bb46c703347 100644 --- a/server/commands/reports.py +++ b/server/commands/reports.py @@ -23,8 +23,6 @@ def import_external_reports(workspace_name=None): os.path.join(CONF.getConfigPath(), "plugins")) mappers_manager = MapperManager() - - if workspace_name: query = Workspace.query.filter_by(name=workspace_name) else: From c9ea72735feae35918df121025158e609dd28a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 11 Dec 2017 14:59:07 -0300 Subject: [PATCH 0628/1506] Fix vuln template csv import Fix builtin CSV files so all use valid severity values. Also, I added a script to do this with custom CSVs. --- data/cwe.csv | 1684 ++++++++++++++++++++-------------------- data/cwe_all.csv | 1684 ++++++++++++++++++++-------------------- data/cwe_en.csv | 1504 +++++++++++++++++------------------ data/cwe_es.csv | 174 ++--- data/fix_severities.py | 63 ++ 5 files changed, 2586 insertions(+), 2523 deletions(-) create mode 100644 data/fix_severities.py diff --git a/data/cwe.csv b/data/cwe.csv index 2c446633647..7941463b896 100644 --- a/data/cwe.csv +++ b/data/cwe.csv @@ -1,7 +1,7 @@ cwe,name,description,resolution,exploitation,references CWE-119,EN-Improper Restriction of Operations within the Bounds of a Memory Buffer (Type: Class),"The software performs operations on a memory buffer, but it can read from or write to a memory location that is outside of the intended boundary of the buffer. Certain languages allow direct addressing of memory locations and do not automatically ensure that these locations are valid for the memory buffer that is being referenced. This can cause read or write operations to be performed on memory locations that may be associated with other variables, data structures, or internal program data. -As a result, an attacker may be able to execute arbitrary code, alter the intended control flow, read sensitive information, or cause the system to crash.",,High,"Writing Secure Code: Chapter 5, ""Public Enemy #1: The Buffer Overrun"" Page 127; Chapter 14, ""Prevent I18N Buffer Overruns"" Page 441 +As a result, an attacker may be able to execute arbitrary code, alter the intended control flow, read sensitive information, or cause the system to crash.",,high,"Writing Secure Code: Chapter 5, ""Public Enemy #1: The Buffer Overrun"" Page 127; Chapter 14, ""Prevent I18N Buffer Overruns"" Page 441 Using the Strsafe.h Functions: http://msdn.microsoft.com/en-us/library/ms647466.aspx Safe C String Library v1.0.3: http://www.zork.org/safestr/ Address Space Layout Randomization in Windows Vista: http://blogs.msdn.com/michael_howard/archive/2006/05/26/address-space-layout-randomization-in-windows-vista.aspx @@ -11,9 +11,9 @@ Understanding DEP as a mitigation technology part 1: http://blogs.technet.com/b/ The Art of Software Security Assessment: Chapter 5, ""Memory Corruption"", Page 167. The Art of Software Security Assessment: Chapter 5, ""Protection Mechanisms"", Page 189." CWE-123,EN-Write-what-where Condition (Type: Base),"Any condition where the attacker has the ability to write an arbitrary value to an arbitrary location, often as the result of a buffer overflow. -A buffer overflow condition exists when a program attempts to put more data in a buffer than it can hold, or when a program attempts to put data in a memory area outside of the boundaries of a buffer. The simplest type of error, and the most common cause of buffer overflows, is the ""classic"" case in which the program copies the buffer without restricting how much is copied. Other variants exist, but the existence of a classic overflow strongly suggests that the programmer is not considering even the most basic of security protections.",,High,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89" +A buffer overflow condition exists when a program attempts to put more data in a buffer than it can hold, or when a program attempts to put data in a memory area outside of the boundaries of a buffer. The simplest type of error, and the most common cause of buffer overflows, is the ""classic"" case in which the program copies the buffer without restricting how much is copied. Other variants exist, but the existence of a classic overflow strongly suggests that the programmer is not considering even the most basic of security protections.",,high,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89" CWE-129,EN-Improper Validation of Array Index (Type: Base),"The product uses untrusted input when calculating or using an array index, but the product does not validate or incorrectly validates the index to ensure the index references a valid position within the array. -This typically occurs when the pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. This may result in exposure of sensitive information or possibly a crash.",,High,"Writing Secure Code: Chapter 5, ""Array Indexing Errors"" Page 144 +This typically occurs when the pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. This may result in exposure of sensitive information or possibly a crash.",,high,"Writing Secure Code: Chapter 5, ""Array Indexing Errors"" Page 144 Top 25 Series - Rank 14 - Improper Validation of Array Index: http://blogs.sans.org/appsecstreetfighter/2010/03/12/top-25-series-rank-14-improper-validation-of-array-index/ Address Space Layout Randomization in Windows Vista: http://blogs.msdn.com/michael_howard/archive/2006/05/26/address-space-layout-randomization-in-windows-vista.aspx PaX: http://en.wikipedia.org/wiki/PaX @@ -21,10 +21,10 @@ Understanding DEP as a mitigation technology part 1: http://blogs.technet.com/b/ Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html 24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89" CWE-194,EN-Unexpected Sign Extension (Type: Base),"The software performs an operation on a number that causes it to be sign extended when it is transformed into a larger data type. When the original number is negative, this can produce unexpected values that lead to resultant weaknesses. -This can happen in signed and unsigned cases.",,High,"C Language Issues for Application Security: http://www.informit.com/articles/article.aspx?p=686170&seqNum=6 +This can happen in signed and unsigned cases.",,high,"C Language Issues for Application Security: http://www.informit.com/articles/article.aspx?p=686170&seqNum=6 Integral Security: http://www.ddj.com/security/193501774" CWE-20,EN-Improper Input Validation (Type: Class),"The product does not validate or incorrectly validates input that can affect the control flow or data flow of a program. -When software does not validate input properly, an attacker is able to craft the input in a form that is not expected by the rest of the application. This will lead to parts of the system receiving unintended input, which may result in altered control flow, arbitrary control of a resource, or arbitrary code execution.",,High,"Input Validation with ESAPI - Very Important: http://manicode.blogspot.com/2008/08/input-validation-with-esapi.html +When software does not validate input properly, an attacker is able to craft the input in a form that is not expected by the rest of the application. This will lead to parts of the system receiving unintended input, which may result in altered control flow, arbitrary control of a resource, or arbitrary code execution.",,high,"Input Validation with ESAPI - Very Important: http://manicode.blogspot.com/2008/08/input-validation-with-esapi.html OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI Hacking Exposed Web Applications, Second Edition: Input Validation Attacks Input validation or output filtering, which is better?: http://jeremiahgrossman.blogspot.com/2007/01/input-validation-or-output-filtering.html @@ -34,9 +34,9 @@ CWE-200,EN-Information Exposure (Type: Class),"An information exposure is the in The information either is regarded as sensitive within the product's own functionality, such as a private message; or provides information about the product or its environment that could be useful in an attack but is normally not available to the attacker, such as the installation path of a product that is remotely accessible. -Many information exposures are resultant (e.g. PHP script error revealing the full path of the program), but they can also be primary (e.g. timing discrepancies in cryptography). There are many different types of problems that involve information exposures. Their severity can range widely depending on the type of information that is revealed.",,High,Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/ +Many information exposures are resultant (e.g. PHP script error revealing the full path of the program), but they can also be primary (e.g. timing discrepancies in cryptography). There are many different types of problems that involve information exposures. Their severity can range widely depending on the type of information that is revealed.",,high,Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/ CWE-209,EN-Information Exposure Through an Error Message (Type: Base),"The software generates an error message that includes sensitive information about its environment, users, or associated data. -The sensitive information may be valuable information on its own (such as a password), or it may be useful for launching other, more deadly attacks. If an attack fails, an attacker may use error information provided by the server to launch another more focused attack. For example, an attempt to exploit a path traversal weakness (CWE-22) might yield the full pathname of the installed application. In turn, this could be used to select the proper number of "".."" sequences to navigate to the targeted file. An attack using SQL injection (CWE-89) might not initially succeed, but an error message could reveal the malformed query, which would expose query logic and possibly even passwords or other sensitive information used within the query.",,High,"Information Leakage: http://www.webappsec.org/projects/threat/classes/information_leakage.shtml +The sensitive information may be valuable information on its own (such as a password), or it may be useful for launching other, more deadly attacks. If an attack fails, an attacker may use error information provided by the server to launch another more focused attack. For example, an attempt to exploit a path traversal weakness (CWE-22) might yield the full pathname of the installed application. In turn, this could be used to select the proper number of "".."" sequences to navigate to the targeted file. An attack using SQL injection (CWE-89) might not initially succeed, but an error message could reveal the malformed query, which would expose query logic and possibly even passwords or other sensitive information used within the query.",,high,"Information Leakage: http://www.webappsec.org/projects/threat/classes/information_leakage.shtml Secure Programming with Static Analysis: Section 9.2, page 326. Writing Secure Code: Chapter 16, ""General Good Practices."" Page 415 24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183 @@ -44,20 +44,20 @@ Writing Secure Code: Chapter 16, ""General Good Practices."" Page 415 Top 25 Series - Rank 16 - Information Exposure Through an Error Message: http://software-security.sans.org/blog/2010/03/17/top-25-series-rank-16-information-exposure-through-an-error-message The Art of Software Security Assessment: Chapter 3, ""Overly Verbose Error Messages"", Page 75." CWE-234,EN-Failure to Handle Missing Parameter (Type: Variant),"If too few arguments are sent to a function, the function will still pop the expected number of arguments from the stack. Potentially, a variable number of arguments could be exhausted in a function as well. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,High, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,high, CWE-242,EN-Use of Inherently Dangerous Function (Type: Base),"The program calls a function that can never be guaranteed to work safely. -Certain functions behave in dangerous ways regardless of how they are used. Functions in this category were often implemented without taking security concerns into account. The gets() function is unsafe because it does not perform bounds checking on the size of its input. An attacker can easily send arbitrarily-sized input to gets() and overflow the destination buffer. Similarly, the >> operator is unsafe to use when reading into a statically-allocated character array because it does not perform bounds checking on the size of its input. An attacker can easily send arbitrarily-sized input to the >> operator and overflow the destination buffer.",,High,"Herb Schildt's C++ Programming Cookbook: Chapter 5. Working with I/O +Certain functions behave in dangerous ways regardless of how they are used. Functions in this category were often implemented without taking security concerns into account. The gets() function is unsafe because it does not perform bounds checking on the size of its input. An attacker can easily send arbitrarily-sized input to gets() and overflow the destination buffer. Similarly, the >> operator is unsafe to use when reading into a statically-allocated character array because it does not perform bounds checking on the size of its input. An attacker can easily send arbitrarily-sized input to the >> operator and overflow the destination buffer.",,high,"Herb Schildt's C++ Programming Cookbook: Chapter 5. Working with I/O Writing Secure Code: Chapter 5, ""gets and fgets"" Page 163" CWE-243,EN-Creation of chroot Jail Without Changing Working Directory (Type: Variant),"The program uses the chroot() system call to create a jail, but does not change the working directory afterward. This does not prevent access to files outside of the jail. -Improper use of chroot() may allow attackers to escape from the chroot jail. The chroot() function call does not change the process's current working directory, so relative paths may still refer to file system resources outside of the chroot jail after chroot() has been called.",,High, +Improper use of chroot() may allow attackers to escape from the chroot jail. The chroot() function call does not change the process's current working directory, so relative paths may still refer to file system resources outside of the chroot jail after chroot() has been called.",,high, CWE-268,EN-Privilege Chaining (Type: Base),"Two distinct privileges, roles, capabilities, or rights can be combined in a way that allows an entity to perform unsafe actions that would not be allowed without that combination. -Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,High,Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html +Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,high,Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html CWE-271,EN-Privilege Dropping / Lowering Errors (Type: Class),"The software does not drop privileges before passing control of a resource to an actor that does not have those privileges. -In some contexts, a system executing with elevated permissions will hand off a process/file/etc. to another process or user. If the privileges of an entity are not reduced, then elevated privileges are spread throughout a system and possibly to an attacker.",,High,"24 Deadly Sins of Software Security: ""Sin 16: Executing Code With Too Much Privilege."" Page 243 +In some contexts, a system executing with elevated permissions will hand off a process/file/etc. to another process or user. If the privileges of an entity are not reduced, then elevated privileges are spread throughout a system and possibly to an attacker.",,high,"24 Deadly Sins of Software Security: ""Sin 16: Executing Code With Too Much Privilege."" Page 243 The Art of Software Security Assessment: Chapter 9, ""Dropping Privileges Permanently"", Page 479." CWE-285,EN-Improper Authorization (Type: Class),"The software does not perform or incorrectly performs an authorization check when an actor attempts to access a resource or perform an action. Assuming a user with a given identity, authorization is the process of determining whether that user can access a given resource, based on the user's privileges and any permissions or other access-control specifications that apply to the resource. -When access control checks are not applied consistently - or not at all - users are able to access data or perform actions that they should not be allowed to perform. This can lead to a wide range of problems, including information exposures, denial of service, and arbitrary code execution.",,High,"Role Based Access Control and Role Based Security: http://csrc.nist.gov/groups/SNS/rbac/ +When access control checks are not applied consistently - or not at all - users are able to access data or perform actions that they should not be allowed to perform. This can lead to a wide range of problems, including information exposures, denial of service, and arbitrary code execution.",,high,"Role Based Access Control and Role Based Security: http://csrc.nist.gov/groups/SNS/rbac/ Writing Secure Code: Chapter 4, ""Authorization"" Page 114; Chapter 6, ""Determining Appropriate Access Control"" Page 171 Top 25 Series - Rank 5 - Improper Access Control (Authorization): http://blogs.sans.org/appsecstreetfighter/2010/03/04/top-25-series-rank-5-improper-access-control-authorization/ OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI @@ -65,77 +65,77 @@ Authentication using JAAS: http://www.javaranch.com/journal/2008/04/authenticati The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Authorization"", Page 39. The Art of Software Security Assessment: Chapter 11, ""ACL Inheritance"", Page 649." CWE-291,EN-Reliance on IP Address for Authentication (Type: Variant),"The software uses an IP address for authentication. -IP addresses can be easily spoofed. Attackers can forge the source IP address of the packets they send, but response packets will return to the forged IP address. To see the response packets, the attacker has to sniff the traffic between the victim machine and the forged IP address. In order to accomplish the required sniffing, attackers typically attempt to locate themselves on the same subnet as the victim machine. Attackers may be able to circumvent this requirement by using source routing, but source routing is disabled across much of the Internet today. In summary, IP address verification can be a useful part of an authentication scheme, but it should not be the single factor required for authentication.",,High, +IP addresses can be easily spoofed. Attackers can forge the source IP address of the packets they send, but response packets will return to the forged IP address. To see the response packets, the attacker has to sniff the traffic between the victim machine and the forged IP address. In order to accomplish the required sniffing, attackers typically attempt to locate themselves on the same subnet as the victim machine. Attackers may be able to circumvent this requirement by using source routing, but source routing is disabled across much of the Internet today. In summary, IP address verification can be a useful part of an authentication scheme, but it should not be the single factor required for authentication.",,high, CWE-292,EN-DEPRECATED (Duplicate): Trusting Self-reported DNS Name (Type: Variant),"This entry has been deprecated because it was a duplicate of CWE-350. All content has been transferred to CWE-350. -IP addresses can be easily spoofed. Attackers can forge the source IP address of the packets they send, but response packets will return to the forged IP address. To see the response packets, the attacker has to sniff the traffic between the victim machine and the forged IP address. In order to accomplish the required sniffing, attackers typically attempt to locate themselves on the same subnet as the victim machine. Attackers may be able to circumvent this requirement by using source routing, but source routing is disabled across much of the Internet today. In summary, IP address verification can be a useful part of an authentication scheme, but it should not be the single factor required for authentication.",,High, +IP addresses can be easily spoofed. Attackers can forge the source IP address of the packets they send, but response packets will return to the forged IP address. To see the response packets, the attacker has to sniff the traffic between the victim machine and the forged IP address. In order to accomplish the required sniffing, attackers typically attempt to locate themselves on the same subnet as the victim machine. Attackers may be able to circumvent this requirement by using source routing, but source routing is disabled across much of the Internet today. In summary, IP address verification can be a useful part of an authentication scheme, but it should not be the single factor required for authentication.",,high, CWE-293,EN-Using Referer Field for Authentication (Type: Variant),"The referer field in HTTP requests can be easily modified and, as such, is not a valid means of message integrity checking. -IP addresses can be easily spoofed. Attackers can forge the source IP address of the packets they send, but response packets will return to the forged IP address. To see the response packets, the attacker has to sniff the traffic between the victim machine and the forged IP address. In order to accomplish the required sniffing, attackers typically attempt to locate themselves on the same subnet as the victim machine. Attackers may be able to circumvent this requirement by using source routing, but source routing is disabled across much of the Internet today. In summary, IP address verification can be a useful part of an authentication scheme, but it should not be the single factor required for authentication.",,High,"The Art of Software Security Assessment: Chapter 17, ""Referer Request Header"", Page 1030." +IP addresses can be easily spoofed. Attackers can forge the source IP address of the packets they send, but response packets will return to the forged IP address. To see the response packets, the attacker has to sniff the traffic between the victim machine and the forged IP address. In order to accomplish the required sniffing, attackers typically attempt to locate themselves on the same subnet as the victim machine. Attackers may be able to circumvent this requirement by using source routing, but source routing is disabled across much of the Internet today. In summary, IP address verification can be a useful part of an authentication scheme, but it should not be the single factor required for authentication.",,high,"The Art of Software Security Assessment: Chapter 17, ""Referer Request Header"", Page 1030." CWE-294,EN-Authentication Bypass by Capture-replay (Type: Base),"A capture-replay flaw exists when the design of the software makes it possible for a malicious user to sniff network traffic and bypass authentication by replaying it to the server in question to the same effect as the original message (or with minor changes). -Capture-replay attacks are common and can be difficult to defeat without cryptography. They are a subset of network injection attacks that rely on observing previously-sent valid commands, then changing them slightly if necessary and resending the same commands to the server.",,High, +Capture-replay attacks are common and can be difficult to defeat without cryptography. They are a subset of network injection attacks that rely on observing previously-sent valid commands, then changing them slightly if necessary and resending the same commands to the server.",,high, CWE-297,EN-Improper Validation of Certificate with Host Mismatch (Type: Variant),"The software communicates with a host that provides a certificate, but the software does not properly ensure that the certificate is actually associated with that host. Even if a certificate is well-formed, signed, and follows the chain of trust, it may simply be a valid certificate for a different site than the site that the software is interacting with. If the certificate's host-specific data is not properly checked - such as the Common Name (CN) in the Subject or the Subject Alternative Name (SAN) extension of an X.509 certificate - it may be possible for a redirection or spoofing attack to allow a malicious host with a valid certificate to provide data, impersonating a trusted host. In order to ensure data integrity, the certificate must be valid and it must pertain to the site that is being accessed. -Even if the software attempts to check the hostname, it is still possible to incorrectly check the hostname. For example, attackers could create a certificate with a name that begins with a trusted name followed by a NUL byte, which could cause some string-based comparisons to only examine the portion that contains the trusted name.",,High,"The Most Dangerous Code in the World: Validating SSL Certificates in Non-Browser Software: http://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf +Even if the software attempts to check the hostname, it is still possible to incorrectly check the hostname. For example, attackers could create a certificate with a name that begins with a trusted name followed by a NUL byte, which could cause some string-based comparisons to only examine the portion that contains the trusted name.",,high,"The Most Dangerous Code in the World: Validating SSL Certificates in Non-Browser Software: http://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf Why Eve and Mallory Love Android: An Analysis of Android SSL (In)Security: http://www2.dcsec.uni-hannover.de/files/android/p50-fahl.pdf Secure programming with the OpenSSL API, Part 2: Secure handshake: http://www.ibm.com/developerworks/library/l-openssl2/index.html An Introduction to OpenSSL Programming (Part I): http://www.rtfm.com/openssl-examples/part1.pdf 24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347" CWE-308,EN-Use of Single-factor Authentication (Type: Base),"The use of single-factor authentication can lead to unnecessary risk of compromise when compared with the benefits of a dual-factor authentication scheme. -While the use of multiple authentication schemes is simply piling on more complexity on top of authentication, it is inestimably valuable to have such measures of redundancy. The use of weak, reused, and common passwords is rampant on the internet. Without the added protection of multiple authentication schemes, a single mistake can result in the compromise of an account. For this reason, if multiple schemes are possible and also easy to use, they should be implemented and required.",,High, +While the use of multiple authentication schemes is simply piling on more complexity on top of authentication, it is inestimably valuable to have such measures of redundancy. The use of weak, reused, and common passwords is rampant on the internet. Without the added protection of multiple authentication schemes, a single mistake can result in the compromise of an account. For this reason, if multiple schemes are possible and also easy to use, they should be implemented and required.",,high, CWE-321,EN-Use of Hard-coded Cryptographic Key (Type: Base),"The use of a hard-coded cryptographic key significantly increases the possibility that encrypted data may be recovered. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '...' manipulation is useful for bypassing some path traversal protection schemes. On some Windows systems, it is equivalent to ""..\.."" and might bypass checks that assume only two dots are valid. Incomplete filtering, such as removal of ""./"" sequences, can ultimately produce valid "".."" sequences due to a collapse into unsafe value (CWE-182).",,High, +The '...' manipulation is useful for bypassing some path traversal protection schemes. On some Windows systems, it is equivalent to ""..\.."" and might bypass checks that assume only two dots are valid. Incomplete filtering, such as removal of ""./"" sequences, can ultimately produce valid "".."" sequences due to a collapse into unsafe value (CWE-182).",,high, CWE-322,EN-Key Exchange without Entity Authentication (Type: Base),"The software performs a key exchange with an actor without verifying the identity of that actor. -Performing a key exchange will preserve the integrity of the information sent between two entities, but this will not guarantee that the entities are who they claim they are. This may enable a set of ""man-in-the-middle"" attacks. Typically, this involves a victim client that contacts a malicious server that is impersonating a trusted server. If the client skips authentication or ignores an authentication failure, the malicious server may request authentication information from the user. The malicious server can then use this authentication information to log in to the trusted server using the victim's credentials, sniff traffic between the victim and trusted server, etc.",,High,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347 +Performing a key exchange will preserve the integrity of the information sent between two entities, but this will not guarantee that the entities are who they claim they are. This may enable a set of ""man-in-the-middle"" attacks. Typically, this involves a victim client that contacts a malicious server that is impersonating a trusted server. If the client skips authentication or ignores an authentication failure, the malicious server may request authentication information from the user. The malicious server can then use this authentication information to log in to the trusted server using the victim's credentials, sniff traffic between the victim and trusted server, etc.",,high,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347 The Art of Software Security Assessment: Chapter 2, ""Untrustworthy Credentials"", Page 37." CWE-323,"EN-Reusing a Nonce, Key Pair in Encryption (Type: Base)","Nonces should be used for the present occasion and only once. -Performing a key exchange will preserve the integrity of the information sent between two entities, but this will not guarantee that the entities are who they claim they are. This may enable a set of ""man-in-the-middle"" attacks. Typically, this involves a victim client that contacts a malicious server that is impersonating a trusted server. If the client skips authentication or ignores an authentication failure, the malicious server may request authentication information from the user. The malicious server can then use this authentication information to log in to the trusted server using the victim's credentials, sniff traffic between the victim and trusted server, etc.",,High, +Performing a key exchange will preserve the integrity of the information sent between two entities, but this will not guarantee that the entities are who they claim they are. This may enable a set of ""man-in-the-middle"" attacks. Typically, this involves a victim client that contacts a malicious server that is impersonating a trusted server. If the client skips authentication or ignores an authentication failure, the malicious server may request authentication information from the user. The malicious server can then use this authentication information to log in to the trusted server using the victim's credentials, sniff traffic between the victim and trusted server, etc.",,high, CWE-360,EN-Trust of System Event Data (Type: Base),"Security based on event locations are insecure and can be spoofed. -Events are a messaging system which may provide control data to programs listening for events. Events often do not have any type of authentication framework to allow them to be verified from a trusted source. Any application, in Windows, on a given desktop can send a message to any window on the same desktop. There is no authentication framework for these messages. Therefore, any message can be used to manipulate any process on the desktop if the process does not check the validity and safeness of those messages.",,High, +Events are a messaging system which may provide control data to programs listening for events. Events often do not have any type of authentication framework to allow them to be verified from a trusted source. Any application, in Windows, on a given desktop can send a message to any window on the same desktop. There is no authentication framework for these messages. Therefore, any message can be used to manipulate any process on the desktop if the process does not check the validity and safeness of those messages.",,high, CWE-378,EN-Creation of Temporary File With Insecure Permissions (Type: Base),"Opening temporary files without appropriate measures or controls can leave the file, its contents and any function that it impacts vulnerable to attack. -If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,High, +If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,high, CWE-416,EN-Use After Free (Type: Base),"Referencing memory after it has been freed can cause a program to crash, use unexpected values, or execute code. The use of previously-freed memory can have any number of adverse consequences, ranging from the corruption of valid data to the execution of arbitrary code, depending on the instantiation and timing of the flaw. The simplest way data corruption may occur involves the system's reuse of the freed memory. Use-after-free errors have two common and sometimes overlapping causes: Error conditions and other exceptional circumstances. Confusion over which part of the program is responsible for freeing the memory. In this scenario, the memory in question is allocated to another pointer validly at some point after it has been freed. The original pointer to the freed memory is used again and points to somewhere within the new allocation. As the data is changed, it corrupts the validly used memory; this induces undefined behavior in the process. -If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,High,"24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143" +If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,high,"24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143" CWE-457,EN-Use of Uninitialized Variable (Type: Variant),"The code uses a variable that has not been initialized, leading to unpredictable or unintended results. -In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,High,"Exploiting Uninitialized Data: http://www.felinemenace.org/~mercy/papers/UBehavior/UBehavior.zip +In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,high,"Exploiting Uninitialized Data: http://www.felinemenace.org/~mercy/papers/UBehavior/UBehavior.zip MS08-014 : The Case of the Uninitialized Stack Variable Vulnerability: http://blogs.technet.com/swi/archive/2008/03/11/the-case-of-the-uninitialized-stack-variable-vulnerability.aspx 24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143 The Art of Software Security Assessment: Chapter 7, ""Variable Initialization"", Page 312." CWE-467,EN-Use of sizeof() on a Pointer Type (Type: Variant),"The code calls sizeof() on a malloced pointer type, which always returns the wordsize/8. This can produce an unexpected result if the programmer intended to determine how much memory has been allocated. -Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,High,EXP01-A. Do not take the sizeof a pointer to determine the size of a type: https://www.securecoding.cert.org/confluence/display/seccode/EXP01-A.+Do+not+take+the+sizeof+a+pointer+to+determine+the+size+of+a+type +Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,high,EXP01-A. Do not take the sizeof a pointer to determine the size of a type: https://www.securecoding.cert.org/confluence/display/seccode/EXP01-A.+Do+not+take+the+sizeof+a+pointer+to+determine+the+size+of+a+type CWE-486,EN-Comparison of Classes by Name (Type: Variant),"The program compares classes by name, which can cause it to use the wrong class when multiple classes can have the same name. -If the decision to trust the methods and data of an object is based on the name of a class, it is possible for malicious users to send objects of the same name as trusted classes and thereby gain the trust afforded to known classes and types.",,High, +If the decision to trust the methods and data of an object is based on the name of a class, it is possible for malicious users to send objects of the same name as trusted classes and thereby gain the trust afforded to known classes and types.",,high, CWE-493,EN-Critical Public Variable Without Final Modifier (Type: Variant),"The product has a critical public variable that is not final, which allows the variable to be modified to contain unexpected values. -If a field is non-final and public, it can be changed once the value is set by any function that has access to the class which contains the field. This could lead to a vulnerability if other parts of the program make assumptions about the contents of that field.",,High, +If a field is non-final and public, it can be changed once the value is set by any function that has access to the class which contains the field. This could lead to a vulnerability if other parts of the program make assumptions about the contents of that field.",,high, CWE-499,EN-Serializable Class Containing Sensitive Data (Type: Variant),"The code contains a class with sensitive data, but the class does not explicitly deny serialization. The data can be accessed by serializing the class through another class. -Serializable classes are effectively open classes since data cannot be hidden in them. Classes that do not explicitly deny serialization can be serialized by any other class, which can then in turn use the data stored inside it.",,High, +Serializable classes are effectively open classes since data cannot be hidden in them. Classes that do not explicitly deny serialization can be serialized by any other class, which can then in turn use the data stored inside it.",,high, CWE-500,EN-Public Static Field Not Marked Final (Type: Variant),"An object contains a public static field that is not marked final, which might allow it to be modified in unexpected ways. -Public static variables can be read without an accessor and changed without a mutator by any classes in the application.",,High, +Public static variables can be read without an accessor and changed without a mutator by any classes in the application.",,high, CWE-515,EN-Covert Storage Channel (Type: Base),"A covert storage channel transfers information through the setting of bits by one program and the reading of those bits by another. What distinguishes this case from that of ordinary operation is that the bits are used to convey encoded information. -Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,High, +Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,high, CWE-639,EN-Authorization Bypass Through User-Controlled Key (Type: Base),"The system's authorization functionality does not prevent one user from gaining access to another user's data or record by modifying the key value identifying the data. -Retrieval of a user record occurs in the system based on some key value that is under user control. The key would typically identify a user related record stored in the system and would be used to lookup that record for presentation to the user. It is likely that an attacker would have to be an authenticated user in the system. However, the authorization process would not properly check the data access operation to ensure that the authenticated user performing the operation has sufficient entitlements to perform the requested data access, hence bypassing any other authorization checks present in the system. One manifestation of this weakness would be if a system used sequential or otherwise easily guessable session ids that would allow one user to easily switch to another user's session and read/modify their data.",,High, +Retrieval of a user record occurs in the system based on some key value that is under user control. The key would typically identify a user related record stored in the system and would be used to lookup that record for presentation to the user. It is likely that an attacker would have to be an authenticated user in the system. However, the authorization process would not properly check the data access operation to ensure that the authenticated user performing the operation has sufficient entitlements to perform the requested data access, hence bypassing any other authorization checks present in the system. One manifestation of this weakness would be if a system used sequential or otherwise easily guessable session ids that would allow one user to easily switch to another user's session and read/modify their data.",,high, CWE-640,EN-Weak Password Recovery Mechanism for Forgotten Password (Type: Base),"The software contains a mechanism for users to recover or change their passwords without knowing the original password, but the mechanism is weak. It is common for an application to have a mechanism that provides a means for a user to gain access to their account in the event they forget their password. Very often the password recovery mechanism is weak, which has the effect of making it more likely that it would be possible for a person other than the legitimate system user to gain access to that user's account. -This weakness may be that the security question is too easy to guess or find an answer to (e.g. because it is too common). Or there might be an implementation weakness in the password recovery mechanism code that may for instance trick the system into e-mailing the new password to an e-mail account other than that of the user. There might be no throttling done on the rate of password resets so that a legitimate user can be denied service by an attacker if an attacker tries to recover their password in a rapid succession. The system may send the original password to the user rather than generating a new temporary password. In summary, password recovery functionality, if not carefully designed and implemented can often become the system's weakest link that can be misused in a way that would allow an attacker to gain unauthorized access to the system. Weak password recovery schemes completely undermine a strong password authentication scheme.",,High,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" +This weakness may be that the security question is too easy to guess or find an answer to (e.g. because it is too common). Or there might be an implementation weakness in the password recovery mechanism code that may for instance trick the system into e-mailing the new password to an e-mail account other than that of the user. There might be no throttling done on the rate of password resets so that a legitimate user can be denied service by an attacker if an attacker tries to recover their password in a rapid succession. The system may send the original password to the user rather than generating a new temporary password. In summary, password recovery functionality, if not carefully designed and implemented can often become the system's weakest link that can be misused in a way that would allow an attacker to gain unauthorized access to the system. Weak password recovery schemes completely undermine a strong password authentication scheme.",,high,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" CWE-642,EN-External Control of Critical State Data (Type: Class),"The software stores security-critical state information about its users, or the software itself, in a location that is accessible to unauthorized actors. If an attacker can modify the state information without detection, then it could be used to perform unauthorized actions or access unexpected resources, since the application programmer does not expect that the state can be changed. -State information can be stored in various locations such as a cookie, in a hidden web form field, input parameter or argument, an environment variable, a database record, within a settings file, etc. All of these locations have the potential to be modified by an attacker. When this state information is used to control security or determine resource usage, then it may create a vulnerability. For example, an application may perform authentication, then save the state in an ""authenticated=true"" cookie. An attacker may simply create this cookie in order to bypass the authentication.",,High,"Top 10 2007-Insecure Direct Object Reference: http://www.owasp.org/index.php/Top_10_2007-A4 +State information can be stored in various locations such as a cookie, in a hidden web form field, input parameter or argument, an environment variable, a database record, within a settings file, etc. All of these locations have the potential to be modified by an attacker. When this state information is used to control security or determine resource usage, then it may create a vulnerability. For example, an application may perform authentication, then save the state in an ""authenticated=true"" cookie. An attacker may simply create this cookie in order to bypass the authentication.",,high,"Top 10 2007-Insecure Direct Object Reference: http://www.owasp.org/index.php/Top_10_2007-A4 HMAC: http://en.wikipedia.org/wiki/Hmac 24 Deadly Sins of Software Security: ""Sin 4: Use of Magic URLs, Predictable Cookies, and Hidden Form Fields."" Page 75" CWE-643,EN-Improper Neutralization of Data within XPath Expressions (XPath Injection) (Type: Base),"The software uses external input to dynamically construct an XPath expression used to retrieve data from an XML database, but it does not neutralize or incorrectly neutralizes that input. This allows an attacker to control the structure of the query. -The net effect is that the attacker will have control over the information selected from the XML database and may use that ability to control application flow, modify logic, retrieve unauthorized data, or bypass important checks (e.g. authentication).",,High,"XPath Injection: http://www.webappsec.org/projects/threat/classes/xpath_injection.shtml +The net effect is that the attacker will have control over the information selected from the XML database and may use that ability to control application flow, modify logic, retrieve unauthorized data, or bypass important checks (e.g. authentication).",,high,"XPath Injection: http://www.webappsec.org/projects/threat/classes/xpath_injection.shtml The Art of Software Security Assessment: Chapter 17, ""XPath Injection"", Page 1070." CWE-644,EN-Improper Neutralization of HTTP Headers for Scripting Syntax (Type: Variant),"The application does not neutralize or incorrectly neutralizes web scripting syntax in HTTP headers that can be used by web browser components that can process raw headers, such as Flash. An attacker may be able to conduct cross-site scripting and other attacks against users who have these components enabled. -If an application does not neutralize user controlled data being placed in the header of an HTTP response coming from the server, the header may contain a script that will get executed in the client's browser context, potentially resulting in a cross site scripting vulnerability or possibly an HTTP response splitting attack. It is important to carefully control data that is being placed both in HTTP response header and in the HTTP response body to ensure that no scripting syntax is present, taking various encodings into account.",,High, +If an application does not neutralize user controlled data being placed in the header of an HTTP response coming from the server, the header may contain a script that will get executed in the client's browser context, potentially resulting in a cross site scripting vulnerability or possibly an HTTP response splitting attack. It is important to carefully control data that is being placed both in HTTP response header and in the HTTP response body to ensure that no scripting syntax is present, taking various encodings into account.",,high, CWE-645,EN-Overly Restrictive Account Lockout Mechanism (Type: Base),"The software contains an account lockout protection mechanism, but the mechanism is too restrictive and can be triggered too easily. This allows attackers to deny service to legitimate users by causing their accounts to be locked out. -Account lockout is a security feature often present in applications as a countermeasure to the brute force attack on the password based authentication mechanism of the system. After a certain number of failed login attempts, the users' account may be disabled for a certain period of time or until it is unlocked by an administrator. Other security events may also possibly trigger account lockout. However, an attacker may use this very security feature to deny service to legitimate system users. It is therefore important to ensure that the account lockout security mechanism is not overly restrictive.",,High, +Account lockout is a security feature often present in applications as a countermeasure to the brute force attack on the password based authentication mechanism of the system. After a certain number of failed login attempts, the users' account may be disabled for a certain period of time or until it is unlocked by an administrator. Other security events may also possibly trigger account lockout. However, an attacker may use this very security feature to deny service to legitimate system users. It is therefore important to ensure that the account lockout security mechanism is not overly restrictive.",,high, CWE-646,EN-Reliance on File Name or Extension of Externally-Supplied File (Type: Variant),"The software allows a file to be uploaded, but it relies on the file name or extension of the file to determine the appropriate behaviors. This could be used by attackers to cause the file to be misclassified and processed in a dangerous fashion. -An application might use the file name or extension of of a user-supplied file to determine the proper course of action, such as selecting the correct process to which control should be passed, deciding what data should be made available, or what resources should be allocated. If the attacker can cause the code to misclassify the supplied file, then the wrong action could occur. For example, an attacker could supply a file that ends in a "".php.gif"" extension that appears to be a GIF image, but would be processed as PHP code. In extreme cases, code execution is possible, but the attacker could also cause exhaustion of resources, denial of service, exposure of debug or system data (including application source code), or being bound to a particular server side process. This weakness may be due to a vulnerability in any of the technologies used by the web and application servers, due to misconfiguration, or resultant from another flaw in the application itself.",,High, +An application might use the file name or extension of of a user-supplied file to determine the proper course of action, such as selecting the correct process to which control should be passed, deciding what data should be made available, or what resources should be allocated. If the attacker can cause the code to misclassify the supplied file, then the wrong action could occur. For example, an attacker could supply a file that ends in a "".php.gif"" extension that appears to be a GIF image, but would be processed as PHP code. In extreme cases, code execution is possible, but the attacker could also cause exhaustion of resources, denial of service, exposure of debug or system data (including application source code), or being bound to a particular server side process. This weakness may be due to a vulnerability in any of the technologies used by the web and application servers, due to misconfiguration, or resultant from another flaw in the application itself.",,high, CWE-647,EN-Use of Non-Canonical URL Paths for Authorization Decisions (Type: Variant),"The software defines policy namespaces and makes authorization decisions based on the assumption that a URL is canonical. This can allow a non-canonical URL to bypass the authorization. If an application defines policy namespaces and makes authorization decisions based on the URL, but it does not require or convert to a canonical URL before making the authorization decision, then it opens the application to attack. For example, if the application only wants to allow access to http://www.example.com/mypage, then the attacker might be able to bypass this restriction using equivalent URLs such as: http://WWW.EXAMPLE.COM/mypage @@ -143,19 +143,19 @@ http://www.example.com/%6Dypage (alternate encoding) http://192.168.1.1/mypage (IP address) http://www.example.com/mypage/ (trailing /) http://www.example.com:80/mypage -Therefore it is important to specify access control policy that is based on the path information in some canonical form with all alternate encodings rejected (which can be accomplished by a default deny rule).",,High, +Therefore it is important to specify access control policy that is based on the path information in some canonical form with all alternate encodings rejected (which can be accomplished by a default deny rule).",,high, CWE-649,EN-Reliance on Obfuscation or Encryption of Security-Relevant Inputs without Integrity Checking (Type: Base),"The software uses obfuscation or encryption of inputs that should not be mutable by an external actor, but the software does not use integrity checks to detect if those inputs have been modified. -When an application relies on obfuscation or incorrectly applied / weak encryption to protect client-controllable tokens or parameters, that may have an effect on the user state, system state, or some decision made on the server. Without protecting the tokens/parameters for integrity, the application is vulnerable to an attack where an adversary blindly traverses the space of possible values of the said token/parameter in order to attempt to gain an advantage. The goal of the attacker is to find another admissible value that will somehow elevate his or her privileges in the system, disclose information or change the behavior of the system in some way beneficial to the attacker. If the application does not protect these critical tokens/parameters for integrity, it will not be able to determine that these values have been tampered with. Measures that are used to protect data for confidentiality should not be relied upon to provide the integrity service.",,High, +When an application relies on obfuscation or incorrectly applied / weak encryption to protect client-controllable tokens or parameters, that may have an effect on the user state, system state, or some decision made on the server. Without protecting the tokens/parameters for integrity, the application is vulnerable to an attack where an adversary blindly traverses the space of possible values of the said token/parameter in order to attempt to gain an advantage. The goal of the attacker is to find another admissible value that will somehow elevate his or her privileges in the system, disclose information or change the behavior of the system in some way beneficial to the attacker. If the application does not protect these critical tokens/parameters for integrity, it will not be able to determine that these values have been tampered with. Measures that are used to protect data for confidentiality should not be relied upon to provide the integrity service.",,high, CWE-650,EN-Trusting HTTP Permission Methods on the Server Side (Type: Variant),"The server contains a protection mechanism that assumes that any URI that is accessed using HTTP GET will not cause a state change to the associated resource. This might allow attackers to bypass intended access restrictions and conduct resource modification and deletion attacks, since some applications allow GET to modify state. -An application may disallow the HTTP requests to perform DELETE, PUT and POST operations on the resource representation, believing that it will be enough to prevent unintended resource alterations. Even though the HTTP GET specification requires that GET requests should not have side effects, there is nothing in the HTTP protocol itself that prevents the HTTP GET method from performing more than just query of the data. For instance, it is a common practice with REST based Web Services to have HTTP GET requests modifying resources on the server side. Whenever that happens however, the access control needs to be properly enforced in the application. No assumptions should be made that only HTTP DELETE, PUT, and POST methods have the power to alter the representation of the resource being accessed in the request.",,High, +An application may disallow the HTTP requests to perform DELETE, PUT and POST operations on the resource representation, believing that it will be enough to prevent unintended resource alterations. Even though the HTTP GET specification requires that GET requests should not have side effects, there is nothing in the HTTP protocol itself that prevents the HTTP GET method from performing more than just query of the data. For instance, it is a common practice with REST based Web Services to have HTTP GET requests modifying resources on the server side. Whenever that happens however, the access control needs to be properly enforced in the application. No assumptions should be made that only HTTP DELETE, PUT, and POST methods have the power to alter the representation of the resource being accessed in the request.",,high, CWE-652,EN-Improper Neutralization of Data within XQuery Expressions (XQuery Injection) (Type: Base),"The software uses external input to dynamically construct an XQuery expression used to retrieve data from an XML database, but it does not neutralize or incorrectly neutralizes that input. This allows an attacker to control the structure of the query. -The net effect is that the attacker will have control over the information selected from the XML database and may use that ability to control application flow, modify logic, retrieve unauthorized data, or bypass important checks (e.g. authentication).",,High, +The net effect is that the attacker will have control over the information selected from the XML database and may use that ability to control application flow, modify logic, retrieve unauthorized data, or bypass important checks (e.g. authentication).",,high, CWE-676,EN-Use of Potentially Dangerous Function (Type: Base),"The program invokes a potentially dangerous function that could introduce a vulnerability if it is used incorrectly, but the function can also be used safely. -Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,High,"Security Development Lifecycle (SDL) Banned Function Calls: http://msdn.microsoft.com/en-us/library/bb288454.aspx +Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,high,"Security Development Lifecycle (SDL) Banned Function Calls: http://msdn.microsoft.com/en-us/library/bb288454.aspx Writing Secure Code: Chapter 5, ""Safe String Handling"" Page 156, 160 The Art of Software Security Assessment: Chapter 8, ""C String Handling"", Page 388." CWE-682,EN-Incorrect Calculation (Type: Class),"The software performs a calculation that generates incorrect or unintended results that are later used in security-critical decisions or resource management.. -When software performs a security-critical calculation incorrectly, it might lead to incorrect resource allocations, incorrect privilege assignments, or failed comparisons among other things. Many of the direct results of an incorrect calculation can lead to even larger problems such as failed protection mechanisms or even arbitrary code execution.",,High,"SafeInt: http://safeint.codeplex.com/ +When software performs a security-critical calculation incorrectly, it might lead to incorrect resource allocations, incorrect privilege assignments, or failed comparisons among other things. Many of the direct results of an incorrect calculation can lead to even larger problems such as failed protection mechanisms or even arbitrary code execution.",,high,"SafeInt: http://safeint.codeplex.com/ 24 Deadly Sins of Software Security: ""Sin 7: Integer Overflows."" Page 119 The Art of Software Security Assessment: Chapter 6, ""Signed Integer Boundaries"", Page 220." CWE-78,EN-Improper Neutralization of Special Elements used in an OS Command (OS Command Injection) (Type: Base),"The software constructs all or part of an OS command using externally-influenced input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could modify the intended OS command when it is sent to a downstream component.. @@ -163,7 +163,7 @@ This could allow attackers to execute unexpected, dangerous commands directly on There are at least two subtypes of OS command injection: The application intends to execute a single, fixed program that is under its own control. It intends to use externally-supplied inputs as arguments to that program. For example, the program might use system(""nslookup [HOSTNAME]"") to run nslookup and allow the user to supply a HOSTNAME, which is used as an argument. Attackers cannot prevent nslookup from executing. However, if the program does not remove command separators from the HOSTNAME argument, attackers could place the separators into the arguments, which allows them to execute their own program after nslookup has finished executing. The application accepts an input that it uses to fully select which program to run, as well as which commands to use. The application simply redirects this entire command to the operating system. For example, the program might use ""exec([COMMAND])"" to execute the [COMMAND] that was supplied by the user. If the COMMAND is under attacker control, then the attacker can execute arbitrary commands or programs. If the command is being executed using functions like exec() and CreateProcess(), the attacker might not be able to combine multiple commands together in the same line. -From a weakness standpoint, these variants represent distinct programmer errors. In the first variant, the programmer clearly intends that input from untrusted parties will be part of the arguments in the command to be executed. In the second variant, the programmer does not intend for the command to be accessible to any untrusted party, but the programmer probably has not accounted for alternate ways in which malicious attackers can provide input.",,High,"Exploiting Software: How to Break Code +From a weakness standpoint, these variants represent distinct programmer errors. In the first variant, the programmer clearly intends that input from untrusted parties will be part of the arguments in the command to be executed. In the second variant, the programmer does not intend for the command to be accessible to any untrusted party, but the programmer probably has not accounted for alternate ways in which malicious attackers can provide input.",,high,"Exploiting Software: How to Break Code Meta-Character Vulnerabilities: http://www.cs.purdue.edu/homes/cs390s/slides/week09.pdf OS Commanding: http://projects.webappsec.org/OS-Commanding The World Wide Web Security FAQ: http://www.w3.org/Security/Faq/wwwsf4.html @@ -174,11 +174,11 @@ OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ES Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html The Art of Software Security Assessment: Chapter 8, ""Shell Metacharacters"", Page 425." CWE-784,EN-Reliance on Cookies without Validation and Integrity Checking in a Security Decision (Type: Variant),"The application uses a protection mechanism that relies on the existence or values of a cookie, but it does not properly ensure that the cookie is valid for the associated user. -Attackers can easily modify cookies, within the browser or by implementing the client-side code outside of the browser. Attackers can bypass protection mechanisms such as authorization and authentication by modifying the cookie to contain an expected value.",,High,"Unforgivable Vulnerabilities: http://cve.mitre.org/docs/docs-2007/unforgivable.pdf +Attackers can easily modify cookies, within the browser or by implementing the client-side code outside of the browser. Attackers can bypass protection mechanisms such as authorization and authentication by modifying the cookie to contain an expected value.",,high,"Unforgivable Vulnerabilities: http://cve.mitre.org/docs/docs-2007/unforgivable.pdf Writing Secure Code: Chapter 13, ""Sensitive Data in Cookies and Fields"" Page 435" CWE-862,EN-Missing Authorization (Type: Class),"The software does not perform an authorization check when an actor attempts to access a resource or perform an action. Assuming a user with a given identity, authorization is the process of determining whether that user can access a given resource, based on the user's privileges and any permissions or other access-control specifications that apply to the resource. -When access control checks are not applied, users are able to access data or perform actions that they should not be allowed to perform. This can lead to a wide range of problems, including information exposures, denial of service, and arbitrary code execution.",,High,"Role Based Access Control and Role Based Security: http://csrc.nist.gov/groups/SNS/rbac/ +When access control checks are not applied, users are able to access data or perform actions that they should not be allowed to perform. This can lead to a wide range of problems, including information exposures, denial of service, and arbitrary code execution.",,high,"Role Based Access Control and Role Based Security: http://csrc.nist.gov/groups/SNS/rbac/ Writing Secure Code: Chapter 4, ""Authorization"" Page 114; Chapter 6, ""Determining Appropriate Access Control"" Page 171 Top 25 Series - Rank 5 - Improper Access Control (Authorization): http://blogs.sans.org/appsecstreetfighter/2010/03/04/top-25-series-rank-5-improper-access-control-authorization/ OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI @@ -186,16 +186,16 @@ Authentication using JAAS: http://www.javaranch.com/journal/2008/04/authenticati The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Authorization"", Page 39." CWE-863,EN-Incorrect Authorization (Type: Class),"The software performs an authorization check when an actor attempts to access a resource or perform an action, but it does not correctly perform the check. This allows attackers to bypass intended access restrictions. Assuming a user with a given identity, authorization is the process of determining whether that user can access a given resource, based on the user's privileges and any permissions or other access-control specifications that apply to the resource. -When access control checks are incorrectly applied, users are able to access data or perform actions that they should not be allowed to perform. This can lead to a wide range of problems, including information exposures, denial of service, and arbitrary code execution.",,High,"Role Based Access Control and Role Based Security: http://csrc.nist.gov/groups/SNS/rbac/ +When access control checks are incorrectly applied, users are able to access data or perform actions that they should not be allowed to perform. This can lead to a wide range of problems, including information exposures, denial of service, and arbitrary code execution.",,high,"Role Based Access Control and Role Based Security: http://csrc.nist.gov/groups/SNS/rbac/ Writing Secure Code: Chapter 4, ""Authorization"" Page 114; Chapter 6, ""Determining Appropriate Access Control"" Page 171 Top 25 Series - Rank 5 - Improper Access Control (Authorization): http://blogs.sans.org/appsecstreetfighter/2010/03/04/top-25-series-rank-5-improper-access-control-authorization/ Authentication using JAAS: http://www.javaranch.com/journal/2008/04/authentication-using-JAAS.html OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Authorization"", Page 39." CWE-99,EN-Improper Control of Resource Identifiers (Resource Injection) (Type: Base),"The software receives input from an upstream component, but it does not restrict or incorrectly restricts the input before it is used as an identifier for a resource that may be outside the intended sphere of control. -This may enable an attacker to access or modify otherwise protected system resources.",,High, +This may enable an attacker to access or modify otherwise protected system resources.",,high, CWE-120,EN-Buffer Copy without Checking Size of Input (Classic Buffer Overflow) (Type: Base),"The program copies an input buffer to an output buffer without verifying that the size of the input buffer is less than the size of the output buffer, leading to a buffer overflow. -A buffer overflow condition exists when a program attempts to put more data in a buffer than it can hold, or when a program attempts to put data in a memory area outside of the boundaries of a buffer. The simplest type of error, and the most common cause of buffer overflows, is the ""classic"" case in which the program copies the buffer without restricting how much is copied. Other variants exist, but the existence of a classic overflow strongly suggests that the programmer is not considering even the most basic of security protections.",,High to Very High,"Writing Secure Code: Chapter 5, ""Public Enemy #1: The Buffer Overrun"" Page 127 +A buffer overflow condition exists when a program attempts to put more data in a buffer than it can hold, or when a program attempts to put data in a memory area outside of the boundaries of a buffer. The simplest type of error, and the most common cause of buffer overflows, is the ""classic"" case in which the program copies the buffer without restricting how much is copied. Other variants exist, but the existence of a classic overflow strongly suggests that the programmer is not considering even the most basic of security protections.",,high,"Writing Secure Code: Chapter 5, ""Public Enemy #1: The Buffer Overrun"" Page 127 24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89 Using the Strsafe.h Functions: http://msdn.microsoft.com/en-us/library/ms647466.aspx Safe C String Library v1.0.3: http://www.zork.org/safestr/ @@ -209,12 +209,12 @@ The Art of Software Security Assessment: Chapter 3, ""Nonexecutable Stack"", Pag The Art of Software Security Assessment: Chapter 5, ""Protection Mechanisms"", Page 189. The Art of Software Security Assessment: Chapter 8, ""C String Handling"", Page 388." CWE-122,EN-Heap-based Buffer Overflow (Type: Variant),"A heap overflow condition is a buffer overflow, where the buffer that can be overwritten is allocated in the heap portion of memory, generally meaning that the buffer was allocated using a routine such as malloc(). -A buffer overflow condition exists when a program attempts to put more data in a buffer than it can hold, or when a program attempts to put data in a memory area outside of the boundaries of a buffer. The simplest type of error, and the most common cause of buffer overflows, is the ""classic"" case in which the program copies the buffer without restricting how much is copied. Other variants exist, but the existence of a classic overflow strongly suggests that the programmer is not considering even the most basic of security protections.",,High to Very High,"Writing Secure Code: Chapter 5, ""Heap Overruns"" Page 138 +A buffer overflow condition exists when a program attempts to put more data in a buffer than it can hold, or when a program attempts to put data in a memory area outside of the boundaries of a buffer. The simplest type of error, and the most common cause of buffer overflows, is the ""classic"" case in which the program copies the buffer without restricting how much is copied. Other variants exist, but the existence of a classic overflow strongly suggests that the programmer is not considering even the most basic of security protections.",,high,"Writing Secure Code: Chapter 5, ""Heap Overruns"" Page 138 24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89 The Art of Software Security Assessment: Chapter 3, ""Nonexecutable Stack"", Page 76. The Art of Software Security Assessment: Chapter 5, ""Protection Mechanisms"", Page 189." CWE-131,EN-Incorrect Calculation of Buffer Size (Type: Base),"The software does not correctly calculate the size to be used when allocating a buffer, which could lead to a buffer overflow. -If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,High to Very High,"SafeInt: http://safeint.codeplex.com/ +If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,high,"SafeInt: http://safeint.codeplex.com/ Top 25 Series - Rank 18 - Incorrect Calculation of Buffer Size: http://software-security.sans.org/blog/2010/03/19/top-25-series-rank-18-incorrect-calculation-of-buffer-size Address Space Layout Randomization in Windows Vista: http://blogs.msdn.com/michael_howard/archive/2006/05/26/address-space-layout-randomization-in-windows-vista.aspx Understanding DEP as a mitigation technology part 1: http://blogs.technet.com/b/srd/archive/2009/06/12/understanding-dep-as-a-mitigation-technology-part-1.aspx @@ -225,31 +225,31 @@ Writing Secure Code: Chapter 20, ""Integer Overflows"" Page 620 The Art of Software Security Assessment: Chapter 8, ""Incrementing Pointers Incorrectly"", Page 401." CWE-22,EN-Improper Limitation of a Pathname to a Restricted Directory (Path Traversal) (Type: Class),"The software uses external input to construct a pathname that is intended to identify a file or directory that is located underneath a restricted parent directory, but the software does not properly neutralize special elements within the pathname that can cause the pathname to resolve to a location that is outside of the restricted directory. Many file operations are intended to take place within a restricted directory. By using special elements such as "".."" and ""/"" separators, attackers can escape outside of the restricted location to access files or directories that are elsewhere on the system. One of the most common special elements is the ""../"" sequence, which in most modern operating systems is interpreted as the parent directory of the current location. This is referred to as relative path traversal. Path traversal also covers the use of absolute pathnames such as ""/usr/local/bin"", which may also be useful in accessing unexpected files. This is referred to as absolute path traversal. -In many programming languages, the injection of a null byte (the 0 or NUL) may allow an attacker to truncate a generated filename to widen the scope of attack. For example, the software may add "".txt"" to any pathname, thus limiting the attacker to text files, but a null injection may effectively remove this restriction.",,High to Very High,"Writing Secure Code: Chapter 11, ""Directory Traversal and Using Parent Paths (..)"" Page 370 +In many programming languages, the injection of a null byte (the 0 or NUL) may allow an attacker to truncate a generated filename to widen the scope of attack. For example, the software may add "".txt"" to any pathname, thus limiting the attacker to text files, but a null injection may effectively remove this restriction.",,high,"Writing Secure Code: Chapter 11, ""Directory Traversal and Using Parent Paths (..)"" Page 370 OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI Testing for Path Traversal (OWASP-AZ-001): http://www.owasp.org/index.php/Testing_for_Path_Traversal_(OWASP-AZ-001) Top 25 Series - Rank 7 - Path Traversal: http://blogs.sans.org/appsecstreetfighter/2010/03/09/top-25-series-rank-7-path-traversal/ Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html The Art of Software Security Assessment: Chapter 9, ""Filenames and Paths"", Page 503." CWE-311,EN-Missing Encryption of Sensitive Data (Type: Base),"The software does not encrypt sensitive or critical information before storage or transmission. -The lack of proper data encryption passes up the guarantees of confidentiality, integrity, and accountability that properly implemented encryption conveys.",,High to Very High,"Writing Secure Code: Chapter 9, ""Protecting Secret Data"" Page 299 +The lack of proper data encryption passes up the guarantees of confidentiality, integrity, and accountability that properly implemented encryption conveys.",,high,"Writing Secure Code: Chapter 9, ""Protecting Secret Data"" Page 299 24 Deadly Sins of Software Security: ""Sin 17: Failure to Protect Stored Data."" Page 253 Top 25 Series - Rank 10 - Missing Encryption of Sensitive Data: http://blogs.sans.org/appsecstreetfighter/2010/02/26/top-25-series-rank-10-missing-encryption-of-sensitive-data/ The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Encryption"", Page 43. SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf" CWE-464,EN-Addition of Data Structure Sentinel (Type: Base),"The accidental addition of a data-structure sentinel can cause serious programming logic problems. -Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,High to Very High, +Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,high, CWE-67,EN-Improper Handling of Windows Device Names (Type: Variant),"The software constructs pathnames from user input, but it does not handle or incorrectly handles a pathname containing a Windows device name such as AUX or CON. This typically leads to denial of service or an information exposure when the application attempts to process the pathname as a regular file. -Not properly handling virtual filenames (e.g. AUX, CON, PRN, COM1, LPT1) can result in different types of vulnerabilities. In some cases an attacker can request a device via injection of a virtual filename in a URL, which may cause an error that leads to a denial of service or an error page that reveals sensitive information. A software system that allows device names to bypass filtering runs the risk of an attacker injecting malicious code in a file with the name of a device.",,High to Very High,"Writing Secure Code +Not properly handling virtual filenames (e.g. AUX, CON, PRN, COM1, LPT1) can result in different types of vulnerabilities. In some cases an attacker can request a device via injection of a virtual filename in a URL, which may cause an error that leads to a denial of service or an error page that reveals sensitive information. A software system that allows device names to bypass filtering runs the risk of an attacker injecting malicious code in a file with the name of a device.",,high,"Writing Secure Code The Art of Software Security Assessment: Chapter 11, ""Device Files"", Page 666." CWE-73,EN-External Control of File Name or Path (Type: Class),"The software allows user input to control or influence paths or file names that are used in filesystem operations. This could allow an attacker to access or modify system files or other files that are critical to the application. Path manipulation errors occur when the following two conditions are met: 1. An attacker can specify a path used in an operation on the filesystem. 2. By specifying the resource, the attacker gains a capability that would not otherwise be permitted. -For example, the program may give the attacker the ability to overwrite the specified file or run with a configuration controlled by the attacker.",,High to Very High,OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI +For example, the program may give the attacker the ability to overwrite the specified file or run with a configuration controlled by the attacker.",,high,OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI CWE-76,EN-Improper Neutralization of Equivalent Special Elements (Type: Base),"The software properly neutralizes certain special elements, but it improperly neutralizes equivalent special elements. -The software may have a fixed list of special characters it believes is complete. However, there may be alternate encodings, or representations that also have the same meaning. For example, the software may filter out a leading slash (/) to prevent absolute path names, but does not account for a tilde (~) followed by a user name, which on some *nix systems could be expanded to an absolute pathname. Alternately, the software might filter a dangerous ""-e"" command-line switch when calling an external program, but it might not account for ""--exec"" or other switches that have the same semantics.",,High to Very High, +The software may have a fixed list of special characters it believes is complete. However, there may be alternate encodings, or representations that also have the same meaning. For example, the software may filter out a leading slash (/) to prevent absolute path names, but does not account for a tilde (~) followed by a user name, which on some *nix systems could be expanded to an absolute pathname. Alternately, the software might filter a dangerous ""-e"" command-line switch when calling an external program, but it might not account for ""--exec"" or other switches that have the same semantics.",,high, CWE-79,EN-Improper Neutralization of Input During Web Page Generation (Cross-site Scripting) (Type: Base),"The software does not neutralize or incorrectly neutralizes user-controllable input before it is placed in output that is used as a web page that is served to other users. Cross-site scripting (XSS) vulnerabilities occur when: 1. Untrusted data enters a web application, typically from a web request. @@ -263,7 +263,7 @@ The server reads data directly from the HTTP request and reflects it back in the The application stores dangerous data in a database, message forum, visitor log, or other trusted data store. At a later time, the dangerous data is subsequently read back into the application and included in dynamic content. From an attacker's perspective, the optimal place to inject malicious content is in an area that is displayed to either many users or particularly interesting users. Interesting users typically have elevated privileges in the application or interact with sensitive data that is valuable to the attacker. If one of these users executes malicious content, the attacker may be able to perform privileged operations on behalf of the user or gain access to sensitive data belonging to the user. For example, the attacker might inject XSS into a log message, which might not be handled properly when an administrator views the logs. In DOM-based XSS, the client performs the injection of XSS into the page; in the other types, the server performs the injection. DOM-based XSS generally involves server-controlled, trusted script that is sent to the client, such as Javascript that performs sanity checks on a form before the user submits it. If the server-supplied script processes user-supplied data and then injects it back into the web page (such as with dynamic HTML), then DOM-based XSS is possible. Once the malicious script is injected, the attacker can perform a variety of malicious activities. The attacker could transfer private information, such as cookies that may include session information, from the victim's machine to the attacker. The attacker could send malicious requests to a web site on behalf of the victim, which could be especially dangerous to the site if the victim has administrator privileges to manage that site. Phishing attacks could be used to emulate trusted web sites and trick the victim into entering a password, allowing the attacker to compromise the victim's account on that web site. Finally, the script could exploit a vulnerability in the web browser itself possibly taking over the victim's machine, sometimes referred to as ""drive-by hacking."" -In many cases, the attack can be launched without the victim even being aware of it. Even with careful users, attackers frequently use a variety of methods to encode the malicious portion of the attack, such as URL encoding or Unicode, so the request looks less suspicious.",,High to Very High,"XSS Attacks +In many cases, the attack can be launched without the victim even being aware of it. Even with careful users, attackers frequently use a variety of methods to encode the malicious portion of the attack, such as URL encoding or Unicode, so the request looks less suspicious.",,high,"XSS Attacks 24 Deadly Sins of Software Security: ""Sin 2: Web-Server Related Vulnerabilities (XSS, XSRF, and Response Splitting)."" Page 31 24 Deadly Sins of Software Security: ""Sin 3: Web-Client Related Vulnerabilities (XSS)."" Page 63 Cross-site scripting: http://en.wikipedia.org/wiki/Cross-site_scripting @@ -283,24 +283,24 @@ DOM based XSS Prevention Cheat Sheet: http://www.owasp.org/index.php/DOM_based_X Top 25 series - Rank 1 - Cross Site Scripting: http://blogs.sans.org/appsecstreetfighter/2010/02/22/top-25-series-rank-1-cross-site-scripting/ The Art of Software Security Assessment: Chapter 17, ""Cross Site Scripting"", Page 1071." CWE-80,EN-Improper Neutralization of Script-Related HTML Tags in a Web Page (Basic XSS) (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special characters such as ""<"", "">"", and ""&"" that could be interpreted as web-scripting elements when they are sent to a downstream component that processes web pages. -This may allow such characters to be treated as control characters, which are executed client-side in the context of the user's session. Although this can be classified as an injection problem, the more pertinent issue is the improper conversion of such special characters to respective context-appropriate entities before displaying them to the user.",,High to Very High, +This may allow such characters to be treated as control characters, which are executed client-side in the context of the user's session. Although this can be classified as an injection problem, the more pertinent issue is the improper conversion of such special characters to respective context-appropriate entities before displaying them to the user.",,high, CWE-98,EN-Improper Control of Filename for Include/Require Statement in PHP Program (PHP Remote File Inclusion) (Type: Base),"The PHP application receives input from an upstream component, but it does not restrict or incorrectly restricts the input before its usage in ""require,"" ""include,"" or similar functions. -In certain versions and configurations of PHP, this can allow an attacker to specify a URL to a remote location from which the software will obtain the code to execute. In other cases in association with path traversal, the attacker can specify a local file that may contain executable statements that can be parsed by PHP.",,High to Very High,"Testing for Path Traversal (OWASP-AZ-001): http://www.owasp.org/index.php/Testing_for_Path_Traversal_(OWASP-AZ-001) +In certain versions and configurations of PHP, this can allow an attacker to specify a URL to a remote location from which the software will obtain the code to execute. In other cases in association with path traversal, the attacker can specify a local file that may contain executable statements that can be parsed by PHP.",,high,"Testing for Path Traversal (OWASP-AZ-001): http://www.owasp.org/index.php/Testing_for_Path_Traversal_(OWASP-AZ-001) Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html A Study in Scarlet: http://www.cgisecurity.com/lib/studyinscarlet.txt Suhosin: http://www.hardened-php.net/suhosin/ Top 25 Series - Rank 13 - PHP File Inclusion: http://blogs.sans.org/appsecstreetfighter/2010/03/11/top-25-series-rank-13-php-file-inclusion/" CWE-188,EN-Reliance on Data/Memory Layout (Type: Base),"The software makes invalid assumptions about how protocol data or memory is organized at a lower level, resulting in unintended program behavior. -For example, an attacker might succeed in authentication by providing a small password that matches the associated portion of the larger, correct password.",,Low,"The Art of Software Security Assessment: Chapter 6, ""Structure Padding"", Page 284." +For example, an attacker might succeed in authentication by providing a small password that matches the associated portion of the larger, correct password.",,low,"The Art of Software Security Assessment: Chapter 6, ""Structure Padding"", Page 284." CWE-197,EN-Numeric Truncation Error (Type: Base),"Truncation errors occur when a primitive is cast to a primitive of a smaller size and data is lost in the conversion. -When a primitive is cast to a smaller primitive, the high order bits of the large value are lost in the conversion, potentially resulting in an unexpected value that is not equal to the original value. This value may be required as an index into a buffer, a loop iterator, or simply necessary state data. In any case, the value cannot be trusted and the system will be in an undefined state. While this method may be employed viably to isolate the low bits of a value, this usage is rare, and truncation usually implies that an implementation error has occurred.",,Low,"The Art of Software Security Assessment: Chapter 6, ""Truncation"", Page 259." +When a primitive is cast to a smaller primitive, the high order bits of the large value are lost in the conversion, potentially resulting in an unexpected value that is not equal to the original value. This value may be required as an index into a buffer, a loop iterator, or simply necessary state data. In any case, the value cannot be trusted and the system will be in an undefined state. While this method may be employed viably to isolate the low bits of a value, this usage is rare, and truncation usually implies that an implementation error has occurred.",,low,"The Art of Software Security Assessment: Chapter 6, ""Truncation"", Page 259." CWE-252,EN-Unchecked Return Value (Type: Base),"The software does not check the return value from a method or function, which can prevent it from detecting unexpected states and conditions. -Two common programmer assumptions are ""this function call can never fail"" and ""it doesn't matter if this function call fails"". If an attacker can force the function to fail or otherwise return a value that is not expected, then the subsequent program logic could lead to a vulnerability, because the software is not in a state that the programmer assumes. For example, if the program calls a function to drop privileges but does not check the return code to ensure that privileges were successfully dropped, then the program will continue to operate with the higher privileges.",,Low,"The Art of Software Security Assessment: Chapter 7, ""Program Building Blocks"" Page 341. +Two common programmer assumptions are ""this function call can never fail"" and ""it doesn't matter if this function call fails"". If an attacker can force the function to fail or otherwise return a value that is not expected, then the subsequent program logic could lead to a vulnerability, because the software is not in a state that the programmer assumes. For example, if the program calls a function to drop privileges but does not check the return code to ensure that privileges were successfully dropped, then the program will continue to operate with the higher privileges.",,low,"The Art of Software Security Assessment: Chapter 7, ""Program Building Blocks"" Page 341. Writing Secure Code: Chapter 20, ""Checking Returns"" Page 624 24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183 ERR10-CPP. Check for error conditions: https://www.securecoding.cert.org/confluence/display/cplusplus/ERR10-CPP.+Check+for+error+conditions" CWE-253,EN-Incorrect Check of Function Return Value (Type: Base),"The software incorrectly checks a return value from a function, which prevents the software from detecting errors or exceptional conditions. -Two common programmer assumptions are ""this function call can never fail"" and ""it doesn't matter if this function call fails"". If an attacker can force the function to fail or otherwise return a value that is not expected, then the subsequent program logic could lead to a vulnerability, because the software is not in a state that the programmer assumes. For example, if the program calls a function to drop privileges but does not check the return code to ensure that privileges were successfully dropped, then the program will continue to operate with the higher privileges.",,Low,"The Art of Software Security Assessment: Chapter 7, ""Return Value Testing and Interpretation"", Page 340." +Two common programmer assumptions are ""this function call can never fail"" and ""it doesn't matter if this function call fails"". If an attacker can force the function to fail or otherwise return a value that is not expected, then the subsequent program logic could lead to a vulnerability, because the software is not in a state that the programmer assumes. For example, if the program calls a function to drop privileges but does not check the return code to ensure that privileges were successfully dropped, then the program will continue to operate with the higher privileges.",,low,"The Art of Software Security Assessment: Chapter 7, ""Return Value Testing and Interpretation"", Page 340." CWE-296,EN-Improper Following of a Certificates Chain of Trust (Type: Base),"The software does not follow, or incorrectly follows, the chain of trust for a certificate back to a trusted root certificate, resulting in incorrect trust of any resource that is associated with that certificate. If a system does not follow the chain of trust of a certificate to a root server, the certificate loses all usefulness as a metric of trust. Essentially, the trust gained from a certificate is derived from a chain of trust -- with a reputable trusted entity at the end of that list. The end user must trust that reputable source, and this reputable source must vouch for the resource in question through the medium of the certificate. In some cases, this trust traverses several entities who vouch for one another. The entity trusted by the end user is at one end of this trust chain, while the certificate-wielding resource is at the other end of the chain. If the user receives a certificate at the end of one of these trust chains and then proceeds to check only that the first link in the chain, no real trust has been derived, since the entire chain must be traversed back to a trusted source to verify the certificate. @@ -308,65 +308,65 @@ There are several ways in which the chain of trust might be broken, including bu Any certificate in the chain is self-signed, unless it the root. Not every intermediate certificate is checked, starting from the original certificate all the way up to the root certificate. An intermediate, CA-signed certificate does not have the expected Basic Constraints or other important extensions. -The root certificate has been compromised or authorized to the wrong party.",,Low,"The Most Dangerous Code in the World: Validating SSL Certificates in Non-Browser Software: http://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf +The root certificate has been compromised or authorized to the wrong party.",,low,"The Most Dangerous Code in the World: Validating SSL Certificates in Non-Browser Software: http://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf 24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347" CWE-298,EN-Improper Validation of Certificate Expiration (Type: Variant),"A certificate expiration is not validated or is incorrectly validated, so trust may be assigned to certificates that have been abandoned due to age. -When the expiration of a certificate is not taken into account, no trust has necessarily been conveyed through it. Therefore, the validity of the certificate cannot be verified and all benefit of the certificate is lost.",,Low,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347" +When the expiration of a certificate is not taken into account, no trust has necessarily been conveyed through it. Therefore, the validity of the certificate cannot be verified and all benefit of the certificate is lost.",,low,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347" CWE-324,EN-Use of a Key Past its Expiration Date (Type: Base),"The product uses a cryptographic key or password past its expiration date, which diminishes its safety significantly by increasing the timing window for cracking attacks against that key. -While the expiration of keys does not necessarily ensure that they are compromised, it is a significant concern that keys which remain in use for prolonged periods of time have a decreasing probability of integrity. For this reason, it is important to replace keys within a period of time proportional to their strength.",,Low,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347" +While the expiration of keys does not necessarily ensure that they are compromised, it is a significant concern that keys which remain in use for prolonged periods of time have a decreasing probability of integrity. For this reason, it is important to replace keys within a period of time proportional to their strength.",,low,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347" CWE-379,EN-Creation of Temporary File in Directory with Incorrect Permissions (Type: Base),"The software creates a temporary file in a directory whose permissions allow unintended actors to determine the file's existence or otherwise access that file. -On some operating systems, the fact that the temporary file exists may be apparent to any user with sufficient privileges to access that directory. Since the file is visible, the application that is using the temporary file could be known. If one has access to list the processes on the system, the attacker has gained information about what the user is doing at that time. By correlating this with the applications the user is running, an attacker could potentially discover what a user's actions are. From this, higher levels of security could be breached.",,Low,"The Art of Software Security Assessment: Chapter 9, ""Temporary Files"", Page 538." +On some operating systems, the fact that the temporary file exists may be apparent to any user with sufficient privileges to access that directory. Since the file is visible, the application that is using the temporary file could be known. If one has access to list the processes on the system, the attacker has gained information about what the user is doing at that time. By correlating this with the applications the user is running, an attacker could potentially discover what a user's actions are. From this, higher levels of security could be breached.",,low,"The Art of Software Security Assessment: Chapter 9, ""Temporary Files"", Page 538." CWE-462,EN-Duplicate Key in Associative List (Alist) (Type: Base),"Duplicate keys in associative lists can lead to non-unique keys being mistaken for an error. -A duplicate key entry -- if the alist is designed properly -- could be used as a constant time replace function. However, duplicate key entries could be inserted by mistake. Because of this ambiguity, duplicate key entries in an association list are not recommended and should not be allowed.",,Low, +A duplicate key entry -- if the alist is designed properly -- could be used as a constant time replace function. However, duplicate key entries could be inserted by mistake. Because of this ambiguity, duplicate key entries in an association list are not recommended and should not be allowed.",,low, CWE-479,EN-Signal Handler Use of a Non-reentrant Function (Type: Variant),"The program defines a signal handler that calls a non-reentrant function. Non-reentrant functions are functions that cannot safely be called, interrupted, and then recalled before the first call has finished without resulting in memory corruption. This can lead to an unexpected system state an unpredictable results with a variety of potential consequences depending on context, including denial of service and code execution. -Many functions are not reentrant, but some of them can result in the corruption of memory if they are used in a signal handler. The function call syslog() is an example of this. In order to perform its functionality, it allocates a small amount of memory as ""scratch space."" If syslog() is suspended by a signal call and the signal handler calls syslog(), the memory used by both of these functions enters an undefined, and possibly, exploitable state. Implementations of malloc() and free() manage metadata in global structures in order to track which memory is allocated versus which memory is available, but they are non-reentrant. Simultaneous calls to these functions can cause corruption of the metadata.",,Low,"The Art of Software Security Assessment: Chapter 13, ""Signal Vulnerabilities"", Page 791." +Many functions are not reentrant, but some of them can result in the corruption of memory if they are used in a signal handler. The function call syslog() is an example of this. In order to perform its functionality, it allocates a small amount of memory as ""scratch space."" If syslog() is suspended by a signal call and the signal handler calls syslog(), the memory used by both of these functions enters an undefined, and possibly, exploitable state. Implementations of malloc() and free() manage metadata in global structures in order to track which memory is allocated versus which memory is available, but they are non-reentrant. Simultaneous calls to these functions can cause corruption of the metadata.",,low,"The Art of Software Security Assessment: Chapter 13, ""Signal Vulnerabilities"", Page 791." CWE-480,EN-Use of Incorrect Operator (Type: Base),"The programmer accidentally uses the wrong operator, which changes the application logic in security-relevant ways. Non-reentrant functions are functions that cannot safely be called, interrupted, and then recalled before the first call has finished without resulting in memory corruption. This can lead to an unexpected system state an unpredictable results with a variety of potential consequences depending on context, including denial of service and code execution. -Many functions are not reentrant, but some of them can result in the corruption of memory if they are used in a signal handler. The function call syslog() is an example of this. In order to perform its functionality, it allocates a small amount of memory as ""scratch space."" If syslog() is suspended by a signal call and the signal handler calls syslog(), the memory used by both of these functions enters an undefined, and possibly, exploitable state. Implementations of malloc() and free() manage metadata in global structures in order to track which memory is allocated versus which memory is available, but they are non-reentrant. Simultaneous calls to these functions can cause corruption of the metadata.",,Low,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289." +Many functions are not reentrant, but some of them can result in the corruption of memory if they are used in a signal handler. The function call syslog() is an example of this. In order to perform its functionality, it allocates a small amount of memory as ""scratch space."" If syslog() is suspended by a signal call and the signal handler calls syslog(), the memory used by both of these functions enters an undefined, and possibly, exploitable state. Implementations of malloc() and free() manage metadata in global structures in order to track which memory is allocated versus which memory is available, but they are non-reentrant. Simultaneous calls to these functions can cause corruption of the metadata.",,low,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289." CWE-481,EN-Assigning instead of Comparing (Type: Variant),"The code uses an operator for assignment when the intention was to perform a comparison. -In many languages the compare statement is very close in appearance to the assignment statement and are often confused. This bug is generally the result of a typo and usually causes obvious problems with program execution. If the comparison is in an if statement, the if statement will usually evaluate the value of the right-hand side of the predicate.",,Low,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289." +In many languages the compare statement is very close in appearance to the assignment statement and are often confused. This bug is generally the result of a typo and usually causes obvious problems with program execution. If the comparison is in an if statement, the if statement will usually evaluate the value of the right-hand side of the predicate.",,low,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289." CWE-482,EN-Comparing instead of Assigning (Type: Variant),"The code uses an operator for comparison when the intention was to perform an assignment. -In many languages, the compare statement is very close in appearance to the assignment statement; they are often confused.",,Low,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289." +In many languages, the compare statement is very close in appearance to the assignment statement; they are often confused.",,low,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289." CWE-483,EN-Incorrect Block Delimitation (Type: Variant),"The code does not explicitly delimit a block that is intended to contain 2 or more statements, creating a logic error. -In some languages, braces (or other delimiters) are optional for blocks. When the delimiter is omitted, it is possible to insert a logic error in which a statement is thought to be in a block but is not. In some cases, the logic error can have security implications.",,Low, +In some languages, braces (or other delimiters) are optional for blocks. When the delimiter is omitted, it is possible to insert a logic error in which a statement is thought to be in a block but is not. In some cases, the logic error can have security implications.",,low, CWE-641,EN-Improper Restriction of Names for Files and Other Resources (Type: Base),"The application constructs the name of a file or other resource using input from an upstream component, but does not restrict or incorrectly restricts the resulting name. -This may produce resultant weaknesses. For instance, if the names of these resources contain scripting characters, it is possible that a script may get executed in the client's browser if the application ever displays the name of the resource on a dynamically generated web page. Alternately, if the resources are consumed by some application parser, a specially crafted name can exploit some vulnerability internal to the parser, potentially resulting in execution of arbitrary code on the server machine. The problems will vary based on the context of usage of such malformed resource names and whether vulnerabilities are present in or assumptions are made by the targeted technology that would make code execution possible.",,Low, +This may produce resultant weaknesses. For instance, if the names of these resources contain scripting characters, it is possible that a script may get executed in the client's browser if the application ever displays the name of the resource on a dynamically generated web page. Alternately, if the resources are consumed by some application parser, a specially crafted name can exploit some vulnerability internal to the parser, potentially resulting in execution of arbitrary code on the server machine. The problems will vary based on the context of usage of such malformed resource names and whether vulnerabilities are present in or assumptions are made by the targeted technology that would make code execution possible.",,low, CWE-648,EN-Incorrect Use of Privileged APIs (Type: Base),"The application does not conform to the API requirements for a function call that requires extra privileges. This could allow attackers to gain privileges by causing the function to be called incorrectly. When an application contains certain functions that perform operations requiring an elevated level of privilege, the caller of a privileged API must be careful to: ensure that assumptions made by the APIs are valid, such as validity of arguments account for known weaknesses in the design/implementation of the API call the API from a safe context If the caller of the API does not follow these requirements, then it may allow a malicious user or process to elevate their privilege, hijack the process, or steal sensitive data. -For instance, it is important to know if privileged APIs do not shed their privileges before returning to the caller or if the privileged function might make certain assumptions about the data, context or state information passed to it by the caller. It is important to always know when and how privileged APIs can be called in order to ensure that their elevated level of privilege cannot be exploited.",,Low, +For instance, it is important to know if privileged APIs do not shed their privileges before returning to the caller or if the privileged function might make certain assumptions about the data, context or state information passed to it by the caller. It is important to always know when and how privileged APIs can be called in order to ensure that their elevated level of privilege cannot be exploited.",,low, CWE-762,EN-Mismatched Memory Management Routines (Type: Variant),"The application attempts to return a memory resource to the system, but it calls a release function that is not compatible with the function that was originally used to allocate that resource. This weakness can be generally described as mismatching memory management routines, such as: The memory was allocated on the stack (automatically), but it was deallocated using the memory management routine free() (CWE-590), which is intended for explicitly allocated heap memory. The memory was allocated explicitly using one set of memory management functions, and deallocated using a different set. For example, memory might be allocated with malloc() in C++ instead of the new operator, and then deallocated with the delete operator. -When the memory management functions are mismatched, the consequences may be as severe as code execution, memory corruption, or program crash. Consequences and ease of exploit will vary depending on the implementation of the routines and the object being managed.",,Low,"boost C++ Library Smart Pointers: http://www.boost.org/doc/libs/1_38_0/libs/smart_ptr/smart_ptr.htm +When the memory management functions are mismatched, the consequences may be as severe as code execution, memory corruption, or program crash. Consequences and ease of exploit will vary depending on the implementation of the routines and the object being managed.",,low,"boost C++ Library Smart Pointers: http://www.boost.org/doc/libs/1_38_0/libs/smart_ptr/smart_ptr.htm Valgrind: http://valgrind.org/" CWE-783,EN-Operator Precedence Logic Error (Type: Variant),"The program uses an expression in which operator precedence causes incorrect logic to be used. -While often just a bug, operator precedence logic errors can have serious consequences if they are used in security-critical code, such as making an authentication decision.",,Low,"EXP00-C. Use parentheses for precedence of operation: https://www.securecoding.cert.org/confluence/display/seccode/EXP00-C.+Use+parentheses+for+precedence+of+operation +While often just a bug, operator precedence logic errors can have serious consequences if they are used in security-critical code, such as making an authentication decision.",,low,"EXP00-C. Use parentheses for precedence of operation: https://www.securecoding.cert.org/confluence/display/seccode/EXP00-C.+Use+parentheses+for+precedence+of+operation The Art of Software Security Assessment: Chapter 6, ""Precedence"", Page 287." CWE-789,EN-Uncontrolled Memory Allocation (Type: Variant),"The product allocates memory based on an untrusted size value, but it does not validate or incorrectly validates the size, allowing arbitrary amounts of memory to be allocated. -This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. These problems may be resultant from missing sentinel values (CWE-463) or trusting a user-influenced input length variable.",,Low,"The Art of Software Security Assessment: Chapter 10, ""Resource Limits"", Page 574." +This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. These problems may be resultant from missing sentinel values (CWE-463) or trusting a user-influenced input length variable.",,low,"The Art of Software Security Assessment: Chapter 10, ""Resource Limits"", Page 574." CWE-333,EN-Improper Handling of Insufficient Entropy in TRNG (Type: Variant),"True random number generators (TRNG) generally have a limited source of entropy and therefore can fail or block. -The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,Low to Medium, +The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,medium, CWE-367,EN-Time-of-check Time-of-use (TOCTOU) Race Condition (Type: Base),"The software checks the state of a resource before using that resource, but the resource's state can change between the check and the use in a way that invalidates the results of the check. This can cause the software to perform invalid actions when the resource is in an unexpected state. -This weakness can be security-relevant when an attacker can influence the state of the resource between check and use. This can happen with shared resources such as files, memory, or even variables in multithreaded programs.",,Low to Medium,"Portably Solving File TOCTTOU Races with Hardness Amplification: http://www.usenix.org/events/fast08/tech/tsafrir.html +This weakness can be security-relevant when an attacker can influence the state of the resource between check and use. This can happen with shared resources such as files, memory, or even variables in multithreaded programs.",,medium,"Portably Solving File TOCTTOU Races with Hardness Amplification: http://www.usenix.org/events/fast08/tech/tsafrir.html 24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205 The Art of Software Security Assessment: Chapter 9, ""TOCTOU"", Page 527." CWE-404,EN-Improper Resource Shutdown or Release (Type: Base),"The program does not release or incorrectly releases a resource before it is made available for re-use. -When a resource is created or allocated, the developer is responsible for properly releasing the resource as well as accounting for all potential paths of expiration or invalidation, such as a set period of time or revocation.",,Low to Medium,"24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143" +When a resource is created or allocated, the developer is responsible for properly releasing the resource as well as accounting for all potential paths of expiration or invalidation, such as a set period of time or revocation.",,medium,"24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143" CWE-407,EN-Algorithmic Complexity (Type: Base),"An algorithm in a product has an inefficient worst-case computational complexity that may be detrimental to system performance and can be triggered by an attacker, typically using crafted manipulations that ensure that the worst case is being reached. -In the absence of a policy to restrict asymmetric resource consumption, the application or system cannot distinguish between legitimate transmissions and traffic intended to serve as an amplifying attack on target systems. Systems can often be configured to restrict the amount of traffic sent out on behalf of a client, based on the client's origin or access level. This is usually defined in a resource allocation policy. In the absence of a mechanism to keep track of transmissions, the system or application can be easily abused to transmit asymmetrically greater traffic than the request or client should be permitted to.",,Low to Medium,Algorithmic Complexity Attacks: http://www.cs.rice.edu/~scrosby/hash/CrosbyWallach_UsenixSec2003/index.html +In the absence of a policy to restrict asymmetric resource consumption, the application or system cannot distinguish between legitimate transmissions and traffic intended to serve as an amplifying attack on target systems. Systems can often be configured to restrict the amount of traffic sent out on behalf of a client, based on the client's origin or access level. This is usually defined in a resource allocation policy. In the absence of a mechanism to keep track of transmissions, the system or application can be easily abused to transmit asymmetrically greater traffic than the request or client should be permitted to.",,medium,Algorithmic Complexity Attacks: http://www.cs.rice.edu/~scrosby/hash/CrosbyWallach_UsenixSec2003/index.html CWE-415,EN-Double Free (Type: Variant),"The product calls free() twice on the same memory address, potentially leading to modification of unexpected memory locations. -When a program calls free() twice with the same argument, the program's memory management data structures become corrupted. This corruption can cause the program to crash or, in some circumstances, cause two later calls to malloc() to return the same pointer. If malloc() returns the same value twice and the program later gives the attacker control over the data that is written into this doubly-allocated memory, the program becomes vulnerable to a buffer overflow attack.",,Low to Medium,"24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143 +When a program calls free() twice with the same argument, the program's memory management data structures become corrupted. This corruption can cause the program to crash or, in some circumstances, cause two later calls to malloc() to return the same pointer. If malloc() returns the same value twice and the program later gives the attacker control over the data that is written into this doubly-allocated memory, the program becomes vulnerable to a buffer overflow attack.",,medium,"24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143 The Art of Software Security Assessment: Chapter 7, ""Double Frees"", Page 379." CWE-59,EN-Improper Link Resolution Before File Access (Link Following) (Type: Base),"The software attempts to access a file based on the filename, but it does not properly prevent that filename from identifying a link or shortcut that resolves to an unintended resource. -Some functions that offer security features supported by the OS are not available on all versions of the OS in common use. Likewise, functions are often deprecated or made obsolete for security reasons and should not be used.",,Low to Medium,"The Art of Software Security Assessment: Chapter 9, ""Symbolic Link Attacks"", Page 518." +Some functions that offer security features supported by the OS are not available on all versions of the OS in common use. Likewise, functions are often deprecated or made obsolete for security reasons and should not be used.",,medium,"The Art of Software Security Assessment: Chapter 9, ""Symbolic Link Attacks"", Page 518." CWE-601,EN-URL Redirection to Untrusted Site (Open Redirect) (Type: Variant),"A web application accepts a user-controlled input that specifies a link to an external site, and uses that link in a Redirect. This simplifies phishing attacks. -An http parameter may contain a URL value and could cause the web application to redirect the request to the specified URL. By modifying the URL value to a malicious site, an attacker may successfully launch a phishing scam and steal user credentials. Because the server name in the modified link is identical to the original site, phishing attempts have a more trustworthy appearance.",,Low to Medium,"Exploitable Redirects on the Web: Identification, Prevalence, and Defense: http://www.cs.indiana.edu/cgi-pub/cshue/research/woot08.pdf +An http parameter may contain a URL value and could cause the web application to redirect the request to the specified URL. By modifying the URL value to a malicious site, an attacker may successfully launch a phishing scam and steal user credentials. Because the server name in the modified link is identical to the original site, phishing attempts have a more trustworthy appearance.",,medium,"Exploitable Redirects on the Web: Identification, Prevalence, and Defense: http://www.cs.indiana.edu/cgi-pub/cshue/research/woot08.pdf Open redirect vulnerabilities: definition and prevention: http://www.net-security.org/dl/insecure/INSECURE-Mag-17.pdf Top 25 Series - Rank 23 - Open Redirect: http://software-security.sans.org/blog/2010/03/25/top-25-series-rank-23-open-redirect OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI" @@ -374,17 +374,17 @@ CWE-749,EN-Exposed Dangerous Method or Function (Type: Base),"The software provi This weakness can lead to a wide variety of resultant weaknesses, depending on the behavior of the exposed method. It can apply to any number of technologies and approaches, such as ActiveX controls, Java functions, IOCTLs, and so on. The exposure can occur in a few different ways: 1) The function/method was never intended to be exposed to outside actors. -2) The function/method was only intended to be accessible to a limited set of actors, such as Internet-based access from a single web site.",,Low to Medium,"No description: http://msdn.microsoft.com/workshop/components/activex/safety.asp +2) The function/method was only intended to be accessible to a limited set of actors, such as Internet-based access from a single web site.",,medium,"No description: http://msdn.microsoft.com/workshop/components/activex/safety.asp No description: http://msdn.microsoft.com/workshop/components/activex/security.asp" CWE-755,EN-Improper Handling of Exceptional Conditions (Type: Class),"The software does not handle or incorrectly handles an exceptional condition. The programmer may assume that certain events or conditions will never occur or do not need to be worried about, such as low memory conditions, lack of access to resources due to restrictive permissions, or misbehaving clients or components. However, attackers may intentionally trigger these unusual conditions, thus violating the programmer's assumptions, possibly introducing instability, incorrect behavior, or a vulnerability. -Note that this entry is not exclusively about the use of exceptions and exception handling, which are mechanisms for both checking and handling unusual or unexpected conditions.",,Low to Medium, +Note that this entry is not exclusively about the use of exceptions and exception handling, which are mechanisms for both checking and handling unusual or unexpected conditions.",,medium, CWE-766,EN-Critical Variable Declared Public (Type: Variant),"The software declares a critical variable or field to be public when intended security policy requires it to be private. -When software is operating in a concurrent environment and repeatedly unlocks a critical resource, the consequences will vary based on the type of lock, the lock's implementation, and the resource being protected. In some situations such as with semaphores, the resources are pooled and extra calls to unlock will increase the count for the number of available resources, likely resulting in a crash or unpredictable behavior when the system nears capacity.",,Low to Medium, +When software is operating in a concurrent environment and repeatedly unlocks a critical resource, the consequences will vary based on the type of lock, the lock's implementation, and the resource being protected. In some situations such as with semaphores, the resources are pooled and extra calls to unlock will increase the count for the number of available resources, likely resulting in a crash or unpredictable behavior when the system nears capacity.",,medium, CWE-767,EN-Access to Critical Private Variable via Public Method (Type: Variant),"The software defines a public method that reads or modifies a private variable. -If an attacker modifies the variable to contain unexpected values, this could violate assumptions from other parts of the code. Additionally, if an attacker can read the private variable, it may expose sensitive information or make it easier to launch further attacks.",,Low to Medium, +If an attacker modifies the variable to contain unexpected values, this could violate assumptions from other parts of the code. Additionally, if an attacker can read the private variable, it may expose sensitive information or make it easier to launch further attacks.",,medium, CWE-776,EN-Improper Restriction of Recursive Entity References in DTDs (XML Entity Expansion) (Type: Variant),"The software uses XML documents and allows their structure to be defined with a Document Type Definition (DTD), but it does not properly control the number of recursive definitions of entities. -If the DTD contains a large number of nested or recursive entities, this can lead to explosive growth of data when parsed, causing a denial of service.",,Low to Medium,"Multiple vendors XML parser (and SOAP/WebServices server) Denial of Service attack using DTD: http://www.securityfocus.com/archive/1/303509 +If the DTD contains a large number of nested or recursive entities, this can lead to explosive growth of data when parsed, causing a denial of service.",,medium,"Multiple vendors XML parser (and SOAP/WebServices server) Denial of Service attack using DTD: http://www.securityfocus.com/archive/1/303509 XML security: Preventing XML bombs: http://searchsoftwarequality.techtarget.com/expert/KnowledgebaseAnswer/0,289625,sid92_gci1168442,00.html?asrc=SS_CLA_302%20%20558&psrc=CLT_92# Dismantling an XML-Bomb: http://blog.didierstevens.com/2008/09/23/dismantling-an-xml-bomb/ XML Entity Expansion: http://projects.webappsec.org/XML-Entity-Expansion @@ -392,11 +392,11 @@ Tip: Configure SAX parsers for secure processing: http://www.ibm.com/developerwo XML Denial of Service Attacks and Defenses: http://msdn.microsoft.com/en-us/magazine/ee335713.aspx Preventing Entity Expansion Attacks in JAXB: http://blog.bdoughan.com/2011/03/preventing-entity-expansion-attacks-in.html" CWE-777,EN-Regular Expression without Anchors (Type: Variant),"The software uses a regular expression to perform neutralization, but the regular expression is not anchored and may allow malicious or malformed data to slip through. -When performing tasks such as whitelist validation, data is examined and possibly modified to ensure that it is well-formed and adheres to a list of safe values. If the regular expression is not anchored, malicious or malformed data may be included before or after any string matching the regular expression. The type of malicious data that is allowed will depend on the context of the application and which anchors are omitted from the regular expression.",,Low to Medium, +When performing tasks such as whitelist validation, data is examined and possibly modified to ensure that it is well-formed and adheres to a list of safe values. If the regular expression is not anchored, malicious or malformed data may be included before or after any string matching the regular expression. The type of malicious data that is allowed will depend on the context of the application and which anchors are omitted from the regular expression.",,medium, CWE-779,EN-Logging of Excessive Data (Type: Base),"The software logs too much information, making log files hard to process and possibly hindering recovery efforts or forensic analysis after an attack. -While logging is a good practice in general, and very high levels of logging are appropriate for debugging stages of development, too much logging in a production environment might hinder a system administrator's ability to detect anomalous conditions. This can provide cover for an attacker while attempting to penetrate a system, clutter the audit trail for forensic analysis, or make it more difficult to debug problems in a production environment.",,Low to Medium, +While logging is a good practice in general, and very high levels of logging are appropriate for debugging stages of development, too much logging in a production environment might hinder a system administrator's ability to detect anomalous conditions. This can provide cover for an attacker while attempting to penetrate a system, clutter the audit trail for forensic analysis, or make it more difficult to debug problems in a production environment.",,medium, CWE-781,EN-Improper Address Validation in IOCTL with METHOD_NEITHER I/O Control Code (Type: Variant),"The software defines an IOCTL that uses METHOD_NEITHER for I/O, but it does not validate or incorrectly validates the addresses that are provided. -When an IOCTL uses the METHOD_NEITHER option for I/O control, it is the responsibility of the IOCTL to validate the addresses that have been supplied to it. If validation is missing or incorrect, attackers can supply arbitrary memory addresses, leading to code execution or a denial of service.",,Low to Medium,"Exploiting Common Flaws in Drivers: http://reversemode.com/index.php?option=com_content&task=view&id=38&Itemid=1 +When an IOCTL uses the METHOD_NEITHER option for I/O control, it is the responsibility of the IOCTL to validate the addresses that have been supplied to it. If validation is missing or incorrect, attackers can supply arbitrary memory addresses, leading to code execution or a denial of service.",,medium,"Exploiting Common Flaws in Drivers: http://reversemode.com/index.php?option=com_content&task=view&id=38&Itemid=1 Remote and Local Exploitation of Network Drivers: https://www.blackhat.com/presentations/bh-usa-07/Bulygin/Presentation/bh-usa-07-bulygin.pdf Windows driver vulnerabilities: the METHOD_NEITHER odyssey: http://www.net-security.org/dl/insecure/INSECURE-Mag-18.pdf Buffer Descriptions for I/O Control Codes: http://msdn.microsoft.com/en-us/library/ms795857.aspx @@ -405,24 +405,24 @@ Securing Device Objects: http://msdn.microsoft.com/en-us/library/ms794722.aspx No description: http://www.piotrbania.com/all/articles/ewdd.pdf" CWE-782,EN-Exposed IOCTL with Insufficient Access Control (Type: Variant),"The software implements an IOCTL with functionality that should be restricted, but it does not properly enforce access control for the IOCTL. When an IOCTL contains privileged functionality and is exposed unnecessarily, attackers may be able to access this functionality by invoking the IOCTL. Even if the functionality is benign, if the programmer has assumed that the IOCTL would only be accessed by a trusted process, there may be little or no validation of the incoming data, exposing weaknesses that would never be reachable if the attacker cannot call the IOCTL directly. -The implementations of IOCTLs will differ between operating system types and versions, so the methods of attack and prevention may vary widely.",,Low to Medium,Securing Device Objects: http://msdn.microsoft.com/en-us/library/ms794722.aspx +The implementations of IOCTLs will differ between operating system types and versions, so the methods of attack and prevention may vary widely.",,medium,Securing Device Objects: http://msdn.microsoft.com/en-us/library/ms794722.aspx CWE-117,EN-Improper Output Neutralization for Logs (Type: Base),"The software does not neutralize or incorrectly neutralizes output that is written to logs. This can allow an attacker to forge log entries or inject malicious content into logs. Log forging vulnerabilities occur when: Data enters an application from an untrusted source. -The data is written to an application or system log file.",,Medium,"Exploiting Software: How to Break Code +The data is written to an application or system log file.",,medium,"Exploiting Software: How to Break Code The night the log was forged: http://doc.novsu.ac.ru/oreilly/tcpip/puis/ch10_05.htm OWASP TOP 10: http://www.owasp.org/index.php/Top_10_2007" CWE-124,EN-Buffer Underwrite (Buffer Underflow) (Type: Base),"The software writes to a buffer using an index or pointer that references a memory location prior to the beginning of the buffer. -This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used.",,Medium,"Buffer UNDERFLOWS: What do you know about it?: http://seclists.org/vuln-dev/2004/Jan/0022.html +This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used.",,medium,"Buffer UNDERFLOWS: What do you know about it?: http://seclists.org/vuln-dev/2004/Jan/0022.html 24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89" CWE-128,EN-Wrap-around Error (Type: Base),"Wrap around errors occur whenever a value is incremented past the maximum value for its type and therefore ""wraps around"" to a very small, negative, or undefined value. -This typically occurs when the pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. This may result in exposure of sensitive information or possibly a crash.",,Medium,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89 +This typically occurs when the pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. This may result in exposure of sensitive information or possibly a crash.",,medium,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89 The Art of Software Security Assessment: Chapter 6, ""Signed Integer Boundaries"", Page 220." CWE-170,EN-Improper Null Termination (Type: Base),"The software does not terminate or incorrectly terminates a string or array with a null character or equivalent terminator. -Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,Medium, +Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,medium, CWE-190,EN-Integer Overflow or Wraparound (Type: Base),"The software performs a calculation that can produce an integer overflow or wraparound, when the logic assumes that the resulting value will always be larger than the original value. This can introduce other weaknesses when the calculation is used for resource management or execution control. -An integer overflow or wraparound occurs when an integer value is incremented to a value that is too large to store in the associated representation. When this occurs, the value may wrap to become a very small or negative number. While this may be intended behavior in circumstances that rely on wrapping, it can have security consequences if the wrap is unexpected. This is especially the case if the integer overflow can be triggered using user-supplied inputs. This becomes security-critical when the result is used to control looping, make a security decision, or determine the offset or size in behaviors such as memory allocation, copying, concatenation, etc.",,Medium,"An overview of common programming security vulnerabilities and possible solutions: http://fort-knox.org/thesis.pdf +An integer overflow or wraparound occurs when an integer value is incremented to a value that is too large to store in the associated representation. When this occurs, the value may wrap to become a very small or negative number. While this may be intended behavior in circumstances that rely on wrapping, it can have security consequences if the wrap is unexpected. This is especially the case if the integer overflow can be triggered using user-supplied inputs. This becomes security-critical when the result is used to control looping, make a security decision, or determine the offset or size in behaviors such as memory allocation, copying, concatenation, etc.",,medium,"An overview of common programming security vulnerabilities and possible solutions: http://fort-knox.org/thesis.pdf Basic Integer Overflows: http://www.phrack.org/issues.html?issue=60&id=10#article Writing Secure Code: Chapter 20, ""Integer Overflows"" Page 620 24 Deadly Sins of Software Security: ""Sin 7: Integer Overflows."" Page 119 @@ -430,38 +430,38 @@ SafeInt: http://safeint.codeplex.com/ Top 25 Series - Rank 17 - Integer Overflow Or Wraparound: http://software-security.sans.org/blog/2010/03/18/top-25-series-rank-17-integer-overflow-or-wraparound The Art of Software Security Assessment: Chapter 6, ""Signed Integer Boundaries"", Page 220." CWE-196,EN-Unsigned to Signed Conversion Error (Type: Variant),"An unsigned-to-signed conversion error takes place when a large unsigned primitive is used as a signed value. -It is dangerous to rely on implicit casts between signed and unsigned numbers because the result can take on an unexpected value and violate assumptions made by the program.",,Medium,"The Art of Software Security Assessment: Chapter 6, ""Type Conversions"", Page 223." +It is dangerous to rely on implicit casts between signed and unsigned numbers because the result can take on an unexpected value and violate assumptions made by the program.",,medium,"The Art of Software Security Assessment: Chapter 6, ""Type Conversions"", Page 223." CWE-202,EN-Exposure of Sensitive Data Through Data Queries (Type: Variant),"When trying to keep information confidential, an attacker can often infer some of the information by using statistics. -In situations where data should not be tied to individual users, but a large number of users should be able to make queries that ""scrub"" the identity of users, it may be possible to get information about a user -- e.g., by specifying search terms that are known to be unique to that user.",,Medium, +In situations where data should not be tied to individual users, but a large number of users should be able to make queries that ""scrub"" the identity of users, it may be possible to get information about a user -- e.g., by specifying search terms that are known to be unique to that user.",,medium, CWE-250,EN-Execution with Unnecessary Privileges (Type: Class),"The software performs an operation at a privilege level that is higher than the minimum level required, which creates new weaknesses or amplifies the consequences of other weaknesses. New weaknesses can be exposed because running with extra privileges, such as root or Administrator, can disable the normal security checks being performed by the operating system or surrounding environment. Other pre-existing weaknesses can turn into security vulnerabilities if they occur while operating at raised privileges. -Privilege management functions can behave in some less-than-obvious ways, and they have different quirks on different platforms. These inconsistencies are particularly pronounced if you are transitioning from one non-root user to another. Signal handlers and spawned processes run at the privilege of the owning process, so if a process is running as root when a signal fires or a sub-process is executed, the signal handler or sub-process will operate with root privileges.",,Medium,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ +Privilege management functions can behave in some less-than-obvious ways, and they have different quirks on different platforms. These inconsistencies are particularly pronounced if you are transitioning from one non-root user to another. Signal handlers and spawned processes run at the privilege of the owning process, so if a process is running as root when a signal fires or a sub-process is executed, the signal handler or sub-process will operate with root privileges.",,medium,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html Writing Secure Code: Chapter 7, ""Running with Least Privilege"" Page 207 Federal Desktop Core Configuration: http://nvd.nist.gov/fdcc/index.cfm 24 Deadly Sins of Software Security: ""Sin 16: Executing Code With Too Much Privilege."" Page 243 The Art of Software Security Assessment: Chapter 9, ""Privilege Vulnerabilities"", Page 477." CWE-269,EN-Improper Privilege Management (Type: Base),"The software does not properly assign, modify, track, or check privileges for an actor, creating an unintended sphere of control for that actor. -Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,Medium,"24 Deadly Sins of Software Security: ""Sin 16: Executing Code With Too Much Privilege."" Page 243 +Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,medium,"24 Deadly Sins of Software Security: ""Sin 16: Executing Code With Too Much Privilege."" Page 243 The Art of Software Security Assessment: Chapter 9, ""Dropping Privileges Permanently"", Page 479." CWE-273,EN-Improper Check for Dropped Privileges (Type: Base),"The software attempts to drop privileges but does not check or incorrectly checks to see if the drop succeeded. -If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,Medium, +If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,medium, CWE-276,EN-Incorrect Default Permissions (Type: Variant),"The software, upon installation, sets incorrect permissions for an object that exposes it to an unintended actor. -If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,Medium,"The Art of Software Security Assessment: Chapter 3, ""Insecure Defaults"", Page 69." +If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,medium,"The Art of Software Security Assessment: Chapter 3, ""Insecure Defaults"", Page 69." CWE-299,EN-Improper Check for Certificate Revocation (Type: Variant),"The software does not check or incorrectly checks the revocation status of a certificate, which may cause it to use a certificate that has been compromised. -An improper check for certificate revocation is a far more serious flaw than related certificate failures. This is because the use of any revoked certificate is almost certainly malicious. The most common reason for certificate revocation is compromise of the system in question, with the result that no legitimate servers will be using a revoked certificate, unless they are sorely out of sync.",,Medium,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347" +An improper check for certificate revocation is a far more serious flaw than related certificate failures. This is because the use of any revoked certificate is almost certainly malicious. The most common reason for certificate revocation is compromise of the system in question, with the result that no legitimate servers will be using a revoked certificate, unless they are sorely out of sync.",,medium,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347" CWE-301,EN-Reflection Attack in an Authentication Protocol (Type: Variant),"Simple authentication protocols are subject to reflection attacks if a malicious user can use the target machine to impersonate a trusted user. -A mutual authentication protocol requires each party to respond to a random challenge by the other party by encrypting it with a pre-shared key. Often, however, such protocols employ the same pre-shared key for communication with a number of different entities. A malicious user or an attacker can easily compromise this protocol without possessing the correct key by employing a reflection attack on the protocol.",,Medium,"The Art of Software Security Assessment: Chapter 2, ""Insufficient Validation"", Page 38." +A mutual authentication protocol requires each party to respond to a random challenge by the other party by encrypting it with a pre-shared key. Often, however, such protocols employ the same pre-shared key for communication with a number of different entities. A malicious user or an attacker can easily compromise this protocol without possessing the correct key by employing a reflection attack on the protocol.",,medium,"The Art of Software Security Assessment: Chapter 2, ""Insufficient Validation"", Page 38." CWE-329,EN-Not Using a Random IV with CBC Mode (Type: Variant),"Not using a random initialization Vector (IV) with Cipher Block Chaining (CBC) Mode causes algorithms to be susceptible to dictionary attacks. -This weakness is especially dangerous when the hash is used in security algorithms that require the one-way property to hold. For example, if an authentication system takes an incoming password and generates a hash, then compares the hash to another hash that it has stored in its authentication database, then the ability to create a collision could allow an attacker to provide an alternate password that produces the same target hash, bypassing authentication.",,Medium,"The Art of Software Security Assessment: Chapter 2, ""Initialization Vectors"", Page 42." +This weakness is especially dangerous when the hash is used in security algorithms that require the one-way property to hold. For example, if an authentication system takes an incoming password and generates a hash, then compares the hash to another hash that it has stored in its authentication database, then the ability to create a collision could allow an attacker to provide an alternate password that produces the same target hash, bypassing authentication.",,medium,"The Art of Software Security Assessment: Chapter 2, ""Initialization Vectors"", Page 42." CWE-332,EN-Insufficient Entropy in PRNG (Type: Variant),"The lack of entropy available for, or used by, a Pseudo-Random Number Generator (PRNG) can be a stability and security threat. -When software generates predictable values in a context requiring unpredictability, it may be possible for an attacker to guess the next value that will be generated, and use this guess to impersonate another user or access sensitive information.",,Medium,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf +When software generates predictable values in a context requiring unpredictability, it may be possible for an attacker to guess the next value that will be generated, and use this guess to impersonate another user or access sensitive information.",,medium,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf CWE-338,EN-Use of Cryptographically Weak PRNG (Type: Base),"The product uses a Pseudo-Random Number Generator (PRNG) in a security context, but the PRNG is not cryptographically strong. -The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,Medium,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" +The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,medium,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-353,EN-Missing Support for Integrity Check (Type: Base),"The software uses a transmission protocol that does not include a mechanism for verifying the integrity of the data during transmission, such as a checksum. -If integrity check values or ""checksums"" are omitted from a protocol, there is no way of determining if data has been corrupted in transmission. The lack of checksum functionality in a protocol removes the first application-level check of data that can be used. The end-to-end philosophy of checks states that integrity checks should be performed at the lowest level that they can be completely implemented. Excluding further sanity checks and input validation performed by applications, the protocol's checksum is the most important level of checksum, since it can be performed more completely than at any previous level and takes into account entire messages, as opposed to single packets.",,Medium,"24 Deadly Sins of Software Security: ""Sin 15: Not Updating Easily."" Page 231" +If integrity check values or ""checksums"" are omitted from a protocol, there is no way of determining if data has been corrupted in transmission. The lack of checksum functionality in a protocol removes the first application-level check of data that can be used. The end-to-end philosophy of checks states that integrity checks should be performed at the lowest level that they can be completely implemented. Excluding further sanity checks and input validation performed by applications, the protocol's checksum is the most important level of checksum, since it can be performed more completely than at any previous level and takes into account entire messages, as opposed to single packets.",,medium,"24 Deadly Sins of Software Security: ""Sin 15: Not Updating Easily."" Page 231" CWE-354,EN-Improper Validation of Integrity Check Value (Type: Base),"The software does not validate or incorrectly validates the integrity check values or ""checksums"" of a message. This may prevent it from detecting if the data has been modified or corrupted in transmission. -Improper validation of checksums before use results in an unnecessary risk that can easily be mitigated. The protocol specification describes the algorithm used for calculating the checksum. It is then a simple matter of implementing the calculation and verifying that the calculated checksum and the received checksum match. Improper verification of the calculated checksum and the received checksum can lead to far greater consequences.",,Medium, +Improper validation of checksums before use results in an unnecessary risk that can easily be mitigated. The protocol specification describes the algorithm used for calculating the checksum. It is then a simple matter of implementing the calculation and verifying that the calculated checksum and the received checksum match. Improper verification of the calculated checksum and the received checksum can lead to far greater consequences.",,medium, CWE-362,EN-Concurrent Execution using Shared Resource with Improper Synchronization (Race Condition) (Type: Class),"The program contains a code sequence that can run concurrently with other code, and the code sequence requires temporary, exclusive access to a shared resource, but a timing window exists in which the shared resource can be modified by another code sequence that is operating concurrently. This can have security implications when the expected synchronization is in security-critical code, such as recording whether a user is authenticated or modifying important state information that should not be influenced by an outsider. A race condition occurs within concurrent environments, and is effectively a property of a code sequence. Depending on the context, a code sequence may be in the form of a function call, a small number of instructions, a series of program invocations, etc. @@ -469,7 +469,7 @@ A race condition violates these properties, which are closely related: Exclusivity - the code sequence is given exclusive access to the shared resource, i.e., no other code sequence can modify properties of the shared resource before the original sequence has completed execution. Atomicity - the code sequence is behaviorally atomic, i.e., no other thread or process can concurrently execute the same sequence of instructions (or a subset) against the same resource. A race condition exists when an ""interfering code sequence"" can still access the shared resource, violating exclusivity. Programmers may assume that certain code sequences execute too quickly to be affected by an interfering code sequence; when they are not, this violates atomicity. For example, the single ""x++"" statement may appear atomic at the code layer, but it is actually non-atomic at the instruction layer, since it involves a read (the original value of x), followed by a computation (x+1), followed by a write (save the result to x). -The interfering code sequence could be ""trusted"" or ""untrusted."" A trusted interfering code sequence occurs within the program; it cannot be modified by the attacker, and it can only be invoked indirectly. An untrusted interfering code sequence can be authored directly by the attacker, and typically it is external to the vulnerable program.",,Medium,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205 +The interfering code sequence could be ""trusted"" or ""untrusted."" A trusted interfering code sequence occurs within the program; it cannot be modified by the attacker, and it can only be invoked indirectly. An untrusted interfering code sequence can be authored directly by the attacker, and typically it is external to the vulnerable program.",,medium,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205 volatile - Multithreaded Programmer's Best Friend: http://www.ddj.com/cpp/184403766 Thread-safe webapps using Spring: http://www.javalobby.org/articles/thread-safe/index.jsp Prevent race conditions: http://www.ibm.com/developerworks/library/l-sprace.html @@ -494,7 +494,7 @@ Signal handler vulnerabilities are often classified based on the absence of a sp Avoiding shared state Using synchronization in the signal handler Using synchronization in the regular code -Disabling or masking other signals, which provides atomicity (which effectively ensures exclusivity)",,Medium,"Delivering Signals for Fun and Profit: http://lcamtuf.coredump.cx/signals.txt +Disabling or masking other signals, which provides atomicity (which effectively ensures exclusivity)",,medium,"Delivering Signals for Fun and Profit: http://lcamtuf.coredump.cx/signals.txt Race Condition: Signal Handling: http://www.fortify.com/vulncat/en/vulncat/cpp/race_condition_signal_handling.html 24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205 The Art of Software Security Assessment: Chapter 13, ""Signal Vulnerabilities"", Page 791." @@ -512,7 +512,7 @@ Signal handler vulnerabilities are often classified based on the absence of a sp Avoiding shared state Using synchronization in the signal handler Using synchronization in the regular code -Disabling or masking other signals, which provides atomicity (which effectively ensures exclusivity)",,Medium,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205" +Disabling or masking other signals, which provides atomicity (which effectively ensures exclusivity)",,medium,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205" CWE-366,EN-Race Condition within a Thread (Type: Base),"If two threads of execution use a resource simultaneously, there exists the possibility that resources may be used while invalid, in turn making the state of execution undefined. Race conditions frequently occur in signal handlers, since signal handlers support asynchronous actions. These race conditions have a variety of root causes and symptoms. Attackers may be able to exploit a signal handler race condition to cause the software state to be corrupted, possibly leading to a denial of service or even code execution. These issues occur when non-reentrant functions, or state-sensitive actions occur in the signal handler, where they may be called at any time. These behaviors can violate assumptions being made by the ""regular"" code that is interrupted, or by other signal handlers that may also be invoked. If these functions are called at an inopportune moment - such as while a non-reentrant function is already running - memory corruption could occur that may be exploitable for code execution. Another signal race condition commonly found occurs when free is called within a signal handler, resulting in a double free and therefore a write-what-where condition. Even if a given pointer is set to NULL after it has been freed, a race condition still exists between the time the memory was freed and the pointer was set to NULL. This is especially problematic if the same signal handler has been set for more than one signal -- since it means that the signal handler itself may be reentered. @@ -527,46 +527,46 @@ Signal handler vulnerabilities are often classified based on the absence of a sp Avoiding shared state Using synchronization in the signal handler Using synchronization in the regular code -Disabling or masking other signals, which provides atomicity (which effectively ensures exclusivity)",,Medium,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205 +Disabling or masking other signals, which provides atomicity (which effectively ensures exclusivity)",,medium,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205 The Art of Software Security Assessment: Chapter 13, ""Race Conditions"", Page 759." CWE-369,EN-Divide By Zero (Type: Base),"The product divides a value by zero. -This weakness typically occurs when an unexpected value is provided to the product, or if an error occurs that is not properly detected. It frequently occurs in calculations involving physical dimensions such as size, length, width, and height.",,Medium,"No description: http://www.cprogramming.com/tutorial/exceptions.html +This weakness typically occurs when an unexpected value is provided to the product, or if an error occurs that is not properly detected. It frequently occurs in calculations involving physical dimensions such as size, length, width, and height.",,medium,"No description: http://www.cprogramming.com/tutorial/exceptions.html No description: http://msdn.microsoft.com/en-us/library/ms173160(VS.80).aspx" CWE-370,EN-Missing Check for Certificate Revocation after Initial Check (Type: Base),"The software does not check the revocation status of a certificate after its initial revocation check, which can cause the software to perform privileged actions even after the certificate is revoked at a later time. -If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,Medium,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205" +If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,medium,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205" CWE-374,EN-Passing Mutable Objects to an Untrusted Method (Type: Base),"Sending non-cloned mutable data as an argument may result in that data being altered or deleted by the called function, thereby putting the calling function into an undefined state. -If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,Medium,"Does Java pass by reference or pass by value?: http://www.javaworld.com/javaworld/javaqa/2000-05/03-qa-0526-pass.html +If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,medium,"Does Java pass by reference or pass by value?: http://www.javaworld.com/javaworld/javaqa/2000-05/03-qa-0526-pass.html Java: The Complete Reference, J2SE 5th Edition" CWE-375,EN-Returning a Mutable Object to an Untrusted Caller (Type: Base),"Sending non-cloned mutable data as a return value may result in that data being altered or deleted by the calling function, thereby putting the class in an undefined state. -If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,Medium, +If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,medium, CWE-385,EN-Covert Timing Channel (Type: Base),"Covert timing channels convey information by modulating some aspect of system behavior over time, so that the program receiving the information can observe system behavior and infer protected information. In some instances, knowing when data is transmitted between parties can provide a malicious user with privileged information. Also, externally monitoring the timing of operations can potentially reveal sensitive data. For example, a cryptographic operation can expose its internal state if the time it takes to perform the operation varies, based on the state. -Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,Medium, +Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,medium, CWE-390,EN-Detection of Error Condition Without Action (Type: Class),"The software detects a specific error, but takes no actions to handle the error. In some instances, knowing when data is transmitted between parties can provide a malicious user with privileged information. Also, externally monitoring the timing of operations can potentially reveal sensitive data. For example, a cryptographic operation can expose its internal state if the time it takes to perform the operation varies, based on the state. -Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,Medium,"24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183" +Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,medium,"24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183" CWE-391,EN-Unchecked Error Condition (Type: Base),"Ignoring exceptions and other error conditions may allow an attacker to induce unexpected behavior unnoticed. In some instances, knowing when data is transmitted between parties can provide a malicious user with privileged information. Also, externally monitoring the timing of operations can potentially reveal sensitive data. For example, a cryptographic operation can expose its internal state if the time it takes to perform the operation varies, based on the state. -Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,Medium, +Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,medium, CWE-401,EN-Improper Release of Memory Before Removing Last Reference (Memory Leak) (Type: Base),"The software does not sufficiently track and release allocated memory after it has been used, which slowly consumes remaining memory. -This is often triggered by improper handling of malformed data or unexpectedly interrupted sessions.",,Medium,How to Break Software Security +This is often triggered by improper handling of malformed data or unexpectedly interrupted sessions.",,medium,How to Break Software Security CWE-460,EN-Improper Cleanup on Thrown Exception (Type: Variant),"The product does not clean up its state or incorrectly cleans up its state when an exception is thrown, leading to unexpected state or control flow. -In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,Medium, +In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,medium, CWE-468,EN-Incorrect Pointer Scaling (Type: Base),"In C and C++, one may often accidentally refer to the wrong memory due to the semantics of when math operations are implicitly scaled. -Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,Medium,"The Art of Software Security Assessment: Chapter 6, ""Pointer Arithmetic"", Page 277." +Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,medium,"The Art of Software Security Assessment: Chapter 6, ""Pointer Arithmetic"", Page 277." CWE-469,EN-Use of Pointer Subtraction to Determine Size (Type: Base),"The application subtracts one pointer from another in order to determine size, but this calculation can be incorrect if the pointers do not exist in the same memory chunk. -Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,Medium, +Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,medium, CWE-476,EN-NULL Pointer Dereference (Type: Base),"A NULL pointer dereference occurs when the application dereferences a pointer that it expects to be valid, but is NULL, typically causing a crash or exit. -NULL pointer dereference issues can occur through a number of flaws, including race conditions, and simple programming omissions.",,Medium, +NULL pointer dereference issues can occur through a number of flaws, including race conditions, and simple programming omissions.",,medium, CWE-484,EN-Omitted Break Statement in Switch (Type: Base),"The program omits a break statement within a switch or similar construct, causing code associated with multiple conditions to execute. This can cause problems when the programmer only intended to execute code associated with one condition. -This can lead to critical code executing in situations where it should not.",,Medium,"The Art of Software Security Assessment: Chapter 7, ""Switch Statements"", Page 337." +This can lead to critical code executing in situations where it should not.",,medium,"The Art of Software Security Assessment: Chapter 7, ""Switch Statements"", Page 337." CWE-487,EN-Reliance on Package-level Scope (Type: Variant),"Java packages are not inherently closed; therefore, relying on them for code security is not a good practice. -If the decision to trust the methods and data of an object is based on the name of a class, it is possible for malicious users to send objects of the same name as trusted classes and thereby gain the trust afforded to known classes and types.",,Medium, +If the decision to trust the methods and data of an object is based on the name of a class, it is possible for malicious users to send objects of the same name as trusted classes and thereby gain the trust afforded to known classes and types.",,medium, CWE-492,EN-Use of Inner Class Containing Sensitive Data (Type: Variant),"Inner classes are translated into classes that are accessible at package scope and may expose code that the programmer intended to keep private to attackers. Data can ""bleed"" from one session to another through member variables of singleton objects, such as Servlets, and objects from a shared pool. -In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,Medium, +In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,medium, CWE-494,EN-Download of Code Without Integrity Check (Type: Base),"The product downloads source code or an executable from a remote location and executes the code without sufficiently verifying the origin and integrity of the code. -An attacker can execute malicious code by compromising the host server, performing DNS spoofing, or modifying the code in transit.",,Medium,"Introduction to Code Signing: http://msdn.microsoft.com/en-us/library/ms537361(VS.85).aspx +An attacker can execute malicious code by compromising the host server, performing DNS spoofing, or modifying the code in transit.",,medium,"Introduction to Code Signing: http://msdn.microsoft.com/en-us/library/ms537361(VS.85).aspx Authenticode: http://msdn.microsoft.com/en-us/library/ms537359(v=VS.85).aspx Code Signing Guide: http://developer.apple.com/documentation/Security/Conceptual/CodeSigningGuide/Introduction/chapter_1_section_1.html Secure Software Updates: Disappointments and New Challenges: http://prisms.cs.umass.edu/~kevinfu/papers/secureupdates-hotsec06.pdf @@ -574,61 +574,61 @@ Secure Software Updates: Disappointments and New Challenges: http://prisms.cs.um Top 25 Series - Rank 20 - Download of Code Without Integrity Check: http://blogs.sans.org/appsecstreetfighter/2010/04/05/top-25-series-rank-20-download-code-integrity-check/ Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html" CWE-498,EN-Cloneable Class Containing Sensitive Information (Type: Variant),"The code contains a class with sensitive data, but the class is cloneable. The data can then be accessed by cloning the class. -Cloneable classes are effectively open classes, since data cannot be hidden in them. Classes that do not explicitly deny cloning can be cloned by any other class without running the constructor.",,Medium, +Cloneable classes are effectively open classes, since data cannot be hidden in them. Classes that do not explicitly deny cloning can be cloned by any other class without running the constructor.",,medium, CWE-502,EN-Deserialization of Untrusted Data (Type: Variant),"The application deserializes untrusted data without sufficiently verifying that the resulting data will be valid. It is often convenient to serialize objects for communication or to save them for later use. However, deserialized data or code can often be modified without using the provided accessor functions if it does not use cryptography to protect itself. Furthermore, any cryptography would still be client-side security -- which is a dangerous security assumption. -Data that is untrusted can not be trusted to be well-formed.",,Medium,"Unserializing user-supplied data, a bad idea: http://heine.familiedeelstra.com/security/unserialize +Data that is untrusted can not be trusted to be well-formed.",,medium,"Unserializing user-supplied data, a bad idea: http://heine.familiedeelstra.com/security/unserialize Why Python Pickle is Insecure: http://nadiana.com/python-pickle-insecure" CWE-532,EN-Information Exposure Through Log Files (Type: Variant),"Information written to log files can be of a sensitive nature and give valuable guidance to an attacker or expose sensitive user information. -While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,Medium, +While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,medium, CWE-602,EN-Client-Side Enforcement of Server-Side Security (Type: Base),"The software is composed of a server that relies on the client to implement a mechanism that is intended to protect the server. -When the server relies on protection mechanisms placed on the client side, an attacker can modify the client-side behavior to bypass the protection mechanisms resulting in potentially unexpected interactions between the client and server. The consequences will vary, depending on what the mechanisms are trying to protect.",,Medium,"Writing Secure Code: Chapter 23, ""Client-Side Security Is an Oxymoron"" Page 687" +When the server relies on protection mechanisms placed on the client side, an attacker can modify the client-side behavior to bypass the protection mechanisms resulting in potentially unexpected interactions between the client and server. The consequences will vary, depending on what the mechanisms are trying to protect.",,medium,"Writing Secure Code: Chapter 23, ""Client-Side Security Is an Oxymoron"" Page 687" CWE-665,EN-Improper Initialization (Type: Base),"The software does not initialize or incorrectly initializes a resource, which might leave the resource in an unexpected state when it is accessed or used. -This can have security implications when the associated resource is expected to have certain properties or values, such as a variable that determines whether a user has been authenticated or not.",,Medium,"Exploiting Uninitialized Data: http://www.felinemenace.org/~mercy/papers/UBehavior/UBehavior.zip +This can have security implications when the associated resource is expected to have certain properties or values, such as a variable that determines whether a user has been authenticated or not.",,medium,"Exploiting Uninitialized Data: http://www.felinemenace.org/~mercy/papers/UBehavior/UBehavior.zip MS08-014 : The Case of the Uninitialized Stack Variable Vulnerability: http://blogs.technet.com/swi/archive/2008/03/11/the-case-of-the-uninitialized-stack-variable-vulnerability.aspx The Art of Software Security Assessment: Chapter 7, ""Variable Initialization"", Page 312." CWE-754,EN-Improper Check for Unusual or Exceptional Conditions (Type: Class),"The software does not check or improperly checks for unusual or exceptional conditions that are not expected to occur frequently during day to day operation of the software. The programmer may assume that certain events or conditions will never occur or do not need to be worried about, such as low memory conditions, lack of access to resources due to restrictive permissions, or misbehaving clients or components. However, attackers may intentionally trigger these unusual conditions, thus violating the programmer's assumptions, possibly introducing instability, incorrect behavior, or a vulnerability. -Note that this entry is not exclusively about the use of exceptions and exception handling, which are mechanisms for both checking and handling unusual or unexpected conditions.",,Medium,"The Art of Software Security Assessment: Chapter 7, ""Program Building Blocks"" Page 341 +Note that this entry is not exclusively about the use of exceptions and exception handling, which are mechanisms for both checking and handling unusual or unexpected conditions.",,medium,"The Art of Software Security Assessment: Chapter 7, ""Program Building Blocks"" Page 341 The Art of Software Security Assessment: Chapter 1, ""Exceptional Conditions,"" Page 22 24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183 Top 25 Series - Rank 15 - Improper Check for Unusual or Exceptional Conditions: http://blogs.sans.org/appsecstreetfighter/2010/03/15/top-25-series-rank-15-improper-check-for-unusual-or-exceptional-conditions/" CWE-778,EN-Insufficient Logging (Type: Base),"When a security-critical event occurs, the software either does not record the event or omits important details about the event when logging it. -When security-critical events are not logged properly, such as a failed login attempt, this can make malicious behavior more difficult to detect and may hinder forensic analysis after an attack succeeds.",,Medium,"The Art of Software Security Assessment: Chapter 2, ""Accountability"", Page 40." +When security-critical events are not logged properly, such as a failed login attempt, this can make malicious behavior more difficult to detect and may hinder forensic analysis after an attack succeeds.",,medium,"The Art of Software Security Assessment: Chapter 2, ""Accountability"", Page 40." CWE-780,EN-Use of RSA Algorithm without OAEP (Type: Variant),"The software uses the RSA algorithm but does not incorporate Optimal Asymmetric Encryption Padding (OAEP), which might weaken the encryption. -Padding schemes are often used with cryptographic algorithms to make the plaintext less predictable and complicate attack efforts. The OAEP scheme is often used with RSA to nullify the impact of predictable common text.",,Medium,"RSA Problem: http://people.csail.mit.edu/rivest/RivestKaliski-RSAProblem.pdf +Padding schemes are often used with cryptographic algorithms to make the plaintext less predictable and complicate attack efforts. The OAEP scheme is often used with RSA to nullify the impact of predictable common text.",,medium,"RSA Problem: http://people.csail.mit.edu/rivest/RivestKaliski-RSAProblem.pdf Optimal Asymmetric Encryption Padding: http://en.wikipedia.org/wiki/Optimal_Asymmetric_Encryption_Padding" CWE-908,EN-Use of Uninitialized Resource (Type: Base),"The software uses a resource that has not been properly initialized. -This can have security implications when the associated resource is expected to have certain properties or values.",,Medium,Exploiting Uninitialized Data: http://www.felinemenace.org/~mercy/papers/UBehavior/UBehavior.zip +This can have security implications when the associated resource is expected to have certain properties or values.",,medium,Exploiting Uninitialized Data: http://www.felinemenace.org/~mercy/papers/UBehavior/UBehavior.zip CWE-909,EN-Missing Initialization of Resource (Type: Base),"The software does not initialize a critical resource. -Many resources require initialization before they can be properly used. If a resource is not initialized, it could contain unpredictable or expired data, or it could be initialized to defaults that are invalid. This can have security implications when the resource is expected to have certain properties or values.",,Medium, +Many resources require initialization before they can be properly used. If a resource is not initialized, it could contain unpredictable or expired data, or it could be initialized to defaults that are invalid. This can have security implications when the resource is expected to have certain properties or values.",,medium, CWE-910,EN-Use of Expired File Descriptor (Type: Base),"The software uses or accesses a file descriptor after it has been closed. -After a file descriptor for a particular file or device has been released, it can be reused. The code might not write to the original file, since the reused file descriptor might reference a different file or device.",,Medium, +After a file descriptor for a particular file or device has been released, it can be reused. The code might not write to the original file, since the reused file descriptor might reference a different file or device.",,medium, CWE-911,EN-Improper Update of Reference Count (Type: Base),"The software uses a reference count to manage a resource, but it does not update or incorrectly updates the reference count. -Reference counts can be used when tracking how many objects contain a reference to a particular resource, such as in memory management or garbage collection. When the reference count reaches zero, the resource can be de-allocated or reused because there are no more objects that use it. If the reference count accidentally reaches zero, then the resource might be released too soon, even though it is still in use. If all objects no longer use the resource, but the reference count is not zero, then the resource might not ever be released.",,Medium,Windows Kernel Reference Count Vulnerabilities - Case Study: http://j00ru.vexillium.org/dump/zn_slides.pdf +Reference counts can be used when tracking how many objects contain a reference to a particular resource, such as in memory management or garbage collection. When the reference count reaches zero, the resource can be de-allocated or reused because there are no more objects that use it. If the reference count accidentally reaches zero, then the resource might be released too soon, even though it is still in use. If all objects no longer use the resource, but the reference count is not zero, then the resource might not ever be released.",,medium,Windows Kernel Reference Count Vulnerabilities - Case Study: http://j00ru.vexillium.org/dump/zn_slides.pdf CWE-94,EN-Improper Control of Generation of Code (Code Injection) (Type: Class),"The software constructs all or part of a code segment using externally-influenced input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could modify the syntax or behavior of the intended code segment. When software allows a user's input to contain code syntax, it might be possible for an attacker to craft the code in such a way that it will alter the intended control flow of the software. Such an alteration could lead to arbitrary code execution. -Injection problems encompass a wide variety of issues -- all mitigated in very different ways. For this reason, the most effective way to discuss these weaknesses is to note the distinct features which classify them as injection weaknesses. The most important issue to note is that all injection problems share one thing in common -- i.e., they allow for the injection of control plane data into the user-controlled data plane. This means that the execution of the process may be altered by sending code in through legitimate data channels, using no other mechanism. While buffer overflows, and many other flaws, involve the use of some further issue to gain execution, injection problems need only for the data to be parsed. The most classic instantiations of this category of weakness are SQL injection and format string vulnerabilities.",,Medium,"24 Deadly Sins of Software Security: ""Sin 3: Web-Client Related Vulnerabilities (XSS)."" Page 63" +Injection problems encompass a wide variety of issues -- all mitigated in very different ways. For this reason, the most effective way to discuss these weaknesses is to note the distinct features which classify them as injection weaknesses. The most important issue to note is that all injection problems share one thing in common -- i.e., they allow for the injection of control plane data into the user-controlled data plane. This means that the execution of the process may be altered by sending code in through legitimate data channels, using no other mechanism. While buffer overflows, and many other flaws, involve the use of some further issue to gain execution, injection problems need only for the data to be parsed. The most classic instantiations of this category of weakness are SQL injection and format string vulnerabilities.",,medium,"24 Deadly Sins of Software Security: ""Sin 3: Web-Client Related Vulnerabilities (XSS)."" Page 63" CWE-95,EN-Improper Neutralization of Directives in Dynamically Evaluated Code (Eval Injection) (Type: Base),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes code syntax before using the input in a dynamic evaluation call (e.g. ""eval""). -This may allow an attacker to execute arbitrary code, or at least modify what code can be executed.",,Medium,"No description: http://www.rubycentral.com/book/taint.html +This may allow an attacker to execute arbitrary code, or at least modify what code can be executed.",,medium,"No description: http://www.rubycentral.com/book/taint.html The Art of Software Security Assessment: Chapter 18, ""Inline Evaluation"", Page 1095." CWE-287,EN-Improper Authentication (Type: Class),"When an actor claims to have a given identity, the software does not prove or insufficiently proves that the claim is correct. -Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,Medium to High,"Weak Password Brings 'Happiness' to Twitter Hacker: http://www.wired.com/threatlevel/2009/01/professed-twitt/ +Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,high,"Weak Password Brings 'Happiness' to Twitter Hacker: http://www.wired.com/threatlevel/2009/01/professed-twitt/ Top 10 2007-Broken Authentication and Session Management: http://www.owasp.org/index.php/Top_10_2007-A7 Guide to Authentication: http://www.owasp.org/index.php/Guide_to_Authentication Authentication: http://msdn.microsoft.com/en-us/library/aa374735(VS.85).aspx Writing Secure Code: Chapter 4, ""Authentication"" Page 109" CWE-306,EN-Missing Authentication for Critical Function (Type: Variant),"The software does not perform any authentication for functionality that requires a provable user identity or consumes a significant amount of resources. -Authentication techniques should follow the algorithms that define them exactly, otherwise authentication can be bypassed or more easily subjected to brute force attacks.",,Medium to High,"The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Authentication,"" Page 36 +Authentication techniques should follow the algorithms that define them exactly, otherwise authentication can be bypassed or more easily subjected to brute force attacks.",,high,"The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Authentication,"" Page 36 Top 25 Series - Rank 19 - Missing Authentication for Critical Function: http://blogs.sans.org/appsecstreetfighter/2010/02/23/top-25-series-rank-19-missing-authentication-for-critical-function/ OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI" CWE-319,EN-Cleartext Transmission of Sensitive Information (Type: Base),"The software transmits sensitive or security-critical data in cleartext in a communication channel that can be sniffed by unauthorized actors. -Many communication channels can be ""sniffed"" by attackers during data transmission. For example, network traffic can often be sniffed by any attacker who has access to a network interface. This significantly lowers the difficulty of exploitation by attackers.",,Medium to High,"Top 10 2007-Insecure Communications: http://www.owasp.org/index.php/Top_10_2007-A9 +Many communication channels can be ""sniffed"" by attackers during data transmission. For example, network traffic can often be sniffed by any attacker who has access to a network interface. This significantly lowers the difficulty of exploitation by attackers.",,high,"Top 10 2007-Insecure Communications: http://www.owasp.org/index.php/Top_10_2007-A9 Writing Secure Code: Chapter 9, ""Protecting Secret Data"" Page 299 24 Deadly Sins of Software Security: ""Sin 22: Failing to Protect Network Traffic."" Page 337 Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/" CWE-327,EN-Use of a Broken or Risky Cryptographic Algorithm (Type: Base),"The use of a broken or risky cryptographic algorithm is an unnecessary risk that may result in the exposure of sensitive information. -The use of a non-standard algorithm is dangerous because a determined attacker may be able to break the algorithm and compromise whatever data has been protected. Well-known techniques may exist to break the algorithm.",,Medium to High,"Applied Cryptography: http://www.schneier.com/book-applied.html +The use of a non-standard algorithm is dangerous because a determined attacker may be able to break the algorithm and compromise whatever data has been protected. Well-known techniques may exist to break the algorithm.",,high,"Applied Cryptography: http://www.schneier.com/book-applied.html Handbook of Applied Cryptography: http://www.cacr.math.uwaterloo.ca/hac/ Avoiding bogus encryption products: Snake Oil FAQ: http://www.faqs.org/faqs/cryptography-faq/snake-oil/ SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf @@ -638,7 +638,7 @@ Writing Secure Code: Chapter 8, ""Cryptographic Foibles"" Page 259 Top 25 Series - Rank 24 - Use of a Broken or Risky Cryptographic Algorithm: http://blogs.sans.org/appsecstreetfighter/2010/03/25/top-25-series-rank-24-use-of-a-broken-or-risky-cryptographic-algorithm/ The Art of Software Security Assessment: Chapter 2, ""Insufficient or Obsolete Encryption"", Page 44." CWE-330,EN-Use of Insufficiently Random Values (Type: Class),"The software may use insufficiently random numbers or values in a security context that depends on unpredictable numbers. -When software generates predictable values in a context requiring unpredictability, it may be possible for an attacker to guess the next value that will be generated, and use this guess to impersonate another user or access sensitive information.",,Medium to High,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf +When software generates predictable values in a context requiring unpredictability, it may be possible for an attacker to guess the next value that will be generated, and use this guess to impersonate another user or access sensitive information.",,high,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf Building Secure Software: How to Avoid Security Problems the Right Way Writing Secure Code: Chapter 8, ""Using Poor Random Numbers"" Page 259 24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" @@ -646,22 +646,22 @@ CWE-400,EN-Uncontrolled Resource Consumption (Resource Exhaustion) (Type: Base), Limited resources include memory, file system storage, database connection pool entries, or CPU. If an attacker can trigger the allocation of these limited resources, but the number or size of the resources is not controlled, then the attacker could cause a denial of service that consumes all available resources. This would prevent valid users from accessing the software, and it could potentially have an impact on the surrounding environment. For example, a memory exhaustion attack against an application could slow down the application as well as its host operating system. Resource exhaustion problems have at least two common causes: Error conditions and other exceptional circumstances -Confusion over which part of the program is responsible for releasing the resource",,Medium to High,"Detection and Prediction of Resource-Exhaustion Vulnerabilities: http://homepages.di.fc.ul.pt/~nuno/PAPERS/ISSRE08.pdf +Confusion over which part of the program is responsible for releasing the resource",,high,"Detection and Prediction of Resource-Exhaustion Vulnerabilities: http://homepages.di.fc.ul.pt/~nuno/PAPERS/ISSRE08.pdf Resource exhaustion: http://cr.yp.to/docs/resources.html Resource exhaustion: http://homes.cerias.purdue.edu/~pmeunier/secprog/sanitized/class1/6.resource%20exhaustion.ppt Writing Secure Code: Chapter 17, ""Protecting Against Denial of Service Attacks"" Page 517" CWE-434,EN-Unrestricted Upload of File with Dangerous Type (Type: Base),"The software allows the attacker to upload or transfer files of dangerous types that can be automatically processed within the product's environment. -If code is stored in a file with an extension such as "".inc"" or "".pl"", and the web server does not have a handler for that extension, then the server will likely send the contents of the file directly to the requester without the pre-processing that was expected. When that file contains sensitive information such as database credentials, this may allow the attacker to compromise the application or associated components.",,Medium to High,"Dynamic File Uploads, Security and You: http://shsc.info/FileUploadSecurity +If code is stored in a file with an extension such as "".inc"" or "".pl"", and the web server does not have a handler for that extension, then the server will likely send the contents of the file directly to the requester without the pre-processing that was expected. When that file contains sensitive information such as database credentials, this may allow the attacker to compromise the application or associated components.",,high,"Dynamic File Uploads, Security and You: http://shsc.info/FileUploadSecurity 8 Basic Rules to Implement Secure File Uploads: http://blogs.sans.org/appsecstreetfighter/2009/12/28/8-basic-rules-to-implement-secure-file-uploads/ Top 25 Series - Rank 8 - Unrestricted Upload of Dangerous File Type: http://blogs.sans.org/appsecstreetfighter/2010/02/25/top-25-series-rank-8-unrestricted-upload-of-dangerous-file-type/ Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html The Art of Software Security Assessment: Chapter 17, ""File Uploading"", Page 1068." CWE-64,EN-Windows Shortcut Following (.LNK) (Type: Variant),"The software, when opening a file or directory, does not sufficiently handle when the file is a Windows shortcut (.LNK) whose target is outside of the intended control sphere. This could allow an attacker to cause the software to operate on unauthorized files. -The shortcut (file with the .lnk extension) can permit an attacker to read/write a file that they originally did not have permissions to access.",,Medium to High, +The shortcut (file with the .lnk extension) can permit an attacker to read/write a file that they originally did not have permissions to access.",,high, CWE-681,EN-Incorrect Conversion between Numeric Types (Type: Base),"When converting from one data type to another, such as long to integer, data can be omitted or translated in a way that produces unexpected values. If the resulting values are used in a sensitive context, then dangerous behaviors may occur. -Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,Medium to High,"The Art of Software Security Assessment: Chapter 6, ""Type Conversions"", Page 223." +Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,high,"The Art of Software Security Assessment: Chapter 6, ""Type Conversions"", Page 223." CWE-732,EN-Incorrect Permission Assignment for Critical Resource (Type: Class),"The software specifies permissions for a security-critical resource in a way that allows that resource to be read or modified by unintended actors. -When a resource is given a permissions setting that provides access to a wider range of actors than required, it could lead to the exposure of sensitive information, or the modification of that resource by unintended parties. This is especially dangerous when the resource is related to program configuration, execution or sensitive user data.",,Medium to High,"The Art of Software Security Assessment: Chapter 9, ""File Permissions."" Page 495. +When a resource is given a permissions setting that provides access to a wider range of actors than required, it could lead to the exposure of sensitive information, or the modification of that resource by unintended parties. This is especially dangerous when the resource is related to program configuration, execution or sensitive user data.",,high,"The Art of Software Security Assessment: Chapter 9, ""File Permissions."" Page 495. Building Secure Software: How to Avoid Security Problems the Right Way: Chapter 8, ""Access Control."" Page 194. Top 25 Series - Rank 21 - Incorrect Permission Assignment for Critical Response: http://software-security.sans.org/blog/2010/03/24/top-25-series-rank-21-incorrect-permission-assignment-for-critical-response Federal Desktop Core Configuration: http://nvd.nist.gov/fdcc/index.cfm" @@ -669,7 +669,7 @@ CWE-770,EN-Allocation of Resources Without Limits or Throttling (Type: Base),"Th Command injection vulnerabilities typically occur when: 1. Data enters the application from an untrusted source. 2. The data is part of a string that is executed as a command by the application. -3. By executing the command, the application gives an attacker a privilege or capability that the attacker would not otherwise have.",,Medium to High,"Real-Life Example of a 'Business Logic Defect' (Screen Shots!): http://h30501.www3.hp.com/t5/Following-the-White-Rabbit-A/Real-Life-Example-of-a-Business-Logic-Defect-Screen-Shots/ba-p/22581 +3. By executing the command, the application gives an attacker a privilege or capability that the attacker would not otherwise have.",,high,"Real-Life Example of a 'Business Logic Defect' (Screen Shots!): http://h30501.www3.hp.com/t5/Following-the-White-Rabbit-A/Real-Life-Example-of-a-Business-Logic-Defect-Screen-Shots/ba-p/22581 Detection and Prediction of Resource-Exhaustion Vulnerabilities: http://homepages.di.fc.ul.pt/~nuno/PAPERS/ISSRE08.pdf Resource exhaustion: http://cr.yp.to/docs/resources.html Resource exhaustion: http://homes.cerias.purdue.edu/~pmeunier/secprog/sanitized/class1/6.resource%20exhaustion.ppt @@ -677,15 +677,15 @@ Writing Secure Code: Chapter 17, ""Protecting Against Denial of Service Attacks" Top 25 Series - Rank 22 - Allocation of Resources Without Limits or Throttling: http://blogs.sans.org/appsecstreetfighter/2010/03/23/top-25-series-rank-22-allocation-of-resources-without-limits-or-throttling/ The Art of Software Security Assessment: Chapter 10, ""Resource Limits"", Page 574." CWE-771,EN-Missing Reference to Active Allocated Resource (Type: Base),"The software does not properly maintain a reference to a resource that has been allocated, which prevents the resource from being reclaimed. -This does not necessarily apply in languages or frameworks that automatically perform garbage collection, since the removal of all references may act as a signal that the resource is ready to be reclaimed.",,Medium to High, +This does not necessarily apply in languages or frameworks that automatically perform garbage collection, since the removal of all references may act as a signal that the resource is ready to be reclaimed.",,high, CWE-772,EN-Missing Release of Resource after Effective Lifetime (Type: Base),"The software does not release a resource after its effective lifetime has ended, i.e., after the resource is no longer needed. -When a resource is not released after use, it can allow attackers to cause a denial of service.",,Medium to High, +When a resource is not released after use, it can allow attackers to cause a denial of service.",,high, CWE-773,EN-Missing Reference to Active File Descriptor or Handle (Type: Variant),"The software does not properly maintain references to a file descriptor or handle, which prevents that file descriptor/handle from being reclaimed. -This can cause the software to consume all available file descriptors or handles, which can prevent other processes from performing critical file processing operations.",,Medium to High, +This can cause the software to consume all available file descriptors or handles, which can prevent other processes from performing critical file processing operations.",,high, CWE-774,EN-Allocation of File Descriptors or Handles Without Limits or Throttling (Type: Variant),"The software allocates file descriptors or handles on behalf of an actor without imposing any restrictions on how many descriptors can be allocated, in violation of the intended security policy for that actor. -This can cause the software to consume all available file descriptors or handles, which can prevent other processes from performing critical file processing operations.",,Medium to High,"The Art of Software Security Assessment: Chapter 10, ""Resource Limits"", Page 574." +This can cause the software to consume all available file descriptors or handles, which can prevent other processes from performing critical file processing operations.",,high,"The Art of Software Security Assessment: Chapter 10, ""Resource Limits"", Page 574." CWE-775,EN-Missing Release of File Descriptor or Handle after Effective Lifetime (Type: Variant),"The software does not release a file descriptor or handle after its effective lifetime has ended, i.e., after the file descriptor/handle is no longer needed. -When a file descriptor or handle is not released after use (typically by explicitly closing it), attackers can cause a denial of service by consuming all available file descriptors/handles, or otherwise preventing other system processes from obtaining their own file descriptors/handles.",,Medium to High,"The Art of Software Security Assessment: Chapter 10, ""File Descriptor Leaks"", Page 582." +When a file descriptor or handle is not released after use (typically by explicitly closing it), attackers can cause a denial of service by consuming all available file descriptors/handles, or otherwise preventing other system processes from obtaining their own file descriptors/handles.",,high,"The Art of Software Security Assessment: Chapter 10, ""File Descriptor Leaks"", Page 582." CWE-804,EN-Guessable CAPTCHA (Type: Base),"The software uses a CAPTCHA challenge, but the challenge can be guessed or automatically recognized by a non-human actor. An automated attacker could bypass the intended protection of the CAPTCHA challenge and perform actions at a higher frequency than humanly possible, such as launching spam attacks. There can be several different causes of a guessable CAPTCHA: @@ -693,9 +693,9 @@ An audio or visual image that does not have sufficient distortion from the unobf A question is generated that with a format that can be automatically recognized, such as a math question. A question for which the number of possible answers is limited, such as birth years or favorite sports teams. A general-knowledge or trivia question for which the answer can be accessed using a data base, such as country capitals or popular actors. -Other data associated with the CAPTCHA may provide hints about its contents, such as an image whose filename contains the word that is used in the CAPTCHA.",,Medium to High,Insufficient Anti-automation: http://projects.webappsec.org/Insufficient+Anti-automation +Other data associated with the CAPTCHA may provide hints about its contents, such as an image whose filename contains the word that is used in the CAPTCHA.",,high,Insufficient Anti-automation: http://projects.webappsec.org/Insufficient+Anti-automation CWE-805,EN-Buffer Access with Incorrect Length Value (Type: Base),"The software uses a sequential operation to read or write a buffer, but it uses an incorrect length value that causes it to access memory that is outside of the bounds of the buffer. -When the length value exceeds the size of the destination, a buffer overflow could occur.",,Medium to High,"Writing Secure Code: Chapter 6, ""Why ACLs Are Important"" Page 171 +When the length value exceeds the size of the destination, a buffer overflow could occur.",,high,"Writing Secure Code: Chapter 6, ""Why ACLs Are Important"" Page 171 Address Space Layout Randomization in Windows Vista: http://blogs.msdn.com/michael_howard/archive/2006/05/26/address-space-layout-randomization-in-windows-vista.aspx Limiting buffer overflows with ExecShield: http://www.redhat.com/magazine/009jul05/features/execshield/ PaX: http://en.wikipedia.org/wiki/PaX @@ -705,7 +705,7 @@ Using the Strsafe.h Functions: http://msdn.microsoft.com/en-us/library/ms647466. Understanding DEP as a mitigation technology part 1: http://blogs.technet.com/b/srd/archive/2009/06/12/understanding-dep-as-a-mitigation-technology-part-1.aspx Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html" CWE-806,EN-Buffer Access Using Size of Source Buffer (Type: Variant),"The software uses the size of a source buffer when reading from or writing to a destination buffer, which may cause it to access memory that is outside of the bounds of the buffer. -When the size of the destination is smaller than the size of the source, a buffer overflow could occur.",,Medium to High,"Using the Strsafe.h Functions: http://msdn.microsoft.com/en-us/library/ms647466.aspx +When the size of the destination is smaller than the size of the source, a buffer overflow could occur.",,high,"Using the Strsafe.h Functions: http://msdn.microsoft.com/en-us/library/ms647466.aspx Safe C String Library v1.0.3: http://www.zork.org/safestr/ Address Space Layout Randomization in Windows Vista: http://blogs.msdn.com/michael_howard/archive/2006/05/26/address-space-layout-randomization-in-windows-vista.aspx Limiting buffer overflows with ExecShield: http://www.redhat.com/magazine/009jul05/features/execshield/ @@ -713,79 +713,79 @@ PaX: http://en.wikipedia.org/wiki/PaX Understanding DEP as a mitigation technology part 1: http://blogs.technet.com/b/srd/archive/2009/06/12/understanding-dep-as-a-mitigation-technology-part-1.aspx" CWE-807,EN-Reliance on Untrusted Inputs in a Security Decision (Type: Base),"The application uses a protection mechanism that relies on the existence or values of an input, but the input can be modified by an untrusted actor in a way that bypasses the protection mechanism. Developers may assume that inputs such as cookies, environment variables, and hidden form fields cannot be modified. However, an attacker could change these inputs using customized clients or other attacks. This change might not be detected. When security decisions such as authentication and authorization are made based on the values of these inputs, attackers can bypass the security of the software. -Without sufficient encryption, integrity checking, or other mechanism, any input that originates from an outsider cannot be trusted.",,Medium to High,"Top 25 Series - Rank 6 - Reliance on Untrusted Inputs in a Security Decision: http://blogs.sans.org/appsecstreetfighter/2010/03/05/top-25-series-rank-6-reliance-on-untrusted-inputs-in-a-security-decision/ +Without sufficient encryption, integrity checking, or other mechanism, any input that originates from an outsider cannot be trusted.",,high,"Top 25 Series - Rank 6 - Reliance on Untrusted Inputs in a Security Decision: http://blogs.sans.org/appsecstreetfighter/2010/03/05/top-25-series-rank-6-reliance-on-untrusted-inputs-in-a-security-decision/ HMAC: http://en.wikipedia.org/wiki/Hmac Understanding ASP.NET View State: http://msdn.microsoft.com/en-us/library/ms972976.aspx OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI" CWE-93,EN-Improper Neutralization of CRLF Sequences (CRLF Injection) (Type: Base),"The software uses CRLF (carriage return line feeds) as a special element, e.g. to separate lines or records, but it does not neutralize or incorrectly neutralizes CRLF sequences from inputs. -Since an implicit intent does not specify a particular application to receive the data, any application can process the intent by using an Intent Filter for that intent. This can allow untrusted applications to obtain sensitive data.",,Medium to High,CRLF Injection: http://marc.info/?l=bugtraq&m=102088154213630&w=2 +Since an implicit intent does not specify a particular application to receive the data, any application can process the intent by using an Intent Filter for that intent. This can allow untrusted applications to obtain sensitive data.",,high,CRLF Injection: http://marc.info/?l=bugtraq&m=102088154213630&w=2 CWE-102,EN-Struts: Duplicate Validation Forms (Type: Variant),"The application uses multiple validation forms with the same name, which might cause the Struts Validator to validate a form that the programmer does not expect. -If two validation forms have the same name, the Struts Validator arbitrarily chooses one of the forms to use for input validation and discards the other. This decision might not correspond to the programmer's expectations, possibly leading to resultant weaknesses. Moreover, it indicates that the validation logic is not up-to-date, and can indicate that other, more subtle validation errors are present.",,Unknown, +If two validation forms have the same name, the Struts Validator arbitrarily chooses one of the forms to use for input validation and discards the other. This decision might not correspond to the programmer's expectations, possibly leading to resultant weaknesses. Moreover, it indicates that the validation logic is not up-to-date, and can indicate that other, more subtle validation errors are present.",,unclassified, CWE-103,EN-Struts: Incomplete validate() Method Definition (Type: Variant),"The application has a validator form that either does not define a validate() method, or defines a validate() method but does not call super.validate(). -If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,Unknown, +If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,unclassified, CWE-104,EN-Struts: Form Bean Does Not Extend Validation Class (Type: Variant),"If a form bean does not extend an ActionForm subclass of the Validator framework, it can expose the application to other weaknesses related to insufficient input validation. -If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,Unknown, +If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,unclassified, CWE-105,EN-Struts: Form Field Without Validator (Type: Variant),"The application has a form field that is not validated by a corresponding validation form, which can introduce other weaknesses related to insufficient input validation. -If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,Unknown, +If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,unclassified, CWE-106,EN-Struts: Plug-in Framework not in Use (Type: Variant),"When an application does not use an input validation framework such as the Struts Validator, there is a greater risk of introducing weaknesses related to insufficient input validation. -If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,Unknown, +If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,unclassified, CWE-107,EN-Struts: Unused Validation Form (Type: Variant),"An unused validation form indicates that validation logic is not up-to-date. -It is easy for developers to forget to update validation logic when they remove or rename action form mappings. One indication that validation logic is not being properly maintained is the presence of an unused validation form.",,Unknown, +It is easy for developers to forget to update validation logic when they remove or rename action form mappings. One indication that validation logic is not being properly maintained is the presence of an unused validation form.",,unclassified, CWE-108,EN-Struts: Unvalidated Action Form (Type: Variant),"Every Action Form must have a corresponding validation form. -If a Struts Action Form Mapping specifies a form, it must have a validation form defined under the Struts Validator.",,Unknown, +If a Struts Action Form Mapping specifies a form, it must have a validation form defined under the Struts Validator.",,unclassified, CWE-109,EN-Struts: Validator Turned Off (Type: Variant),"Automatic filtering via a Struts bean has been turned off, which disables the Struts Validator and custom validation logic. This exposes the application to other weaknesses related to insufficient input validation. -If a Struts Action Form Mapping specifies a form, it must have a validation form defined under the Struts Validator.",,Unknown, +If a Struts Action Form Mapping specifies a form, it must have a validation form defined under the Struts Validator.",,unclassified, CWE-11,EN-ASP.NET Misconfiguration: Creating Debug Binary (Type: Variant),"Debugging messages help attackers learn about the system and plan a form of attack. -ASP .NET applications can be configured to produce debug binaries. These binaries give detailed debugging messages and should not be used in production environments. Debug binaries are meant to be used in a development or testing environment and can pose a security risk if they are deployed to production.",,Unknown, +ASP .NET applications can be configured to produce debug binaries. These binaries give detailed debugging messages and should not be used in production environments. Debug binaries are meant to be used in a development or testing environment and can pose a security risk if they are deployed to production.",,unclassified, CWE-110,EN-Struts: Validator Without Form Field (Type: Variant),"Validation fields that do not appear in forms they are associated with indicate that the validation logic is out of date. -It is easy for developers to forget to update validation logic when they make changes to an ActionForm class. One indication that validation logic is not being properly maintained is inconsistencies between the action form and the validation form.",,Unknown, +It is easy for developers to forget to update validation logic when they make changes to an ActionForm class. One indication that validation logic is not being properly maintained is inconsistencies between the action form and the validation form.",,unclassified, CWE-111,EN-Direct Use of Unsafe JNI (Type: Base),"When a Java application uses the Java Native Interface (JNI) to call code written in another programming language, it can expose the application to weaknesses in that code, even if those weaknesses cannot occur in Java. -Many safety features that programmers may take for granted simply do not apply for native code, so you must carefully review all such code for potential problems. The languages used to implement native code may be more susceptible to buffer overflows and other attacks. Native code is unprotected by the security features enforced by the runtime environment, such as strong typing and array bounds checking.",,Unknown,"Fortify Descriptions: http://vulncat.fortifysoftware.com +Many safety features that programmers may take for granted simply do not apply for native code, so you must carefully review all such code for potential problems. The languages used to implement native code may be more susceptible to buffer overflows and other attacks. Native code is unprotected by the security features enforced by the runtime environment, such as strong typing and array bounds checking.",,unclassified,"Fortify Descriptions: http://vulncat.fortifysoftware.com The Java(TM) Tutorial: The Java Native Interface: http://java.sun.com/docs/books/tutorial/native1.1/" CWE-112,EN-Missing XML Validation (Type: Base),"The software accepts XML from an untrusted source but does not validate the XML against the proper schema. -Most successful attacks begin with a violation of the programmer's assumptions. By accepting an XML document without validating it against a DTD or XML schema, the programmer leaves a door open for attackers to provide unexpected, unreasonable, or malicious input.",,Unknown, +Most successful attacks begin with a violation of the programmer's assumptions. By accepting an XML document without validating it against a DTD or XML schema, the programmer leaves a door open for attackers to provide unexpected, unreasonable, or malicious input.",,unclassified, CWE-113,EN-Improper Neutralization of CRLF Sequences in HTTP Headers (HTTP Response Splitting) (Type: Base),"The software receives data from an upstream component, but does not neutralize or incorrectly neutralizes CR and LF characters before the data is included in outgoing HTTP headers. Including unvalidated data in an HTTP header allows an attacker to specify the entirety of the HTTP response rendered by the browser. When an HTTP request contains unexpected CR (carriage return, also given by %0d or \r) and LF (line feed, also given by %0a or \n) characters the server may respond with an output stream that is interpreted as two different HTTP responses (instead of one). An attacker can control the second response and mount attacks such as cross-site scripting and cache poisoning attacks. HTTP response splitting weaknesses may be present when: Data enters a web application through an untrusted source, most frequently an HTTP request. -The data is included in an HTTP response header sent to a web user without being validated for malicious characters.",,Unknown,"OWASP TOP 10: http://www.owasp.org/index.php/Top_10_2007 +The data is included in an HTTP response header sent to a web user without being validated for malicious characters.",,unclassified,"OWASP TOP 10: http://www.owasp.org/index.php/Top_10_2007 24 Deadly Sins of Software Security: ""Sin 2: Web-Server Related Vulnerabilities (XSS, XSRF, and Response Splitting)."" Page 31" CWE-114,EN-Process Control (Type: Base),"Executing commands or loading libraries from an untrusted source or in an untrusted environment can cause an application to execute malicious commands (and payloads) on behalf of an attacker. -Process control vulnerabilities take two forms: 1. An attacker can change the command that the program executes: the attacker explicitly controls what the command is. 2. An attacker can change the environment in which the command executes: the attacker implicitly controls what the command means. Process control vulnerabilities of the first type occur when either data enters the application from an untrusted source and the data is used as part of a string representing a command that is executed by the application. By executing the command, the application gives an attacker a privilege or capability that the attacker would not otherwise have.",,Unknown, +Process control vulnerabilities take two forms: 1. An attacker can change the command that the program executes: the attacker explicitly controls what the command is. 2. An attacker can change the environment in which the command executes: the attacker implicitly controls what the command means. Process control vulnerabilities of the first type occur when either data enters the application from an untrusted source and the data is used as part of a string representing a command that is executed by the application. By executing the command, the application gives an attacker a privilege or capability that the attacker would not otherwise have.",,unclassified, CWE-115,EN-Misinterpretation of Input (Type: Base),"The software misinterprets an input, whether from an attacker or another product, in a security-relevant fashion. -Process control vulnerabilities take two forms: 1. An attacker can change the command that the program executes: the attacker explicitly controls what the command is. 2. An attacker can change the environment in which the command executes: the attacker implicitly controls what the command means. Process control vulnerabilities of the first type occur when either data enters the application from an untrusted source and the data is used as part of a string representing a command that is executed by the application. By executing the command, the application gives an attacker a privilege or capability that the attacker would not otherwise have.",,Unknown, +Process control vulnerabilities take two forms: 1. An attacker can change the command that the program executes: the attacker explicitly controls what the command is. 2. An attacker can change the environment in which the command executes: the attacker implicitly controls what the command means. Process control vulnerabilities of the first type occur when either data enters the application from an untrusted source and the data is used as part of a string representing a command that is executed by the application. By executing the command, the application gives an attacker a privilege or capability that the attacker would not otherwise have.",,unclassified, CWE-118,EN-Improper Access of Indexable Resource (Range Error) (Type: Class),"The software does not restrict or incorrectly restricts operations within the boundaries of a resource that is accessed using an index or pointer, such as memory or files. This can allow an attacker to forge log entries or inject malicious content into logs. Log forging vulnerabilities occur when: Data enters an application from an untrusted source. -The data is written to an application or system log file.",,Unknown, +The data is written to an application or system log file.",,unclassified, CWE-12,EN-ASP.NET Misconfiguration: Missing Custom Error Page (Type: Variant),"An ASP .NET application must enable custom error pages in order to prevent attackers from mining information from the framework's built-in responses. Certain languages allow direct addressing of memory locations and do not automatically ensure that these locations are valid for the memory buffer that is being referenced. This can cause read or write operations to be performed on memory locations that may be associated with other variables, data structures, or internal program data. -As a result, an attacker may be able to execute arbitrary code, alter the intended control flow, read sensitive information, or cause the system to crash.",,Unknown,"19 Deadly Sins of Software Security +As a result, an attacker may be able to execute arbitrary code, alter the intended control flow, read sensitive information, or cause the system to crash.",,unclassified,"19 Deadly Sins of Software Security ASP.NET Misconfiguration: Missing Custom Error Handling: http://www.owasp.org/index.php/ASP.NET_Misconfiguration:_Missing_Custom_Error_Handling" CWE-125,EN-Out-of-bounds Read (Type: Base),"The software reads data past the end, or before the beginning, of the intended buffer. -This typically occurs when the pointer or its index is incremented or decremented to a position beyond the bounds of the buffer or when pointer arithmetic results in a position outside of the valid memory location to name a few. This may result in corruption of sensitive information, a crash, or code execution among other things.",,Unknown,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89" +This typically occurs when the pointer or its index is incremented or decremented to a position beyond the bounds of the buffer or when pointer arithmetic results in a position outside of the valid memory location to name a few. This may result in corruption of sensitive information, a crash, or code execution among other things.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89" CWE-126,EN-Buffer Over-read (Type: Variant),"The software reads from a buffer using buffer access mechanisms such as indexes or pointers that reference memory locations after the targeted buffer. -This typically occurs when the pointer or its index is incremented to a position beyond the bounds of the buffer or when pointer arithmetic results in a position outside of the valid memory location to name a few. This may result in exposure of sensitive information or possibly a crash.",,Unknown, +This typically occurs when the pointer or its index is incremented to a position beyond the bounds of the buffer or when pointer arithmetic results in a position outside of the valid memory location to name a few. This may result in exposure of sensitive information or possibly a crash.",,unclassified, CWE-127,EN-Buffer Under-read (Type: Variant),"The software reads from a buffer using buffer access mechanisms such as indexes or pointers that reference memory locations prior to the targeted buffer. -This typically occurs when the pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. This may result in exposure of sensitive information or possibly a crash.",,Unknown, +This typically occurs when the pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. This may result in exposure of sensitive information or possibly a crash.",,unclassified, CWE-13,EN-ASP.NET Misconfiguration: Password in Configuration File (Type: Variant),"Storing a plaintext password in a configuration file allows anyone who can read the file access to the password-protected resource making them an easy target for attackers. -This typically occurs when the pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. This may result in exposure of sensitive information or possibly a crash.",,Unknown,"How To: Encrypt Configuration Sections in ASP.NET 2.0 Using DPAPI: http://msdn.microsoft.com/en-us/library/ms998280.aspx +This typically occurs when the pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. This may result in exposure of sensitive information or possibly a crash.",,unclassified,"How To: Encrypt Configuration Sections in ASP.NET 2.0 Using DPAPI: http://msdn.microsoft.com/en-us/library/ms998280.aspx How To: Encrypt Configuration Sections in ASP.NET 2.0 Using RSA: http://msdn.microsoft.com/en-us/library/ms998283.aspx .NET Framework Developer's Guide - Securing Connection Strings: http://msdn.microsoft.com/en-us/library/89211k9b(VS.80).aspx" CWE-130,EN-Improper Handling of Length Parameter Inconsistency (Type: Variant),"The software parses a formatted message or structure, but it does not handle or incorrectly handles a length field that is inconsistent with the actual length of the associated data. -If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,Unknown, +If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,unclassified, CWE-132,EN-DEPRECATED (Duplicate): Miscalculated Null Termination (Type: Base),"This entry has been deprecated because it was a duplicate of CWE-170. All content has been transferred to CWE-170. -If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,Unknown, +If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,unclassified, CWE-135,EN-Incorrect Calculation of Multi-Byte String Length (Type: Base),"The software does not correctly calculate the length of strings that can contain wide or multi-byte characters. -If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,Unknown,"Writing Secure Code: Chapter 5, ""Unicode and ANSI Buffer Size Mismatches"" Page 153" +If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,unclassified,"Writing Secure Code: Chapter 5, ""Unicode and ANSI Buffer Size Mismatches"" Page 153" CWE-138,EN-Improper Neutralization of Special Elements (Type: Class),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as control elements or syntactic markers when they are sent to a downstream component. -Most languages and protocols have their own special elements such as characters and reserved words. These special elements can carry control implications. If software does not prevent external control or influence over the inclusion of such special elements, the control flow of the program may be altered from what was intended. For example, both Unix and Windows interpret the symbol < (""less than"") as meaning ""read input from a file"".",,Unknown, +Most languages and protocols have their own special elements such as characters and reserved words. These special elements can carry control implications. If software does not prevent external control or influence over the inclusion of such special elements, the control flow of the program may be altered from what was intended. For example, both Unix and Windows interpret the symbol < (""less than"") as meaning ""read input from a file"".",,unclassified, CWE-14,EN-Compiler Removal of Code to Clear Buffers (Type: Base),"Sensitive memory is cleared according to the source code, but compiler optimizations leave the memory untouched when it is not read from again, aka ""dead store removal."" This compiler optimization error occurs when: 1. Secret data are stored in memory. 2. The secret data are scrubbed from memory by overwriting its contents. -3. The source code is compiled using an optimizing compiler, which identifies and removes the function that overwrites the contents as a dead store because the memory is not used subsequently.",,Unknown,"Writing Secure Code: Chapter 9, ""A Compiler Optimization Caveat"" Page 322 +3. The source code is compiled using an optimizing compiler, which identifies and removes the function that overwrites the contents as a dead store because the memory is not used subsequently.",,unclassified,"Writing Secure Code: Chapter 9, ""A Compiler Optimization Caveat"" Page 322 When scrubbing secrets in memory doesn't work: http://cert.uni-stuttgart.de/archive/bugtraq/2002/11/msg00046.html Some Bad News and Some Good News: http://msdn.microsoft.com/library/default.asp?url=/library/en-us/dncode/html/secure10102002.asp GNU GCC: Optimizer Removes Code Necessary for Security: http://www.derkeiler.com/Mailing-Lists/securityfocus/bugtraq/2002-11/0257.html" @@ -793,334 +793,334 @@ CWE-140,EN-Improper Neutralization of Delimiters (Type: Base),"The software does This compiler optimization error occurs when: 1. Secret data are stored in memory. 2. The secret data are scrubbed from memory by overwriting its contents. -3. The source code is compiled using an optimizing compiler, which identifies and removes the function that overwrites the contents as a dead store because the memory is not used subsequently.",,Unknown, +3. The source code is compiled using an optimizing compiler, which identifies and removes the function that overwrites the contents as a dead store because the memory is not used subsequently.",,unclassified, CWE-141,EN-Improper Neutralization of Parameter/Argument Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as parameter or argument delimiters when they are sent to a downstream component. -As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,Unknown,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408. +As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408. The Art of Software Security Assessment: Chapter 10, ""IFS"", Page 604." CWE-142,EN-Improper Neutralization of Value Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as value delimiters when they are sent to a downstream component. -As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,Unknown,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." +As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." CWE-143,EN-Improper Neutralization of Record Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as record delimiters when they are sent to a downstream component. -As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,Unknown,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." +As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." CWE-144,EN-Improper Neutralization of Line Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as line delimiters when they are sent to a downstream component. -As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,Unknown,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." +As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." CWE-145,EN-Improper Neutralization of Section Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as section delimiters when they are sent to a downstream component. As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions. -One example of a section delimiter is the boundary string in a multipart MIME message. In many cases, doubled line delimiters can serve as a section delimiter.",,Unknown,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." +One example of a section delimiter is the boundary string in a multipart MIME message. In many cases, doubled line delimiters can serve as a section delimiter.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." CWE-146,EN-Improper Neutralization of Expression/Command Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as expression or command delimiters when they are sent to a downstream component. -As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,Unknown,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." +As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." CWE-147,EN-Improper Neutralization of Input Terminators (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as input terminators when they are sent to a downstream component. -For example, a ""."" in SMTP signifies the end of mail message data, whereas a null character can be used for the end of a string.",,Unknown, +For example, a ""."" in SMTP signifies the end of mail message data, whereas a null character can be used for the end of a string.",,unclassified, CWE-148,EN-Improper Neutralization of Input Leaders (Type: Variant),"The application does not properly handle when a leading character or sequence (""leader"") is missing or malformed, or if multiple leaders are used when only one should be allowed. -For example, a ""."" in SMTP signifies the end of mail message data, whereas a null character can be used for the end of a string.",,Unknown, +For example, a ""."" in SMTP signifies the end of mail message data, whereas a null character can be used for the end of a string.",,unclassified, CWE-149,EN-Improper Neutralization of Quoting Syntax (Type: Variant),"Quotes injected into an application can be used to compromise a system. As data are parsed, an injected/absent/duplicate/malformed use of quotes may cause the process to take unexpected actions. -For example, a ""."" in SMTP signifies the end of mail message data, whereas a null character can be used for the end of a string.",,Unknown, +For example, a ""."" in SMTP signifies the end of mail message data, whereas a null character can be used for the end of a string.",,unclassified, CWE-15,EN-External Control of System or Configuration Setting (Type: Base),"One or more system settings or configuration elements can be externally controlled by a user. -Allowing external control of system settings can disrupt service or cause an application to behave in unexpected, and potentially malicious ways.",,Unknown, +Allowing external control of system settings can disrupt service or cause an application to behave in unexpected, and potentially malicious ways.",,unclassified, CWE-150,"EN-Improper Neutralization of Escape, Meta, or Control Sequences (Type: Variant)","The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as escape, meta, or control character sequences when they are sent to a downstream component. -As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,Unknown, +As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified, CWE-151,EN-Improper Neutralization of Comment Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as comment delimiters when they are sent to a downstream component. -As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,Unknown, +As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified, CWE-152,EN-Improper Neutralization of Macro Symbols (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as macro symbols when they are sent to a downstream component. -As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,Unknown, +As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified, CWE-153,EN-Improper Neutralization of Substitution Characters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as substitution characters when they are sent to a downstream component. -As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,Unknown, +As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified, CWE-154,EN-Improper Neutralization of Variable Name Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as variable name delimiters when they are sent to a downstream component. -As data is parsed, an injected delimiter may cause the process to take unexpected actions that result in an attack. Example: ""$"" for an environment variable.",,Unknown, +As data is parsed, an injected delimiter may cause the process to take unexpected actions that result in an attack. Example: ""$"" for an environment variable.",,unclassified, CWE-155,EN-Improper Neutralization of Wildcards or Matching Symbols (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as wildcards or matching symbols when they are sent to a downstream component. -As data is parsed, an injected element may cause the process to take unexpected actions.",,Unknown, +As data is parsed, an injected element may cause the process to take unexpected actions.",,unclassified, CWE-156,EN-Improper Neutralization of Whitespace (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as whitespace when they are sent to a downstream component. -This can include space, tab, etc.",,Unknown, +This can include space, tab, etc.",,unclassified, CWE-157,EN-Failure to Sanitize Paired Delimiters (Type: Variant),"The software does not properly handle the characters that are used to mark the beginning and ending of a group of entities, such as parentheses, brackets, and braces. -This can include space, tab, etc.",,Unknown, +This can include space, tab, etc.",,unclassified, CWE-158,EN-Improper Neutralization of Null Byte or NUL Character (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes NUL characters or null bytes when they are sent to a downstream component. -As data is parsed, an injected NUL character or null byte may cause the software to believe the input is terminated earlier than it actually is, or otherwise cause the input to be misinterpreted. This could then be used to inject potentially dangerous input that occurs after the null byte or otherwise bypass validation routines and other protection mechanisms.",,Unknown,"The Art of Software Security Assessment: Chapter 8, ""NUL Character Injection"", Page 411." +As data is parsed, an injected NUL character or null byte may cause the software to believe the input is terminated earlier than it actually is, or otherwise cause the input to be misinterpreted. This could then be used to inject potentially dangerous input that occurs after the null byte or otherwise bypass validation routines and other protection mechanisms.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""NUL Character Injection"", Page 411." CWE-159,EN-Failure to Sanitize Special Element (Type: Class),"Weaknesses in this attack-focused category do not properly filter and interpret special elements in user-controlled input which could cause adverse effect on the software behavior and integrity. -As data is parsed, an injected NUL character or null byte may cause the software to believe the input is terminated earlier than it actually is, or otherwise cause the input to be misinterpreted. This could then be used to inject potentially dangerous input that occurs after the null byte or otherwise bypass validation routines and other protection mechanisms.",,Unknown, +As data is parsed, an injected NUL character or null byte may cause the software to believe the input is terminated earlier than it actually is, or otherwise cause the input to be misinterpreted. This could then be used to inject potentially dangerous input that occurs after the null byte or otherwise bypass validation routines and other protection mechanisms.",,unclassified, CWE-160,EN-Improper Neutralization of Leading Special Elements (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes leading special elements that could be interpreted in unexpected ways when they are sent to a downstream component. -As data is parsed, improperly handled leading special elements may cause the process to take unexpected actions that result in an attack.",,Unknown, +As data is parsed, improperly handled leading special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, CWE-161,EN-Improper Neutralization of Multiple Leading Special Elements (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes multiple leading special elements that could be interpreted in unexpected ways when they are sent to a downstream component. -As data is parsed, improperly handled multiple leading special elements may cause the process to take unexpected actions that result in an attack.",,Unknown, +As data is parsed, improperly handled multiple leading special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, CWE-162,EN-Improper Neutralization of Trailing Special Elements (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes trailing special elements that could be interpreted in unexpected ways when they are sent to a downstream component. -As data is parsed, improperly handled trailing special elements may cause the process to take unexpected actions that result in an attack.",,Unknown, +As data is parsed, improperly handled trailing special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, CWE-163,EN-Improper Neutralization of Multiple Trailing Special Elements (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes multiple trailing special elements that could be interpreted in unexpected ways when they are sent to a downstream component. -As data is parsed, improperly handled multiple trailing special elements may cause the process to take unexpected actions that result in an attack.",,Unknown, +As data is parsed, improperly handled multiple trailing special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, CWE-164,EN-Improper Neutralization of Internal Special Elements (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes internal special elements that could be interpreted in unexpected ways when they are sent to a downstream component. -As data is parsed, improperly handled internal special elements may cause the process to take unexpected actions that result in an attack.",,Unknown, +As data is parsed, improperly handled internal special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, CWE-165,EN-Improper Neutralization of Multiple Internal Special Elements (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes multiple internal special elements that could be interpreted in unexpected ways when they are sent to a downstream component. -As data is parsed, improperly handled multiple internal special elements may cause the process to take unexpected actions that result in an attack.",,Unknown, +As data is parsed, improperly handled multiple internal special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, CWE-166,EN-Improper Handling of Missing Special Element (Type: Base),"The software receives input from an upstream component, but it does not handle or incorrectly handles when an expected special element is missing. -As data is parsed, improperly handled multiple internal special elements may cause the process to take unexpected actions that result in an attack.",,Unknown, +As data is parsed, improperly handled multiple internal special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, CWE-167,EN-Improper Handling of Additional Special Element (Type: Base),"The software receives input from an upstream component, but it does not handle or incorrectly handles when an additional unexpected special element is missing. -As data is parsed, improperly handled multiple internal special elements may cause the process to take unexpected actions that result in an attack.",,Unknown, +As data is parsed, improperly handled multiple internal special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, CWE-168,EN-Improper Handling of Inconsistent Special Elements (Type: Base),"The software does not handle when an inconsistency exists between two or more special characters or reserved words. -An example of this problem would be if paired characters appear in the wrong order, or if the special characters are not properly nested.",,Unknown, +An example of this problem would be if paired characters appear in the wrong order, or if the special characters are not properly nested.",,unclassified, CWE-172,EN-Encoding Error (Type: Class),"The software does not properly encode or decode the data, resulting in unexpected values. -Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,Unknown, +Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified, CWE-173,EN-Improper Handling of Alternate Encoding (Type: Variant),"The software does not properly handle when an input uses an alternate encoding that is valid for the control sphere to which the input is being sent. -Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,Unknown, +Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified, CWE-174,EN-Double Decoding of the Same Data (Type: Variant),"The software decodes the same input twice, which can limit the effectiveness of any protection mechanism that occurs in between the decoding operations. -Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,Unknown, +Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified, CWE-175,EN-Improper Handling of Mixed Encoding (Type: Variant),"The software does not properly handle when the same input uses several different (mixed) encodings. -Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,Unknown, +Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified, CWE-176,EN-Improper Handling of Unicode Encoding (Type: Variant),"The software does not properly handle when an input contains Unicode encoding. -Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,Unknown,"The Art of Software Security Assessment: Chapter 8, ""Character Sets and Unicode"", Page 446." +Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Character Sets and Unicode"", Page 446." CWE-177,EN-Improper Handling of URL Encoding (Hex Encoding) (Type: Variant),"The software does not properly handle when all or part of an input has been URL encoded. -Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,Unknown, +Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified, CWE-178,EN-Improper Handling of Case Sensitivity (Type: Base),"The software does not properly account for differences in case sensitivity when accessing or determining the properties of a resource, leading to inconsistent results. Improperly handled case sensitive data can lead to several possible consequences, including: case-insensitive passwords reducing the size of the key space, making brute force attacks easier bypassing filters or access controls using alternate names -multiple interpretation errors using alternate names.",,Unknown, +multiple interpretation errors using alternate names.",,unclassified, CWE-179,EN-Incorrect Behavior Order: Early Validation (Type: Base),"The software validates input before applying protection mechanisms that modify the input, which could allow an attacker to bypass the validation via dangerous inputs that only arise after the modification. -Software needs to validate data at the proper time, after data has been canonicalized and cleansed. Early validation is susceptible to various manipulations that result in dangerous inputs that are produced by canonicalization and cleansing.",,Unknown,"The Art of Software Security Assessment: Chapter 8, ""Escaping Metacharacters"", Page 439." +Software needs to validate data at the proper time, after data has been canonicalized and cleansed. Early validation is susceptible to various manipulations that result in dangerous inputs that are produced by canonicalization and cleansing.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Escaping Metacharacters"", Page 439." CWE-180,EN-Incorrect Behavior Order: Validate Before Canonicalize (Type: Base),"The software validates input before it is canonicalized, which prevents the software from detecting data that becomes invalid after the canonicalization step. -This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,Unknown, +This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,unclassified, CWE-181,EN-Incorrect Behavior Order: Validate Before Filter (Type: Base),"The software validates data before it has been filtered, which prevents the software from detecting data that becomes invalid after the filtering step. -This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,Unknown, +This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,unclassified, CWE-182,EN-Collapse of Data into Unsafe Value (Type: Base),"The software filters data in a way that causes it to be reduced or ""collapsed"" into an unsafe value that violates an expected security property. -This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,Unknown,"The Art of Software Security Assessment: Chapter 8, ""Character Stripping Vulnerabilities"", Page 437." +This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Character Stripping Vulnerabilities"", Page 437." CWE-183,EN-Permissive Whitelist (Type: Base),"An application uses a ""whitelist"" of acceptable values, but the whitelist includes at least one unsafe value, leading to resultant weaknesses. -This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,Unknown,"The Art of Software Security Assessment: Chapter 8, ""Eliminating Metacharacters"", Page 435." +This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Eliminating Metacharacters"", Page 435." CWE-184,EN-Incomplete Blacklist (Type: Base),"An application uses a ""blacklist"" of prohibited values, but the blacklist is incomplete. -If an incomplete blacklist is used as a security mechanism, then the software may allow unintended values to pass into the application logic.",,Unknown,"Exploiting Software: How to Break Code +If an incomplete blacklist is used as a security mechanism, then the software may allow unintended values to pass into the application logic.",,unclassified,"Exploiting Software: How to Break Code Blacklist defenses as a breeding ground for vulnerability variants: http://seclists.org/fulldisclosure/2006/Feb/0040.html The Art of Software Security Assessment: Chapter 8, ""Eliminating Metacharacters"", Page 435." CWE-185,EN-Incorrect Regular Expression (Type: Class),"The software specifies a regular expression in a way that causes data to be improperly matched or compared. -When the regular expression is used in protection mechanisms such as filtering or validation, this may allow an attacker to bypass the intended restrictions on the incoming data.",,Unknown,"Writing Secure Code: Chapter 10, ""Using Regular Expressions for Checking Input"" Page 350" +When the regular expression is used in protection mechanisms such as filtering or validation, this may allow an attacker to bypass the intended restrictions on the incoming data.",,unclassified,"Writing Secure Code: Chapter 10, ""Using Regular Expressions for Checking Input"" Page 350" CWE-186,EN-Overly Restrictive Regular Expression (Type: Base),"A regular expression is overly restrictive, which prevents dangerous values from being detected. -When the regular expression is used in protection mechanisms such as filtering or validation, this may allow an attacker to bypass the intended restrictions on the incoming data.",,Unknown, +When the regular expression is used in protection mechanisms such as filtering or validation, this may allow an attacker to bypass the intended restrictions on the incoming data.",,unclassified, CWE-187,EN-Partial Comparison (Type: Base),"The software performs a comparison that only examines a portion of a factor before determining whether there is a match, such as a substring, leading to resultant weaknesses. -For example, an attacker might succeed in authentication by providing a small password that matches the associated portion of the larger, correct password.",,Unknown, +For example, an attacker might succeed in authentication by providing a small password that matches the associated portion of the larger, correct password.",,unclassified, CWE-191,EN-Integer Underflow (Wrap or Wraparound) (Type: Base),"The product subtracts one value from another, such that the result is less than the minimum allowable integer value, which produces a value that is not equal to the correct result. -This can happen in signed and unsigned cases.",,Unknown,"24 Deadly Sins of Software Security: ""Sin 7: Integer Overflows."" Page 119" +This can happen in signed and unsigned cases.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 7: Integer Overflows."" Page 119" CWE-193,EN-Off-by-one Error (Type: Base),"A product calculates or uses an incorrect maximum or minimum value that is 1 more, or 1 less, than the correct value. -This can happen in signed and unsigned cases.",,Unknown,"Third Generation Exploits: http://www.blackhat.com/presentations/bh-europe-01/halvar-flake/bh-europe-01-halvarflake.ppt +This can happen in signed and unsigned cases.",,unclassified,"Third Generation Exploits: http://www.blackhat.com/presentations/bh-europe-01/halvar-flake/bh-europe-01-halvarflake.ppt Off-by-one errors: a brief explanation: http://marc.theaimsgroup.com/?l=secprog&m=108379742110553&w=2 The Frame Pointer Overwrite: http://kaizo.org/mirrors/phrack/phrack55/P55-08 Exploiting Software: How to Break Code (The buffer overflow chapter) 24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89 The Art of Software Security Assessment: Chapter 5, ""Off-by-One Errors"", Page 180." CWE-195,EN-Signed to Unsigned Conversion Error (Type: Variant),"A signed-to-unsigned conversion error takes place when a signed primitive is used as an unsigned value, usually as a size variable. -It is dangerous to rely on implicit casts between signed and unsigned numbers because the result can take on an unexpected value and violate assumptions made by the program.",,Unknown,"The Art of Software Security Assessment: Chapter 6, ""Type Conversions"", Page 223." +It is dangerous to rely on implicit casts between signed and unsigned numbers because the result can take on an unexpected value and violate assumptions made by the program.",,unclassified,"The Art of Software Security Assessment: Chapter 6, ""Type Conversions"", Page 223." CWE-198,EN-Use of Incorrect Byte Ordering (Type: Base),"The software receives input from an upstream component, but it does not account for byte ordering (e.g. big-endian and little-endian) when processing the input, causing an incorrect number or value to be used. -When a primitive is cast to a smaller primitive, the high order bits of the large value are lost in the conversion, potentially resulting in an unexpected value that is not equal to the original value. This value may be required as an index into a buffer, a loop iterator, or simply necessary state data. In any case, the value cannot be trusted and the system will be in an undefined state. While this method may be employed viably to isolate the low bits of a value, this usage is rare, and truncation usually implies that an implementation error has occurred.",,Unknown, +When a primitive is cast to a smaller primitive, the high order bits of the large value are lost in the conversion, potentially resulting in an unexpected value that is not equal to the original value. This value may be required as an index into a buffer, a loop iterator, or simply necessary state data. In any case, the value cannot be trusted and the system will be in an undefined state. While this method may be employed viably to isolate the low bits of a value, this usage is rare, and truncation usually implies that an implementation error has occurred.",,unclassified, CWE-201,EN-Information Exposure Through Sent Data (Type: Variant),"The accidental exposure of sensitive information through sent data refers to the transmission of data which are either sensitive in and of itself or useful in the further exploitation of the system through standard data channels. The information either is regarded as sensitive within the product's own functionality, such as a private message; or provides information about the product or its environment that could be useful in an attack but is normally not available to the attacker, such as the installation path of a product that is remotely accessible. -Many information exposures are resultant (e.g. PHP script error revealing the full path of the program), but they can also be primary (e.g. timing discrepancies in cryptography). There are many different types of problems that involve information exposures. Their severity can range widely depending on the type of information that is revealed.",,Unknown, +Many information exposures are resultant (e.g. PHP script error revealing the full path of the program), but they can also be primary (e.g. timing discrepancies in cryptography). There are many different types of problems that involve information exposures. Their severity can range widely depending on the type of information that is revealed.",,unclassified, CWE-203,EN-Information Exposure Through Discrepancy (Type: Class),"The product behaves differently or sends different responses in a way that exposes security-relevant information about the state of the product, such as whether a particular operation was successful or not. -In situations where data should not be tied to individual users, but a large number of users should be able to make queries that ""scrub"" the identity of users, it may be possible to get information about a user -- e.g., by specifying search terms that are known to be unique to that user.",,Unknown, +In situations where data should not be tied to individual users, but a large number of users should be able to make queries that ""scrub"" the identity of users, it may be possible to get information about a user -- e.g., by specifying search terms that are known to be unique to that user.",,unclassified, CWE-204,EN-Response Discrepancy Information Exposure (Type: Base),"The software provides different responses to incoming requests in a way that allows an actor to determine system state information that is outside of that actor's control sphere. -This issue frequently occurs during authentication, where a difference in failed-login messages could allow an attacker to determine if the username is valid or not. These exposures can be inadvertent (bug) or intentional (design).",,Unknown,"24 Deadly Sins of Software Security: ""Sin 12: Information Leakage."" Page 191" +This issue frequently occurs during authentication, where a difference in failed-login messages could allow an attacker to determine if the username is valid or not. These exposures can be inadvertent (bug) or intentional (design).",,unclassified,"24 Deadly Sins of Software Security: ""Sin 12: Information Leakage."" Page 191" CWE-205,EN-Information Exposure Through Behavioral Discrepancy (Type: Base),"The product's actions indicate important differences based on (1) the internal state of the product or (2) differences from other products in the same class. -For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,Unknown, +For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,unclassified, CWE-206,EN-Information Exposure of Internal State Through Behavioral Inconsistency (Type: Variant),"Two separate operations in a product cause the product to behave differently in a way that is observable to an attacker and reveals security-relevant information about the internal state of the product, such as whether a particular operation was successful or not. -For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,Unknown, +For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,unclassified, CWE-207,EN-Information Exposure Through an External Behavioral Inconsistency (Type: Variant),"The product behaves differently than other products like it, in a way that is observable to an attacker and exposes security-relevant information about which product is being used. -For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,Unknown, +For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,unclassified, CWE-208,EN-Information Exposure Through Timing Discrepancy (Type: Base),"Two separate operations in a product require different amounts of time to complete, in a way that is observable to an actor and reveals security-relevant information about the state of the product, such as whether a particular operation was successful or not. -For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,Unknown, +For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,unclassified, CWE-210,EN-Information Exposure Through Self-generated Error Message (Type: Base),"The software identifies an error condition and creates its own diagnostic or error messages that contain sensitive information. -The sensitive information may be valuable information on its own (such as a password), or it may be useful for launching other, more deadly attacks. If an attack fails, an attacker may use error information provided by the server to launch another more focused attack. For example, an attempt to exploit a path traversal weakness (CWE-22) might yield the full pathname of the installed application. In turn, this could be used to select the proper number of "".."" sequences to navigate to the targeted file. An attack using SQL injection (CWE-89) might not initially succeed, but an error message could reveal the malformed query, which would expose query logic and possibly even passwords or other sensitive information used within the query.",,Unknown,"24 Deadly Sins of Software Security: ""Sin 12: Information Leakage."" Page 191 +The sensitive information may be valuable information on its own (such as a password), or it may be useful for launching other, more deadly attacks. If an attack fails, an attacker may use error information provided by the server to launch another more focused attack. For example, an attempt to exploit a path traversal weakness (CWE-22) might yield the full pathname of the installed application. In turn, this could be used to select the proper number of "".."" sequences to navigate to the targeted file. An attack using SQL injection (CWE-89) might not initially succeed, but an error message could reveal the malformed query, which would expose query logic and possibly even passwords or other sensitive information used within the query.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 12: Information Leakage."" Page 191 The Art of Software Security Assessment: Chapter 3, ""Overly Verbose Error Messages"", Page 75." CWE-211,EN-Information Exposure Through Externally-generated Error Message (Type: Base),"The software performs an operation that triggers an external diagnostic or error message that is not directly generated by the software, such as an error generated by the programming language interpreter that the software uses. The error can contain sensitive system information. -The sensitive information may be valuable information on its own (such as a password), or it may be useful for launching other, more deadly attacks. If an attack fails, an attacker may use error information provided by the server to launch another more focused attack. For example, an attempt to exploit a path traversal weakness (CWE-22) might yield the full pathname of the installed application. In turn, this could be used to select the proper number of "".."" sequences to navigate to the targeted file. An attack using SQL injection (CWE-89) might not initially succeed, but an error message could reveal the malformed query, which would expose query logic and possibly even passwords or other sensitive information used within the query.",,Unknown, +The sensitive information may be valuable information on its own (such as a password), or it may be useful for launching other, more deadly attacks. If an attack fails, an attacker may use error information provided by the server to launch another more focused attack. For example, an attempt to exploit a path traversal weakness (CWE-22) might yield the full pathname of the installed application. In turn, this could be used to select the proper number of "".."" sequences to navigate to the targeted file. An attack using SQL injection (CWE-89) might not initially succeed, but an error message could reveal the malformed query, which would expose query logic and possibly even passwords or other sensitive information used within the query.",,unclassified, CWE-212,EN-Improper Cross-boundary Removal of Sensitive Data (Type: Base),"The software uses a resource that contains sensitive data, but it does not properly remove that data before it stores, transfers, or shares the resource with actors in another control sphere. Resources that may contain sensitive data include documents, packets, messages, databases, etc. While this data may be useful to an individual user or small set of users who share the resource, it may need to be removed before the resource can be shared outside of the trusted group. The process of removal is sometimes called cleansing or scrubbing. -For example, software that is used for editing documents might not remove sensitive data such as reviewer comments or the local pathname where the document is stored. Or, a proxy might not remove an internal IP address from headers before making an outgoing request to an Internet site.",,Unknown, +For example, software that is used for editing documents might not remove sensitive data such as reviewer comments or the local pathname where the document is stored. Or, a proxy might not remove an internal IP address from headers before making an outgoing request to an Internet site.",,unclassified, CWE-213,EN-Intentional Information Exposure (Type: Base),"A product's design or configuration explicitly requires the publication of information that could be regarded as sensitive by an administrator. Resources that may contain sensitive data include documents, packets, messages, databases, etc. While this data may be useful to an individual user or small set of users who share the resource, it may need to be removed before the resource can be shared outside of the trusted group. The process of removal is sometimes called cleansing or scrubbing. -For example, software that is used for editing documents might not remove sensitive data such as reviewer comments or the local pathname where the document is stored. Or, a proxy might not remove an internal IP address from headers before making an outgoing request to an Internet site.",,Unknown, +For example, software that is used for editing documents might not remove sensitive data such as reviewer comments or the local pathname where the document is stored. Or, a proxy might not remove an internal IP address from headers before making an outgoing request to an Internet site.",,unclassified, CWE-214,EN-Information Exposure Through Process Environment (Type: Variant),"A process is invoked with sensitive arguments, environment variables, or other elements that can be seen by other processes on the operating system. -Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,Unknown, +Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified, CWE-215,EN-Information Exposure Through Debug Information (Type: Variant),"The application contains debugging code that can expose sensitive information to untrusted parties. -Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,Unknown, +Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified, CWE-216,EN-Containment Errors (Container Errors) (Type: Class),"This tries to cover various problems in which improper data are included within a ""container."" -Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,Unknown, +Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified, CWE-217,EN-DEPRECATED: Failure to Protect Stored Data from Modification (Type: Base),"This weakness has been deprecated because it incorporated and confused multiple weaknesses. The issues formerly covered in this weakness can be found at CWE-766 and CWE-767. -Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,Unknown, +Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified, CWE-218,EN-DEPRECATED (Duplicate): Failure to provide confidentiality for stored data (Type: Base),"This weakness has been deprecated because it was a duplicate of CWE-493. All content has been transferred to CWE-493. -Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,Unknown, +Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified, CWE-219,EN-Sensitive Data Under Web Root (Type: Variant),"The application stores sensitive data under the web document root with insufficient access control, which might make it accessible to untrusted parties. -Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,Unknown, +Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified, CWE-220,EN-Sensitive Data Under FTP Root (Type: Variant),"The application stores sensitive data under the FTP document root with insufficient access control, which might make it accessible to untrusted parties. Many file operations are intended to take place within a restricted directory. By using special elements such as "".."" and ""/"" separators, attackers can escape outside of the restricted location to access files or directories that are elsewhere on the system. One of the most common special elements is the ""../"" sequence, which in most modern operating systems is interpreted as the parent directory of the current location. This is referred to as relative path traversal. Path traversal also covers the use of absolute pathnames such as ""/usr/local/bin"", which may also be useful in accessing unexpected files. This is referred to as absolute path traversal. -In many programming languages, the injection of a null byte (the 0 or NUL) may allow an attacker to truncate a generated filename to widen the scope of attack. For example, the software may add "".txt"" to any pathname, thus limiting the attacker to text files, but a null injection may effectively remove this restriction.",,Unknown, +In many programming languages, the injection of a null byte (the 0 or NUL) may allow an attacker to truncate a generated filename to widen the scope of attack. For example, the software may add "".txt"" to any pathname, thus limiting the attacker to text files, but a null injection may effectively remove this restriction.",,unclassified, CWE-221,EN-Information Loss or Omission (Type: Class),"The software does not record, or improperly records, security-relevant information that leads to an incorrect decision or hampers later analysis. -This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,Unknown, +This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified, CWE-222,EN-Truncation of Security-relevant Information (Type: Base),"The application truncates the display, recording, or processing of security-relevant information in a way that can obscure the source or nature of an attack. -This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,Unknown, +This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified, CWE-223,EN-Omission of Security-relevant Information (Type: Base),"The application does not record or display information that would be important for identifying the source or nature of an attack, or determining if an action is safe. -This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,Unknown,"The Art of Software Security Assessment: Chapter 2, ""Accountability"", Page 40." +This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified,"The Art of Software Security Assessment: Chapter 2, ""Accountability"", Page 40." CWE-224,EN-Obscured Security-relevant Information by Alternate Name (Type: Base),"The software records security-relevant information according to an alternate name of the affected entity, instead of the canonical name. -This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,Unknown,Writing Secure Code +This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified,Writing Secure Code CWE-225,EN-DEPRECATED (Duplicate): General Information Management Problems (Type: Base),"This weakness can be found at CWE-199. -This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,Unknown, +This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified, CWE-226,EN-Sensitive Information Uncleared Before Release (Type: Base),"The software does not fully clear previously used information in a data structure, file, or other resource, before making that resource available to a party in another control sphere. -This typically results from new data that is not as long as the old data, which leaves portions of the old data still available. Equivalent errors can occur in other situations where the length of data is variable but the associated data structure is not. If memory is not cleared after use, it may allow unintended actors to read the data when the memory is reallocated.",,Unknown, +This typically results from new data that is not as long as the old data, which leaves portions of the old data still available. Equivalent errors can occur in other situations where the length of data is variable but the associated data structure is not. If memory is not cleared after use, it may allow unintended actors to read the data when the memory is reallocated.",,unclassified, CWE-227,EN-Improper Fulfillment of API Contract (API Abuse) (Type: Class),"The software uses an API in a manner contrary to its intended use. -An API is a contract between a caller and a callee. The most common forms of API misuse occurs when the caller does not honor its end of this contract. For example, if a program does not call chdir() after calling chroot(), it violates the contract that specifies how to change the active root directory in a secure fashion. Another good example of library abuse is expecting the callee to return trustworthy DNS information to the caller. In this case, the caller misuses the callee API by making certain assumptions about its behavior (that the return value can be used for authentication purposes). One can also violate the caller-callee contract from the other side. For example, if a coder subclasses SecureRandom and returns a non-random value, the contract is violated.",,Unknown, +An API is a contract between a caller and a callee. The most common forms of API misuse occurs when the caller does not honor its end of this contract. For example, if a program does not call chdir() after calling chroot(), it violates the contract that specifies how to change the active root directory in a secure fashion. Another good example of library abuse is expecting the callee to return trustworthy DNS information to the caller. In this case, the caller misuses the callee API by making certain assumptions about its behavior (that the return value can be used for authentication purposes). One can also violate the caller-callee contract from the other side. For example, if a coder subclasses SecureRandom and returns a non-random value, the contract is violated.",,unclassified, CWE-228,EN-Improper Handling of Syntactically Invalid Structure (Type: Class),"The product does not handle or incorrectly handles input that is not syntactically well-formed with respect to the associated specification. -An API is a contract between a caller and a callee. The most common forms of API misuse occurs when the caller does not honor its end of this contract. For example, if a program does not call chdir() after calling chroot(), it violates the contract that specifies how to change the active root directory in a secure fashion. Another good example of library abuse is expecting the callee to return trustworthy DNS information to the caller. In this case, the caller misuses the callee API by making certain assumptions about its behavior (that the return value can be used for authentication purposes). One can also violate the caller-callee contract from the other side. For example, if a coder subclasses SecureRandom and returns a non-random value, the contract is violated.",,Unknown, +An API is a contract between a caller and a callee. The most common forms of API misuse occurs when the caller does not honor its end of this contract. For example, if a program does not call chdir() after calling chroot(), it violates the contract that specifies how to change the active root directory in a secure fashion. Another good example of library abuse is expecting the callee to return trustworthy DNS information to the caller. In this case, the caller misuses the callee API by making certain assumptions about its behavior (that the return value can be used for authentication purposes). One can also violate the caller-callee contract from the other side. For example, if a coder subclasses SecureRandom and returns a non-random value, the contract is violated.",,unclassified, CWE-229,EN-Improper Handling of Values (Type: Base),"The software does not properly handle when the expected number of values for parameters, fields, or arguments is not provided in input, or if those values are undefined. -An API is a contract between a caller and a callee. The most common forms of API misuse occurs when the caller does not honor its end of this contract. For example, if a program does not call chdir() after calling chroot(), it violates the contract that specifies how to change the active root directory in a secure fashion. Another good example of library abuse is expecting the callee to return trustworthy DNS information to the caller. In this case, the caller misuses the callee API by making certain assumptions about its behavior (that the return value can be used for authentication purposes). One can also violate the caller-callee contract from the other side. For example, if a coder subclasses SecureRandom and returns a non-random value, the contract is violated.",,Unknown, +An API is a contract between a caller and a callee. The most common forms of API misuse occurs when the caller does not honor its end of this contract. For example, if a program does not call chdir() after calling chroot(), it violates the contract that specifies how to change the active root directory in a secure fashion. Another good example of library abuse is expecting the callee to return trustworthy DNS information to the caller. In this case, the caller misuses the callee API by making certain assumptions about its behavior (that the return value can be used for authentication purposes). One can also violate the caller-callee contract from the other side. For example, if a coder subclasses SecureRandom and returns a non-random value, the contract is violated.",,unclassified, CWE-23,EN-Relative Path Traversal (Type: Base),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize sequences such as "".."" that can resolve to a location that is outside of that directory. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,Unknown,"OWASP Attack listing: http://www.owasp.org/index.php/Relative_Path_Traversal +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified,"OWASP Attack listing: http://www.owasp.org/index.php/Relative_Path_Traversal The Art of Software Security Assessment: Chapter 9, ""Filenames and Paths"", Page 503." CWE-230,EN-Improper Handling of Missing Values (Type: Variant),"The software does not handle or incorrectly handles when a parameter, field, or argument name is specified, but the associated value is missing, i.e. it is empty, blank, or null. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,Unknown, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, CWE-231,EN-Improper Handling of Extra Values (Type: Variant),"The software does not handle or incorrectly handles when more values are provided than expected. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,Unknown, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, CWE-232,EN-Improper Handling of Undefined Values (Type: Variant),"The software does not handle or incorrectly handles when a value is not defined or supported for the associated parameter, field, or argument name. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,Unknown, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, CWE-233,EN-Improper Handling of Parameters (Type: Base),"The software does not properly handle when the expected number of parameters, fields, or arguments is not provided in input, or if those parameters are undefined. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,Unknown, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, CWE-235,EN-Improper Handling of Extra Parameters (Type: Variant),"The software does not handle or incorrectly handles when the number of parameters, fields, or arguments with the same name exceeds the expected amount. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,Unknown, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, CWE-236,EN-Improper Handling of Undefined Parameters (Type: Variant),"The software does not handle or incorrectly handles when a particular parameter, field, or argument name is not defined or supported by the product. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,Unknown, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, CWE-237,EN-Improper Handling of Structural Elements (Type: Base),"The software does not handle or incorrectly handles inputs that are related to complex structures. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,Unknown, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, CWE-238,EN-Improper Handling of Incomplete Structural Elements (Type: Variant),"The software does not handle or incorrectly handles when a particular structural element is not completely specified. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,Unknown, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, CWE-239,EN-Failure to Handle Incomplete Element (Type: Variant),"The software does not properly handle when a particular element is not completely specified. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,Unknown, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, CWE-24,EN-Path Traversal: ../filedir (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize ""../"" sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The ""../"" manipulation is the canonical manipulation for operating systems that use ""/"" as directory separators, such as UNIX- and Linux-based systems. In some cases, it is useful for bypassing protection schemes in environments for which ""/"" is supported but not the primary separator, such as Windows, which uses ""\"" but can also accept ""/"".",,Unknown, +The ""../"" manipulation is the canonical manipulation for operating systems that use ""/"" as directory separators, such as UNIX- and Linux-based systems. In some cases, it is useful for bypassing protection schemes in environments for which ""/"" is supported but not the primary separator, such as Windows, which uses ""\"" but can also accept ""/"".",,unclassified, CWE-240,EN-Improper Handling of Inconsistent Structural Elements (Type: Variant),"The software does not handle or incorrectly handles when two or more structural elements should be consistent, but are not. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The ""../"" manipulation is the canonical manipulation for operating systems that use ""/"" as directory separators, such as UNIX- and Linux-based systems. In some cases, it is useful for bypassing protection schemes in environments for which ""/"" is supported but not the primary separator, such as Windows, which uses ""\"" but can also accept ""/"".",,Unknown, +The ""../"" manipulation is the canonical manipulation for operating systems that use ""/"" as directory separators, such as UNIX- and Linux-based systems. In some cases, it is useful for bypassing protection schemes in environments for which ""/"" is supported but not the primary separator, such as Windows, which uses ""\"" but can also accept ""/"".",,unclassified, CWE-241,EN-Improper Handling of Unexpected Data Type (Type: Base),"The software does not handle or incorrectly handles when a particular element is not the expected type, e.g. it expects a digit (0-9) but is provided with a letter (A-Z). This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The ""../"" manipulation is the canonical manipulation for operating systems that use ""/"" as directory separators, such as UNIX- and Linux-based systems. In some cases, it is useful for bypassing protection schemes in environments for which ""/"" is supported but not the primary separator, such as Windows, which uses ""\"" but can also accept ""/"".",,Unknown, +The ""../"" manipulation is the canonical manipulation for operating systems that use ""/"" as directory separators, such as UNIX- and Linux-based systems. In some cases, it is useful for bypassing protection schemes in environments for which ""/"" is supported but not the primary separator, such as Windows, which uses ""\"" but can also accept ""/"".",,unclassified, CWE-244,EN-Improper Clearing of Heap Memory Before Release (Heap Inspection) (Type: Variant),"Using realloc() to resize buffers that store sensitive information can leave the sensitive information exposed to attack, because it is not removed from memory. -When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,Unknown, +When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,unclassified, CWE-245,EN-J2EE Bad Practices: Direct Management of Connections (Type: Variant),"The J2EE application directly manages connections, instead of using the container's connection management facilities. -When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,Unknown, +When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,unclassified, CWE-246,EN-J2EE Bad Practices: Direct Use of Sockets (Type: Variant),"The J2EE application directly uses sockets instead of using framework method calls. -When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,Unknown, +When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,unclassified, CWE-247,EN-DEPRECATED (Duplicate): Reliance on DNS Lookups in a Security Decision (Type: Base),"This entry has been deprecated because it was a duplicate of CWE-350. All content has been transferred to CWE-350. -When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,Unknown, +When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,unclassified, CWE-248,EN-Uncaught Exception (Type: Base),"An exception is thrown from a function, but it is not caught. -When an exception is not caught, it may cause the program to crash or expose sensitive information.",,Unknown, +When an exception is not caught, it may cause the program to crash or expose sensitive information.",,unclassified, CWE-249,EN-DEPRECATED: Often Misused: Path Manipulation (Type: Variant),"This entry has been deprecated because of name confusion and an accidental combination of multiple weaknesses. Most of its content has been transferred to CWE-785. -When an exception is not caught, it may cause the program to crash or expose sensitive information.",,Unknown, +When an exception is not caught, it may cause the program to crash or expose sensitive information.",,unclassified, CWE-25,EN-Path Traversal: /../filedir (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize ""/../"" sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -Sometimes a program checks for ""../"" at the beginning of the input, so a ""/../"" can bypass that check.",,Unknown, +Sometimes a program checks for ""../"" at the beginning of the input, so a ""/../"" can bypass that check.",,unclassified, CWE-26,EN-Path Traversal: /dir/../filename (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize ""/dir/../filename"" sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '/dir/../filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only checks for ""../"" at the beginning of the input, so a ""/../"" can bypass that check.",,Unknown, +The '/dir/../filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only checks for ""../"" at the beginning of the input, so a ""/../"" can bypass that check.",,unclassified, CWE-260,EN-Password in Configuration File (Type: Variant),"The software stores a password in a configuration file that might be accessible to actors who do not know the password. -This can result in compromise of the system for which the password is used. An attacker could gain access to this file and learn the stored password or worse yet, change the password to one of their choosing.",,Unknown,Building Secure Software: How to Avoid Security Problems the Right Way +This can result in compromise of the system for which the password is used. An attacker could gain access to this file and learn the stored password or worse yet, change the password to one of their choosing.",,unclassified,Building Secure Software: How to Avoid Security Problems the Right Way CWE-261,EN-Weak Cryptography for Passwords (Type: Variant),"Obscuring a password with a trivial encoding does not protect the password. -This can result in compromise of the system for which the password is used. An attacker could gain access to this file and learn the stored password or worse yet, change the password to one of their choosing.",,Unknown,"Building Secure Software: How to Avoid Security Problems the Right Way +This can result in compromise of the system for which the password is used. An attacker could gain access to this file and learn the stored password or worse yet, change the password to one of their choosing.",,unclassified,"Building Secure Software: How to Avoid Security Problems the Right Way 24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" CWE-266,EN-Incorrect Privilege Assignment (Type: Base),"A product incorrectly assigns a privilege to a particular actor, creating an unintended sphere of control for that actor. -Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,Unknown,Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html +Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,unclassified,Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html CWE-267,EN-Privilege Defined With Unsafe Actions (Type: Base),"A particular privilege, role, capability, or right can be used to perform unsafe actions that were not intended, even when it is assigned to the correct entity. -Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,Unknown,Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html +Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,unclassified,Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html CWE-27,EN-Path Traversal: dir/../../filename (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize multiple internal ""../"" sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The 'directory/../../filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only removes one ""../"" sequence, so multiple ""../"" can bypass that check. Alternately, this manipulation could be used to bypass a check for ""../"" at the beginning of the pathname, moving up more than one directory level.",,Unknown, +The 'directory/../../filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only removes one ""../"" sequence, so multiple ""../"" can bypass that check. Alternately, this manipulation could be used to bypass a check for ""../"" at the beginning of the pathname, moving up more than one directory level.",,unclassified, CWE-270,EN-Privilege Context Switching Error (Type: Base),"The software does not properly manage privileges while it is switching between different contexts that have different privileges or spheres of control. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The 'directory/../../filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only removes one ""../"" sequence, so multiple ""../"" can bypass that check. Alternately, this manipulation could be used to bypass a check for ""../"" at the beginning of the pathname, moving up more than one directory level.",,Unknown,"Writing Secure Code: Chapter 7, ""Running with Least Privilege"" Page 207 +The 'directory/../../filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only removes one ""../"" sequence, so multiple ""../"" can bypass that check. Alternately, this manipulation could be used to bypass a check for ""../"" at the beginning of the pathname, moving up more than one directory level.",,unclassified,"Writing Secure Code: Chapter 7, ""Running with Least Privilege"" Page 207 Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html" CWE-272,EN-Least Privilege Violation (Type: Base),"The elevated privilege level required to perform operations such as chroot() should be dropped immediately after the operation is performed. -In some contexts, a system executing with elevated permissions will hand off a process/file/etc. to another process or user. If the privileges of an entity are not reduced, then elevated privileges are spread throughout a system and possibly to an attacker.",,Unknown, +In some contexts, a system executing with elevated permissions will hand off a process/file/etc. to another process or user. If the privileges of an entity are not reduced, then elevated privileges are spread throughout a system and possibly to an attacker.",,unclassified, CWE-274,EN-Improper Handling of Insufficient Privileges (Type: Base),"The software does not handle or incorrectly handles when it has insufficient privileges to perform an operation, leading to resultant weaknesses. -If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,Unknown, +If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,unclassified, CWE-277,EN-Insecure Inherited Permissions (Type: Variant),"A product defines a set of insecure permissions that are inherited by objects that are created by the program. -If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,Unknown, +If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,unclassified, CWE-278,EN-Insecure Preserved Inherited Permissions (Type: Variant),"A product inherits a set of insecure permissions for an object, e.g. when copying from an archive file, without user awareness or involvement. -If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,Unknown, +If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,unclassified, CWE-279,EN-Incorrect Execution-Assigned Permissions (Type: Variant),"While it is executing, the software sets the permissions of an object in a way that violates the intended permissions that have been specified by the user. -If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,Unknown, +If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,unclassified, CWE-28,EN-Path Traversal: ..\filedir (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize ""..\"" sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,Unknown, +The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified, CWE-280,EN-Improper Handling of Insufficient Permissions or Privileges (Type: Base),"The application does not handle or incorrectly handles when it has insufficient privileges to access resources or functionality as specified by their permissions. This may cause it to follow unexpected code paths that may leave the application in an invalid state. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,Unknown, +The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified, CWE-281,EN-Improper Preservation of Permissions (Type: Base),"The software does not preserve permissions or incorrectly preserves permissions when copying, restoring, or sharing objects, which can cause them to have less restrictive permissions than intended. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,Unknown, +The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified, CWE-282,EN-Improper Ownership Management (Type: Class),"The software assigns the wrong ownership, or does not properly verify the ownership, of an object or resource. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,Unknown, +The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified, CWE-283,EN-Unverified Ownership (Type: Base),"The software does not properly verify that a critical resource is owned by the proper entity. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,Unknown, +The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified, CWE-284,EN-Improper Access Control (Type: Class),"The software does not restrict or incorrectly restricts access to a resource from an unauthorized actor. Access control involves the use of several protection mechanisms such as authentication (proving the identity of an actor) authorization (ensuring that a given actor can access a resource), and accountability (tracking of activities that were performed). When any mechanism is not applied or otherwise fails, attackers can compromise the security of the software by gaining privileges, reading sensitive information, executing commands, evading detection, etc. There are two distinct behaviors that can introduce access control weaknesses: Specification: incorrect privileges, permissions, ownership, etc. are explicitly specified for either the user or the resource (for example, setting a password file to be world-writable, or giving administrator capabilities to a guest user). This action could be performed by the program or the administrator. -Enforcement: the mechanism contains errors that prevent it from properly enforcing the specified access control requirements (e.g., allowing the user to specify their own privileges, or allowing a syntactically-incorrect ACL to produce insecure settings). This problem occurs within the program itself, in that it does not actually enforce the intended security policy that the administrator specifies.",,Unknown,"Writing Secure Code: Chapter 6, ""Determining Appropriate Access Control"" Page 171 +Enforcement: the mechanism contains errors that prevent it from properly enforcing the specified access control requirements (e.g., allowing the user to specify their own privileges, or allowing a syntactically-incorrect ACL to produce insecure settings). This problem occurs within the program itself, in that it does not actually enforce the intended security policy that the administrator specifies.",,unclassified,"Writing Secure Code: Chapter 6, ""Determining Appropriate Access Control"" Page 171 24 Deadly Sins of Software Security: ""Sin 17: Failure to Protect Stored Data."" Page 253" CWE-286,EN-Incorrect User Management (Type: Class),"The software does not properly manage a user within its environment. -Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,Unknown, +Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,unclassified, CWE-288,EN-Authentication Bypass Using an Alternate Path or Channel (Type: Base),"A product requires authentication, but the product has an alternate path or channel that does not require authentication. -Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,Unknown, +Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,unclassified, CWE-289,EN-Authentication Bypass by Alternate Name (Type: Variant),"The software performs authentication based on the name of a resource being accessed, or the name of the actor performing the access, but it does not properly check all possible names for that resource or actor. -Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,Unknown, +Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,unclassified, CWE-29,EN-Path Traversal: \..\filename (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '\..\filename' (leading backslash dot dot) sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -This is similar to CWE-25, except using ""\"" instead of ""/"". Sometimes a program checks for ""..\"" at the beginning of the input, so a ""\..\"" can bypass that check. It is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,Unknown, +This is similar to CWE-25, except using ""\"" instead of ""/"". Sometimes a program checks for ""..\"" at the beginning of the input, so a ""\..\"" can bypass that check. It is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified, CWE-290,EN-Authentication Bypass by Spoofing (Type: Base),"This attack-focused weakness is caused by improperly implemented authentication schemes that are subject to spoofing attacks. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -This is similar to CWE-25, except using ""\"" instead of ""/"". Sometimes a program checks for ""..\"" at the beginning of the input, so a ""\..\"" can bypass that check. It is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,Unknown,"The Art of Software Security Assessment: Chapter 3, ""Spoofing and Identification"", Page 72." +This is similar to CWE-25, except using ""\"" instead of ""/"". Sometimes a program checks for ""..\"" at the beginning of the input, so a ""\..\"" can bypass that check. It is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified,"The Art of Software Security Assessment: Chapter 3, ""Spoofing and Identification"", Page 72." CWE-295,EN-Improper Certificate Validation (Type: Base),"The software does not validate, or incorrectly validates, a certificate. -When a certificate is invalid or malicious, it might allow an attacker to spoof a trusted entity by using a man-in-the-middle (MITM) attack. The software might connect to a malicious host while believing it is a trusted host, or the software might be deceived into accepting spoofed data that appears to originate from a trusted host.",,Unknown,"Why Eve and Mallory Love Android: An Analysis of Android SSL (In)Security: http://www2.dcsec.uni-hannover.de/files/android/p50-fahl.pdf +When a certificate is invalid or malicious, it might allow an attacker to spoof a trusted entity by using a man-in-the-middle (MITM) attack. The software might connect to a malicious host while believing it is a trusted host, or the software might be deceived into accepting spoofed data that appears to originate from a trusted host.",,unclassified,"Why Eve and Mallory Love Android: An Analysis of Android SSL (In)Security: http://www2.dcsec.uni-hannover.de/files/android/p50-fahl.pdf Computer Security: Art and Science" CWE-30,EN-Path Traversal: \dir\..\filename (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '\dir\..\filename' (leading backslash dot dot) sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -This is similar to CWE-26, except using ""\"" instead of ""/"". The '\dir\..\filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only checks for ""..\"" at the beginning of the input, so a ""\..\"" can bypass that check.",,Unknown, +This is similar to CWE-26, except using ""\"" instead of ""/"". The '\dir\..\filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only checks for ""..\"" at the beginning of the input, so a ""\..\"" can bypass that check.",,unclassified, CWE-300,EN-Channel Accessible by Non-Endpoint (Man-in-the-Middle) (Type: Class),"The product does not adequately verify the identity of actors at both ends of a communication channel, or does not adequately ensure the integrity of the channel, in a way that allows the channel to be accessed or influenced by an actor that is not an endpoint. -In order to establish secure communication between two parties, it is often important to adequately verify the identity of entities at each end of the communication channel. Inadequate or inconsistent verification may result in insufficient or incorrect identification of either communicating entity. This can have negative consequences such as misplaced trust in the entity at the other end of the channel. An attacker can leverage this by interposing between the communicating entities and masquerading as the original entity. In the absence of sufficient verification of identity, such an attacker can eavesdrop and potentially modify the communication between the original entities.",,Unknown,Computer Security: Art and Science +In order to establish secure communication between two parties, it is often important to adequately verify the identity of entities at each end of the communication channel. Inadequate or inconsistent verification may result in insufficient or incorrect identification of either communicating entity. This can have negative consequences such as misplaced trust in the entity at the other end of the channel. An attacker can leverage this by interposing between the communicating entities and masquerading as the original entity. In the absence of sufficient verification of identity, such an attacker can eavesdrop and potentially modify the communication between the original entities.",,unclassified,Computer Security: Art and Science CWE-302,EN-Authentication Bypass by Assumed-Immutable Data (Type: Variant),"The authentication scheme or implementation uses key data elements that are assumed to be immutable, but can be controlled or modified by the attacker. -A mutual authentication protocol requires each party to respond to a random challenge by the other party by encrypting it with a pre-shared key. Often, however, such protocols employ the same pre-shared key for communication with a number of different entities. A malicious user or an attacker can easily compromise this protocol without possessing the correct key by employing a reflection attack on the protocol.",,Unknown, +A mutual authentication protocol requires each party to respond to a random challenge by the other party by encrypting it with a pre-shared key. Often, however, such protocols employ the same pre-shared key for communication with a number of different entities. A malicious user or an attacker can easily compromise this protocol without possessing the correct key by employing a reflection attack on the protocol.",,unclassified, CWE-303,EN-Incorrect Implementation of Authentication Algorithm (Type: Base),"The requirements for the software dictate the use of an established authentication algorithm, but the implementation of the algorithm is incorrect. -This incorrect implementation may allow authentication to be bypassed.",,Unknown, +This incorrect implementation may allow authentication to be bypassed.",,unclassified, CWE-304,EN-Missing Critical Step in Authentication (Type: Base),"The software implements an authentication technique, but it skips a step that weakens the technique. -Authentication techniques should follow the algorithms that define them exactly, otherwise authentication can be bypassed or more easily subjected to brute force attacks.",,Unknown, +Authentication techniques should follow the algorithms that define them exactly, otherwise authentication can be bypassed or more easily subjected to brute force attacks.",,unclassified, CWE-305,EN-Authentication Bypass by Primary Weakness (Type: Base),"The authentication algorithm is sound, but the implemented mechanism can be bypassed as the result of a separate weakness that is primary to the authentication error. -Authentication techniques should follow the algorithms that define them exactly, otherwise authentication can be bypassed or more easily subjected to brute force attacks.",,Unknown, +Authentication techniques should follow the algorithms that define them exactly, otherwise authentication can be bypassed or more easily subjected to brute force attacks.",,unclassified, CWE-307,EN-Improper Restriction of Excessive Authentication Attempts (Type: Base),"The software does not implement sufficient measures to prevent multiple failed authentication attempts within in a short time frame, making it more susceptible to brute force attacks. -Authentication techniques should follow the algorithms that define them exactly, otherwise authentication can be bypassed or more easily subjected to brute force attacks.",,Unknown,"Weak Password Brings 'Happiness' to Twitter Hacker: http://www.wired.com/threatlevel/2009/01/professed-twitt/ +Authentication techniques should follow the algorithms that define them exactly, otherwise authentication can be bypassed or more easily subjected to brute force attacks.",,unclassified,"Weak Password Brings 'Happiness' to Twitter Hacker: http://www.wired.com/threatlevel/2009/01/professed-twitt/ OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI" CWE-31,EN-Path Traversal: dir\..\..\filename (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize 'dir\..\..\filename' (multiple internal backslash dot dot) sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The 'dir\..\..\filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only removes one ""..\"" sequence, so multiple ""..\"" can bypass that check. Alternately, this manipulation could be used to bypass a check for ""..\"" at the beginning of the pathname, moving up more than one directory level.",,Unknown,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" +The 'dir\..\..\filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only removes one ""..\"" sequence, so multiple ""..\"" can bypass that check. Alternately, this manipulation could be used to bypass a check for ""..\"" at the beginning of the pathname, moving up more than one directory level.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-312,EN-Cleartext Storage of Sensitive Information (Type: Base),"The application stores sensitive information in cleartext within a resource that might be accessible to another control sphere. -Because the information is stored in cleartext, attackers could potentially read it. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,Unknown,"Writing Secure Code: Chapter 9, ""Protecting Secret Data"" Page 299 +Because the information is stored in cleartext, attackers could potentially read it. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified,"Writing Secure Code: Chapter 9, ""Protecting Secret Data"" Page 299 The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Encryption"", Page 43. Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/" CWE-313,EN-Cleartext Storage in a File or on Disk (Type: Variant),"The application stores sensitive information in cleartext in a file, or on disk. -The sensitive information could be read by attackers with access to the file, or with physical or administrator access to the raw disk. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,Unknown, +The sensitive information could be read by attackers with access to the file, or with physical or administrator access to the raw disk. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified, CWE-314,EN-Cleartext Storage in the Registry (Type: Variant),"The application stores sensitive information in cleartext in the registry. -Attackers can read the information by accessing the registry key. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,Unknown, +Attackers can read the information by accessing the registry key. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified, CWE-315,EN-Cleartext Storage of Sensitive Information in a Cookie (Type: Variant),"The application stores sensitive information in cleartext in a cookie. -Attackers can use widely-available tools to view the cookie and read the sensitive information. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,Unknown, +Attackers can use widely-available tools to view the cookie and read the sensitive information. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified, CWE-316,EN-Cleartext Storage of Sensitive Information in Memory (Type: Variant),"The application stores sensitive information in cleartext in memory. The sensitive memory might be saved to disk, stored in a core dump, or remain uncleared if the application crashes, or if the programmer does not properly clear the memory before freeing it. -It could be argued that such problems are usually only exploitable by those with administrator privileges. However, swapping could cause the memory to be written to disk and leave it accessible to physical attack afterwards. Core dump files might have insecure permissions or be stored in archive files that are accessible to untrusted people. Or, uncleared sensitive memory might be inadvertently exposed to attackers due to another weakness.",,Unknown, +It could be argued that such problems are usually only exploitable by those with administrator privileges. However, swapping could cause the memory to be written to disk and leave it accessible to physical attack afterwards. Core dump files might have insecure permissions or be stored in archive files that are accessible to untrusted people. Or, uncleared sensitive memory might be inadvertently exposed to attackers due to another weakness.",,unclassified, CWE-317,EN-Cleartext Storage of Sensitive Information in GUI (Type: Variant),"The application stores sensitive information in cleartext within the GUI. -An attacker can often obtain data from a GUI, even if hidden, by using an API to directly access GUI objects such as windows and menus. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,Unknown, +An attacker can often obtain data from a GUI, even if hidden, by using an API to directly access GUI objects such as windows and menus. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified, CWE-318,EN-Cleartext Storage of Sensitive Information in Executable (Type: Variant),"The application stores sensitive information in cleartext in an executable. -Attackers can reverse engineer binary code to obtain secret data. This is especially easy when the cleartext is plain ASCII. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,Unknown, +Attackers can reverse engineer binary code to obtain secret data. This is especially easy when the cleartext is plain ASCII. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified, CWE-32,EN-Path Traversal: ... (Triple Dot) (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '...' (triple dot) sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '...' manipulation is useful for bypassing some path traversal protection schemes. On some Windows systems, it is equivalent to ""..\.."" and might bypass checks that assume only two dots are valid. Incomplete filtering, such as removal of ""./"" sequences, can ultimately produce valid "".."" sequences due to a collapse into unsafe value (CWE-182).",,Unknown, +The '...' manipulation is useful for bypassing some path traversal protection schemes. On some Windows systems, it is equivalent to ""..\.."" and might bypass checks that assume only two dots are valid. Incomplete filtering, such as removal of ""./"" sequences, can ultimately produce valid "".."" sequences due to a collapse into unsafe value (CWE-182).",,unclassified, CWE-325,EN-Missing Required Cryptographic Step (Type: Base),"The software does not implement a required step in a cryptographic algorithm, resulting in weaker encryption than advertised by that algorithm. -Cryptographic implementations should follow the algorithms that define them exactly, otherwise encryption can be weaker than expected.",,Unknown, +Cryptographic implementations should follow the algorithms that define them exactly, otherwise encryption can be weaker than expected.",,unclassified, CWE-326,EN-Inadequate Encryption Strength (Type: Class),"The software stores or transmits sensitive data using an encryption scheme that is theoretically sound, but is not strong enough for the level of protection required. -A weak encryption scheme can be subjected to brute force attacks that have a reasonable chance of succeeding using current attack methods and resources.",,Unknown,"Writing Secure Code: Chapter 8, ""Cryptographic Foibles"" Page 259 +A weak encryption scheme can be subjected to brute force attacks that have a reasonable chance of succeeding using current attack methods and resources.",,unclassified,"Writing Secure Code: Chapter 8, ""Cryptographic Foibles"" Page 259 24 Deadly Sins of Software Security: ""Sin 21: Using the Wrong Cryptography."" Page 315" CWE-328,EN-Reversible One-Way Hash (Type: Base),"The product uses a hashing algorithm that produces a hash value that can be used to determine the original input, or to find an input that can produce the same hash, more efficiently than brute force techniques. -This weakness is especially dangerous when the hash is used in security algorithms that require the one-way property to hold. For example, if an authentication system takes an incoming password and generates a hash, then compares the hash to another hash that it has stored in its authentication database, then the ability to create a collision could allow an attacker to provide an alternate password that produces the same target hash, bypassing authentication.",,Unknown,"MD5 considered harmful today: http://www.phreedom.org/research/rogue-ca/ +This weakness is especially dangerous when the hash is used in security algorithms that require the one-way property to hold. For example, if an authentication system takes an incoming password and generates a hash, then compares the hash to another hash that it has stored in its authentication database, then the ability to create a collision could allow an attacker to provide an alternate password that produces the same target hash, bypassing authentication.",,unclassified,"MD5 considered harmful today: http://www.phreedom.org/research/rogue-ca/ The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Integrity"", Page 47. RFC2898 - PKCS #5: Password-Based Cryptography Specification Version 2.0: http://tools.ietf.org/html/rfc2898 How To Safely Store A Password: http://codahale.com/how-to-safely-store-a-password/ @@ -1131,72 +1131,72 @@ Our password hashing has no clothes: http://www.troyhunt.com/2012/06/our-passwor Should we really use bcrypt/scrypt?: http://www.analyticalengine.net/2012/06/should-we-really-use-bcryptscrypt/" CWE-33,EN-Path Traversal: .... (Multiple Dot) (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '....' (multiple dot) sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '....' manipulation is useful for bypassing some path traversal protection schemes. On some Windows systems, it is equivalent to ""..\..\.."" and might bypass checks that assume only two dots are valid. Incomplete filtering, such as removal of ""./"" sequences, can ultimately produce valid "".."" sequences due to a collapse into unsafe value (CWE-182).",,Unknown, +The '....' manipulation is useful for bypassing some path traversal protection schemes. On some Windows systems, it is equivalent to ""..\..\.."" and might bypass checks that assume only two dots are valid. Incomplete filtering, such as removal of ""./"" sequences, can ultimately produce valid "".."" sequences due to a collapse into unsafe value (CWE-182).",,unclassified, CWE-331,EN-Insufficient Entropy (Type: Base),"The software uses an algorithm or scheme that produces insufficient entropy, leaving patterns or clusters of values that are more likely to occur than others. -When software generates predictable values in a context requiring unpredictability, it may be possible for an attacker to guess the next value that will be generated, and use this guess to impersonate another user or access sensitive information.",,Unknown,Building Secure Software: How to Avoid Security Problems the Right Way +When software generates predictable values in a context requiring unpredictability, it may be possible for an attacker to guess the next value that will be generated, and use this guess to impersonate another user or access sensitive information.",,unclassified,Building Secure Software: How to Avoid Security Problems the Right Way CWE-334,EN-Small Space of Random Values (Type: Base),"The number of possible random values is smaller than needed by the product, making it more susceptible to brute force attacks. -The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,Unknown,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf +The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,unclassified,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf 24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-335,EN-PRNG Seed Error (Type: Class),"A Pseudo-Random Number Generator (PRNG) uses seeds incorrectly. -The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,Unknown,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" +The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-336,EN-Same Seed in PRNG (Type: Base),"A PRNG uses the same seed each time the product is initialized. If an attacker can guess (or knows) the seed, then he/she may be able to determine the ""random"" number produced from the PRNG. -The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,Unknown,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf +The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,unclassified,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf CWE-337,EN-Predictable Seed in PRNG (Type: Base),"A PRNG is initialized from a predictable seed, e.g. using process ID or system time. -The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,Unknown,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf +The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,unclassified,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf 24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-339,EN-Small Seed Space in PRNG (Type: Base),"A PRNG uses a relatively small space of seeds. -The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,Unknown,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf +The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,unclassified,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf CWE-34,EN-Path Traversal: ....// (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '....//' (doubled dot dot slash) sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '....//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then ""....//"" can collapse into the ""../"" unsafe value (CWE-182). It could also be useful when "".."" is removed, if the operating system treats ""//"" and ""/"" as equivalent.",,Unknown, +The '....//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then ""....//"" can collapse into the ""../"" unsafe value (CWE-182). It could also be useful when "".."" is removed, if the operating system treats ""//"" and ""/"" as equivalent.",,unclassified, CWE-340,EN-Predictability Problems (Type: Class),"Weaknesses in this category are related to schemes that generate numbers or identifiers that are more predictable than required by the application. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '....//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then ""....//"" can collapse into the ""../"" unsafe value (CWE-182). It could also be useful when "".."" is removed, if the operating system treats ""//"" and ""/"" as equivalent.",,Unknown,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" +The '....//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then ""....//"" can collapse into the ""../"" unsafe value (CWE-182). It could also be useful when "".."" is removed, if the operating system treats ""//"" and ""/"" as equivalent.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-341,EN-Predictable from Observable State (Type: Base),"A number or object is predictable based on observations that the attacker can make about the state of the system or network, such as time, process ID, etc. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '....//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then ""....//"" can collapse into the ""../"" unsafe value (CWE-182). It could also be useful when "".."" is removed, if the operating system treats ""//"" and ""/"" as equivalent.",,Unknown,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf +The '....//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then ""....//"" can collapse into the ""../"" unsafe value (CWE-182). It could also be useful when "".."" is removed, if the operating system treats ""//"" and ""/"" as equivalent.",,unclassified,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf 24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-342,EN-Predictable Exact Value from Previous Values (Type: Base),"An exact value or random number can be precisely predicted by observing previous values. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '....//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then ""....//"" can collapse into the ""../"" unsafe value (CWE-182). It could also be useful when "".."" is removed, if the operating system treats ""//"" and ""/"" as equivalent.",,Unknown,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf +The '....//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then ""....//"" can collapse into the ""../"" unsafe value (CWE-182). It could also be useful when "".."" is removed, if the operating system treats ""//"" and ""/"" as equivalent.",,unclassified,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf 24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-343,EN-Predictable Value Range from Previous Values (Type: Base),"The software's random number generator produces a series of values which, when observed, can be used to infer a relatively small range of possibilities for the next value that could be generated. -The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,Unknown,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf +The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf Strange Attractors and TCP/IP Sequence Number Analysis: http://www.bindview.com/Services/Razor/Papers/2001/tcpseq.cfm 24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-344,EN-Use of Invariant Value in Dynamically Changing Context (Type: Base),"The product uses a constant value, name, or reference, but this value can (or should) vary across different environments. -The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,Unknown,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf +The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf CWE-345,EN-Insufficient Verification of Data Authenticity (Type: Class),"The software does not sufficiently verify the origin or authenticity of data, in a way that causes it to accept invalid data. -The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,Unknown,"24 Deadly Sins of Software Security: ""Sin 15: Not Updating Easily."" Page 231" +The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 15: Not Updating Easily."" Page 231" CWE-346,EN-Origin Validation Error (Type: Base),"The software does not properly verify that the source of data or communication is valid. -The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,Unknown, +The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified, CWE-347,EN-Improper Verification of Cryptographic Signature (Type: Base),"The software does not verify, or incorrectly verifies, the cryptographic signature for data. -The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,Unknown, +The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified, CWE-348,EN-Use of Less Trusted Source (Type: Base),"The software has two different sources of the same data or information, but it uses the source that has less support for verification, is less trusted, or is less resistant to attack. -The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,Unknown, +The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified, CWE-349,EN-Acceptance of Extraneous Untrusted Data With Trusted Data (Type: Base),"The software, when processing trusted data, accepts any untrusted data that is also included with the trusted data, treating the untrusted data as if it were trusted. -The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,Unknown, +The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified, CWE-35,EN-Path Traversal: .../...// (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '.../...//' (doubled triple dot slash) sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '.../...//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then "".../...//"" can collapse into the ""../"" unsafe value (CWE-182). Removing the first ""../"" yields ""....//""; the second removal yields ""../"". Depending on the algorithm, the software could be susceptible to CWE-34 but not CWE-35, or vice versa.",,Unknown, +The '.../...//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then "".../...//"" can collapse into the ""../"" unsafe value (CWE-182). Removing the first ""../"" yields ""....//""; the second removal yields ""../"". Depending on the algorithm, the software could be susceptible to CWE-34 but not CWE-35, or vice versa.",,unclassified, CWE-350,EN-Reliance on Reverse DNS Resolution for a Security-Critical Action (Type: Variant),"The software performs reverse DNS resolution on an IP address to obtain the hostname and make a security decision, but it does not properly ensure that the IP address is truly associated with the hostname. When the software performs a reverse DNS resolution for an IP address, if an attacker controls the server for that IP address, then the attacker can cause the server to return an arbitrary hostname. As a result, the attacker may be able to bypass authentication, cause the wrong hostname to be recorded in log files to hide activities, or perform other attacks. Attackers can spoof DNS names by either (1) compromising a DNS server and modifying its records (sometimes called DNS cache poisoning), or (2) having legitimate control over a DNS server associated with their IP address. -Since DNS names can be easily spoofed or misreported, and it may be difficult for the software to detect if a trusted DNS server has not been compromised, they do not constitute a valid authentication mechanism.",,Unknown,"24 Deadly Sins of Software Security: ""Sin 15: Not Updating Easily."" Page 231 +Since DNS names can be easily spoofed or misreported, and it may be difficult for the software to detect if a trusted DNS server has not been compromised, they do not constitute a valid authentication mechanism.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 15: Not Updating Easily."" Page 231 24 Deadly Sins of Software Security: ""Sin 24: Trusting Network Name Resolution."" Page 361 The Art of Software Security Assessment: Chapter 16, ""DNS Spoofing"", Page 1002." CWE-351,EN-Insufficient Type Distinction (Type: Base),"The software does not properly distinguish between different types of elements in a way that leads to insecure behavior. When the software performs a reverse DNS resolution for an IP address, if an attacker controls the server for that IP address, then the attacker can cause the server to return an arbitrary hostname. As a result, the attacker may be able to bypass authentication, cause the wrong hostname to be recorded in log files to hide activities, or perform other attacks. Attackers can spoof DNS names by either (1) compromising a DNS server and modifying its records (sometimes called DNS cache poisoning), or (2) having legitimate control over a DNS server associated with their IP address. -Since DNS names can be easily spoofed or misreported, and it may be difficult for the software to detect if a trusted DNS server has not been compromised, they do not constitute a valid authentication mechanism.",,Unknown, +Since DNS names can be easily spoofed or misreported, and it may be difficult for the software to detect if a trusted DNS server has not been compromised, they do not constitute a valid authentication mechanism.",,unclassified, CWE-356,EN-Product UI does not Warn User of Unsafe Actions (Type: Base),"The software's user interface does not warn the user before undertaking an unsafe action on behalf of that user. This makes it easier for attackers to trick users into inflicting damage to their system. -Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,Unknown, +Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,unclassified, CWE-357,EN-Insufficient UI Warning of Dangerous Operations (Type: Base),"The user interface provides a warning to a user regarding dangerous or sensitive operations, but the warning is not noticeable enough to warrant attention. -Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,Unknown, +Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,unclassified, CWE-358,EN-Improperly Implemented Security Check for Standard (Type: Base),"The software does not implement or incorrectly implements one or more security-relevant checks as specified by the design of a standardized algorithm, protocol, or technique. -Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,Unknown, +Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,unclassified, CWE-359,EN-Privacy Violation (Type: Class),"Mishandling private information, such as customer passwords or social security numbers, can compromise user privacy and is often illegal. -Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,Unknown,"AOL man pleads guilty to selling 92m email addies: http://www.theregister.co.uk/2005/02/07/aol_email_theft/ +Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,unclassified,"AOL man pleads guilty to selling 92m email addies: http://www.theregister.co.uk/2005/02/07/aol_email_theft/ Safe Harbor Privacy Framework: http://www.export.gov/safeharbor/ Financial Privacy: The Gramm-Leach Bliley Act (GLBA): http://www.ftc.gov/privacy/glbact/index.html Health Insurance Portability and Accountability Act (HIPAA): http://www.hhs.gov/ocr/hipaa/ @@ -1204,111 +1204,111 @@ California SB-1386: http://info.sen.ca.gov/pub/01-02/bill/sen/sb_1351-1400/sb_13 SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/" CWE-36,EN-Absolute Path Traversal (Type: Base),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize absolute path sequences such as ""/abs/path"" that can resolve to a location that is outside of that directory. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,Unknown,"The Art of Software Security Assessment: Chapter 9, ""Filenames and Paths"", Page 503." +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified,"The Art of Software Security Assessment: Chapter 9, ""Filenames and Paths"", Page 503." CWE-363,EN-Race Condition Enabling Link Following (Type: Base),"The software checks the status of a file or directory before accessing it, which produces a race condition in which the file can be replaced with a link before the access is performed, causing the software to access the wrong file. -While developers might expect that there is a very narrow time window between the time of check and time of use, there is still a race condition. An attacker could cause the software to slow down (e.g. with memory consumption), causing the time window to become larger. Alternately, in some situations, the attacker could win the race by performing a large number of attacks.",,Unknown,"The Art of Software Security Assessment: Chapter 9, ""Race Conditions"", Page 526." +While developers might expect that there is a very narrow time window between the time of check and time of use, there is still a race condition. An attacker could cause the software to slow down (e.g. with memory consumption), causing the time window to become larger. Alternately, in some situations, the attacker could win the race by performing a large number of attacks.",,unclassified,"The Art of Software Security Assessment: Chapter 9, ""Race Conditions"", Page 526." CWE-368,EN-Context Switching Race Condition (Type: Base),"A product performs a series of non-atomic actions to switch between contexts that cross privilege or other security boundaries, but a race condition allows an attacker to modify or misrepresent the product's behavior during the switch. -This is commonly seen in web browser vulnerabilities in which the attacker can perform certain actions while the browser is transitioning from a trusted to an untrusted domain, or vice versa, and the browser performs the actions on one domain using the trust level and resources of the other domain.",,Unknown,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205" +This is commonly seen in web browser vulnerabilities in which the attacker can perform certain actions while the browser is transitioning from a trusted to an untrusted domain, or vice versa, and the browser performs the actions on one domain using the trust level and resources of the other domain.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205" CWE-37,EN-Path Traversal: /absolute/pathname/here (Type: Variant),"A software system that accepts input in the form of a slash absolute path ('/absolute/pathname/here') without appropriate validation can allow an attacker to traverse the file system to unintended locations or access arbitrary files. -This weakness typically occurs when an unexpected value is provided to the product, or if an error occurs that is not properly detected. It frequently occurs in calculations involving physical dimensions such as size, length, width, and height.",,Unknown, +This weakness typically occurs when an unexpected value is provided to the product, or if an error occurs that is not properly detected. It frequently occurs in calculations involving physical dimensions such as size, length, width, and height.",,unclassified, CWE-372,EN-Incomplete Internal State Distinction (Type: Base),"The software does not properly determine which state it is in, causing it to assume it is in state X when in fact it is in state Y, causing it to perform incorrect operations in a security-relevant manner. -If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,Unknown, +If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,unclassified, CWE-373,EN-DEPRECATED: State Synchronization Error (Type: Base),"This entry was deprecated because it overlapped the same concepts as race condition (CWE-362) and Improper Synchronization (CWE-662). -If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,Unknown, +If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,unclassified, CWE-377,EN-Insecure Temporary File (Type: Base),"Creating and using insecure temporary files can leave application and system data vulnerable to attack. -If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,Unknown,"Writing Secure Code: Chapter 23, ""Creating Temporary Files Securely"" Page 682 +If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,unclassified,"Writing Secure Code: Chapter 23, ""Creating Temporary Files Securely"" Page 682 The Art of Software Security Assessment: Chapter 9, ""Temporary Files"", Page 538. The Art of Software Security Assessment: Chapter 11, ""File Squatting"", Page 662." CWE-38,EN-Path Traversal: \absolute\pathname\here (Type: Variant),"A software system that accepts input in the form of a backslash absolute path ('\absolute\pathname\here') without appropriate validation can allow an attacker to traverse the file system to unintended locations or access arbitrary files. -On some operating systems, the fact that the temporary file exists may be apparent to any user with sufficient privileges to access that directory. Since the file is visible, the application that is using the temporary file could be known. If one has access to list the processes on the system, the attacker has gained information about what the user is doing at that time. By correlating this with the applications the user is running, an attacker could potentially discover what a user's actions are. From this, higher levels of security could be breached.",,Unknown, +On some operating systems, the fact that the temporary file exists may be apparent to any user with sufficient privileges to access that directory. Since the file is visible, the application that is using the temporary file could be known. If one has access to list the processes on the system, the attacker has gained information about what the user is doing at that time. By correlating this with the applications the user is running, an attacker could potentially discover what a user's actions are. From this, higher levels of security could be breached.",,unclassified, CWE-382,EN-J2EE Bad Practices: Use of System.exit() (Type: Variant),"A J2EE application uses System.exit(), which also shuts down its container. -On some operating systems, the fact that the temporary file exists may be apparent to any user with sufficient privileges to access that directory. Since the file is visible, the application that is using the temporary file could be known. If one has access to list the processes on the system, the attacker has gained information about what the user is doing at that time. By correlating this with the applications the user is running, an attacker could potentially discover what a user's actions are. From this, higher levels of security could be breached.",,Unknown, +On some operating systems, the fact that the temporary file exists may be apparent to any user with sufficient privileges to access that directory. Since the file is visible, the application that is using the temporary file could be known. If one has access to list the processes on the system, the attacker has gained information about what the user is doing at that time. By correlating this with the applications the user is running, an attacker could potentially discover what a user's actions are. From this, higher levels of security could be breached.",,unclassified, CWE-383,EN-J2EE Bad Practices: Direct Use of Threads (Type: Variant),"Thread management in a Web application is forbidden in some circumstances and is always highly error prone. -Thread management in a web application is forbidden by the J2EE standard in some circumstances and is always highly error prone. Managing threads is difficult and is likely to interfere in unpredictable ways with the behavior of the application container. Even without interfering with the container, thread management usually leads to bugs that are hard to detect and diagnose like deadlock, race conditions, and other synchronization errors.",,Unknown, +Thread management in a web application is forbidden by the J2EE standard in some circumstances and is always highly error prone. Managing threads is difficult and is likely to interfere in unpredictable ways with the behavior of the application container. Even without interfering with the container, thread management usually leads to bugs that are hard to detect and diagnose like deadlock, race conditions, and other synchronization errors.",,unclassified, CWE-386,EN-Symbolic Name not Mapping to Correct Object (Type: Base),"A constant symbolic reference to an object is used, even though the reference can resolve to a different object over time. In some instances, knowing when data is transmitted between parties can provide a malicious user with privileged information. Also, externally monitoring the timing of operations can potentially reveal sensitive data. For example, a cryptographic operation can expose its internal state if the time it takes to perform the operation varies, based on the state. -Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,Unknown, +Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,unclassified, CWE-39,EN-Path Traversal: C:dirname (Type: Variant),"An attacker can inject a drive letter or Windows volume letter ('C:dirname') into a software system to potentially redirect access to an unintended location or arbitrary file. In some instances, knowing when data is transmitted between parties can provide a malicious user with privileged information. Also, externally monitoring the timing of operations can potentially reveal sensitive data. For example, a cryptographic operation can expose its internal state if the time it takes to perform the operation varies, based on the state. -Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,Unknown, +Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,unclassified, CWE-392,EN-Missing Report of Error Condition (Type: Base),"The software encounters an error but does not provide a status code or return value to indicate that an error has occurred. In some instances, knowing when data is transmitted between parties can provide a malicious user with privileged information. Also, externally monitoring the timing of operations can potentially reveal sensitive data. For example, a cryptographic operation can expose its internal state if the time it takes to perform the operation varies, based on the state. -Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,Unknown, +Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,unclassified, CWE-393,EN-Return of Wrong Status Code (Type: Base),"A function or operation returns an incorrect return value or status code that does not indicate an error, but causes the product to modify its behavior based on the incorrect result. -This can lead to unpredictable behavior. If the function is used to make security-critical decisions or provide security-critical information, then the wrong status code can cause the software to assume that an action is safe, even when it is not.",,Unknown, +This can lead to unpredictable behavior. If the function is used to make security-critical decisions or provide security-critical information, then the wrong status code can cause the software to assume that an action is safe, even when it is not.",,unclassified, CWE-394,EN-Unexpected Status Code or Return Value (Type: Base),"The software does not properly check when a function or operation returns a value that is legitimate for the function, but is not expected by the software. -This can lead to unpredictable behavior. If the function is used to make security-critical decisions or provide security-critical information, then the wrong status code can cause the software to assume that an action is safe, even when it is not.",,Unknown, +This can lead to unpredictable behavior. If the function is used to make security-critical decisions or provide security-critical information, then the wrong status code can cause the software to assume that an action is safe, even when it is not.",,unclassified, CWE-395,EN-Use of NullPointerException Catch to Detect NULL Pointer Dereference (Type: Base),"Catching NullPointerException should not be used as an alternative to programmatic checks to prevent dereferencing a null pointer. -This can lead to unpredictable behavior. If the function is used to make security-critical decisions or provide security-critical information, then the wrong status code can cause the software to assume that an action is safe, even when it is not.",,Unknown, +This can lead to unpredictable behavior. If the function is used to make security-critical decisions or provide security-critical information, then the wrong status code can cause the software to assume that an action is safe, even when it is not.",,unclassified, CWE-396,EN-Declaration of Catch for Generic Exception (Type: Base),"Catching overly broad exceptions promotes complex error handling code that is more likely to contain security vulnerabilities. -Multiple catch blocks can get ugly and repetitive, but ""condensing"" catch blocks by catching a high-level class like Exception can obscure exceptions that deserve special treatment or that should not be caught at this point in the program. Catching an overly broad exception essentially defeats the purpose of Java's typed exceptions, and can become particularly dangerous if the program grows and begins to throw new types of exceptions. The new exception types will not receive any attention.",,Unknown,"24 Deadly Sins of Software Security: ""Sin 9: Catching Exceptions."" Page 157" +Multiple catch blocks can get ugly and repetitive, but ""condensing"" catch blocks by catching a high-level class like Exception can obscure exceptions that deserve special treatment or that should not be caught at this point in the program. Catching an overly broad exception essentially defeats the purpose of Java's typed exceptions, and can become particularly dangerous if the program grows and begins to throw new types of exceptions. The new exception types will not receive any attention.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 9: Catching Exceptions."" Page 157" CWE-397,EN-Declaration of Throws for Generic Exception (Type: Base),"Throwing overly broad exceptions promotes complex error handling code that is more likely to contain security vulnerabilities. -Declaring a method to throw Exception or Throwable makes it difficult for callers to perform proper error handling and error recovery. Java's exception mechanism, for example, is set up to make it easy for callers to anticipate what can go wrong and write code to handle each specific exceptional circumstance. Declaring that a method throws a generic form of exception defeats this system.",,Unknown, +Declaring a method to throw Exception or Throwable makes it difficult for callers to perform proper error handling and error recovery. Java's exception mechanism, for example, is set up to make it easy for callers to anticipate what can go wrong and write code to handle each specific exceptional circumstance. Declaring that a method throws a generic form of exception defeats this system.",,unclassified, CWE-398,EN-Indicator of Poor Code Quality (Type: Class),"The code has features that do not directly introduce a weakness or vulnerability, but indicate that the product has not been carefully developed or maintained. -Programs are more likely to be secure when good development practices are followed. If a program is complex, difficult to maintain, not portable, or shows evidence of neglect, then there is a higher likelihood that weaknesses are buried in the code.",,Unknown, +Programs are more likely to be secure when good development practices are followed. If a program is complex, difficult to maintain, not portable, or shows evidence of neglect, then there is a higher likelihood that weaknesses are buried in the code.",,unclassified, CWE-40,EN-Path Traversal: \\UNC\share\name\ (Windows UNC Share) (Type: Variant),"An attacker can inject a Windows UNC share ('\\UNC\share\name') into a software system to potentially redirect access to an unintended location or arbitrary file. -Programs are more likely to be secure when good development practices are followed. If a program is complex, difficult to maintain, not portable, or shows evidence of neglect, then there is a higher likelihood that weaknesses are buried in the code.",,Unknown,"The Art of Software Security Assessment: Chapter 11, ""Filelike Objects"", Page 664." +Programs are more likely to be secure when good development practices are followed. If a program is complex, difficult to maintain, not portable, or shows evidence of neglect, then there is a higher likelihood that weaknesses are buried in the code.",,unclassified,"The Art of Software Security Assessment: Chapter 11, ""Filelike Objects"", Page 664." CWE-402,EN-Transmission of Private Resources into a New Sphere (Resource Leak) (Type: Class),"The software makes resources available to untrusted parties when those resources are only intended to be accessed by the software. -This is often triggered by improper handling of malformed data or unexpectedly interrupted sessions.",,Unknown, +This is often triggered by improper handling of malformed data or unexpectedly interrupted sessions.",,unclassified, CWE-403,EN-Exposure of File Descriptor to Unintended Control Sphere (File Descriptor Leak) (Type: Base),"A process does not close sensitive file descriptors before invoking a child process, which allows the child to perform unauthorized I/O operations using those descriptors. -When a new process is forked or executed, the child process inherits any open file descriptors. When the child process has fewer privileges than the parent process, this might introduce a vulnerability if the child process can access the file descriptor but does not have the privileges to access the associated file.",,Unknown,"File descriptors and setuid applications: https://blogs.oracle.com/paulr/entry/file_descriptors_and_setuid_applications +When a new process is forked or executed, the child process inherits any open file descriptors. When the child process has fewer privileges than the parent process, this might introduce a vulnerability if the child process can access the file descriptor but does not have the privileges to access the associated file.",,unclassified,"File descriptors and setuid applications: https://blogs.oracle.com/paulr/entry/file_descriptors_and_setuid_applications Introduction to Secure Coding Guide: https://developer.apple.com/library/mac/#documentation/security/conceptual/SecureCodingGuide/Articles/AccessControl.html" CWE-405,EN-Asymmetric Resource Consumption (Amplification) (Type: Class),"Software that does not appropriately monitor or control resource consumption can lead to adverse system performance. -This situation is amplified if the software allows malicious users or attackers to consume more resources than their access level permits. Exploiting such a weakness can lead to asymmetric resource consumption, aiding in amplification attacks against the system or the network.",,Unknown, +This situation is amplified if the software allows malicious users or attackers to consume more resources than their access level permits. Exploiting such a weakness can lead to asymmetric resource consumption, aiding in amplification attacks against the system or the network.",,unclassified, CWE-406,EN-Insufficient Control of Network Message Volume (Network Amplification) (Type: Base),"The software does not sufficiently monitor or control transmitted network traffic volume, so that an actor can cause the software to transmit more traffic than should be allowed for that actor. -In the absence of a policy to restrict asymmetric resource consumption, the application or system cannot distinguish between legitimate transmissions and traffic intended to serve as an amplifying attack on target systems. Systems can often be configured to restrict the amount of traffic sent out on behalf of a client, based on the client's origin or access level. This is usually defined in a resource allocation policy. In the absence of a mechanism to keep track of transmissions, the system or application can be easily abused to transmit asymmetrically greater traffic than the request or client should be permitted to.",,Unknown, +In the absence of a policy to restrict asymmetric resource consumption, the application or system cannot distinguish between legitimate transmissions and traffic intended to serve as an amplifying attack on target systems. Systems can often be configured to restrict the amount of traffic sent out on behalf of a client, based on the client's origin or access level. This is usually defined in a resource allocation policy. In the absence of a mechanism to keep track of transmissions, the system or application can be easily abused to transmit asymmetrically greater traffic than the request or client should be permitted to.",,unclassified, CWE-408,EN-Incorrect Behavior Order: Early Amplification (Type: Base),"The software allows an entity to perform a legitimate but expensive operation before authentication or authorization has taken place. -In the absence of a policy to restrict asymmetric resource consumption, the application or system cannot distinguish between legitimate transmissions and traffic intended to serve as an amplifying attack on target systems. Systems can often be configured to restrict the amount of traffic sent out on behalf of a client, based on the client's origin or access level. This is usually defined in a resource allocation policy. In the absence of a mechanism to keep track of transmissions, the system or application can be easily abused to transmit asymmetrically greater traffic than the request or client should be permitted to.",,Unknown, +In the absence of a policy to restrict asymmetric resource consumption, the application or system cannot distinguish between legitimate transmissions and traffic intended to serve as an amplifying attack on target systems. Systems can often be configured to restrict the amount of traffic sent out on behalf of a client, based on the client's origin or access level. This is usually defined in a resource allocation policy. In the absence of a mechanism to keep track of transmissions, the system or application can be easily abused to transmit asymmetrically greater traffic than the request or client should be permitted to.",,unclassified, CWE-409,EN-Improper Handling of Highly Compressed Data (Data Amplification) (Type: Base),"The software does not handle or incorrectly handles a compressed input with a very high compression ratio that produces a large output. -An example of data amplification is a ""decompression bomb,"" a small ZIP file that can produce a large amount of data when it is decompressed.",,Unknown, +An example of data amplification is a ""decompression bomb,"" a small ZIP file that can produce a large amount of data when it is decompressed.",,unclassified, CWE-41,EN-Improper Resolution of Path Equivalence (Type: Base),"The system or application is vulnerable to file system contents disclosure through path equivalence. Path equivalence involves the use of special characters in file and directory names. The associated manipulations are intended to generate multiple names for the same object. -Path equivalence is usually employed in order to circumvent access controls expressed using an incomplete set of file name or file path representations. This is different from path traversal, wherein the manipulations are performed to generate a name for a different object.",,Unknown, +Path equivalence is usually employed in order to circumvent access controls expressed using an incomplete set of file name or file path representations. This is different from path traversal, wherein the manipulations are performed to generate a name for a different object.",,unclassified, CWE-410,EN-Insufficient Resource Pool (Type: Base),"The software's resource pool is not large enough to handle peak demand, which allows an attacker to prevent others from accessing the resource by using a (relatively) large number of requests for resources. -Frequently the consequence is a ""flood"" of connection or sessions.",,Unknown,"Writing Secure Code: Chapter 17, ""Protecting Against Denial of Service Attacks"" Page 517" +Frequently the consequence is a ""flood"" of connection or sessions.",,unclassified,"Writing Secure Code: Chapter 17, ""Protecting Against Denial of Service Attacks"" Page 517" CWE-412,EN-Unrestricted Externally Accessible Lock (Type: Base),"The software properly checks for the existence of a lock, but the lock can be externally controlled or influenced by an actor that is outside of the intended sphere of control. -This prevents the software from acting on associated resources or performing other behaviors that are controlled by the presence of the lock. Relevant locks might include an exclusive lock or mutex, or modifying a shared resource that is treated as a lock. If the lock can be held for an indefinite period of time, then the denial of service could be permanent.",,Unknown, +This prevents the software from acting on associated resources or performing other behaviors that are controlled by the presence of the lock. Relevant locks might include an exclusive lock or mutex, or modifying a shared resource that is treated as a lock. If the lock can be held for an indefinite period of time, then the denial of service could be permanent.",,unclassified, CWE-413,EN-Improper Resource Locking (Type: Base),"The software does not lock or does not correctly lock a resource when the software must have exclusive access to the resource. -When a resource is not properly locked, an attacker could modify the resource while it is being operated on by the software. This might violate the software's assumption that the resource will not change, potentially leading to unexpected behaviors.",,Unknown, +When a resource is not properly locked, an attacker could modify the resource while it is being operated on by the software. This might violate the software's assumption that the resource will not change, potentially leading to unexpected behaviors.",,unclassified, CWE-414,EN-Missing Lock Check (Type: Base),"A product does not check to see if a lock is present before performing sensitive operations on a resource. -When a resource is not properly locked, an attacker could modify the resource while it is being operated on by the software. This might violate the software's assumption that the resource will not change, potentially leading to unexpected behaviors.",,Unknown, +When a resource is not properly locked, an attacker could modify the resource while it is being operated on by the software. This might violate the software's assumption that the resource will not change, potentially leading to unexpected behaviors.",,unclassified, CWE-419,EN-Unprotected Primary Channel (Type: Base),"The software uses a primary channel for administration or restricted functionality, but it does not properly protect the channel. The use of previously-freed memory can have any number of adverse consequences, ranging from the corruption of valid data to the execution of arbitrary code, depending on the instantiation and timing of the flaw. The simplest way data corruption may occur involves the system's reuse of the freed memory. Use-after-free errors have two common and sometimes overlapping causes: Error conditions and other exceptional circumstances. Confusion over which part of the program is responsible for freeing the memory. In this scenario, the memory in question is allocated to another pointer validly at some point after it has been freed. The original pointer to the freed memory is used again and points to somewhere within the new allocation. As the data is changed, it corrupts the validly used memory; this induces undefined behavior in the process. -If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,Unknown, +If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,unclassified, CWE-42,EN-Path Equivalence: filename. (Trailing Dot) (Type: Variant),"A software system that accepts path input in the form of trailing dot ('filedir.') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. The use of previously-freed memory can have any number of adverse consequences, ranging from the corruption of valid data to the execution of arbitrary code, depending on the instantiation and timing of the flaw. The simplest way data corruption may occur involves the system's reuse of the freed memory. Use-after-free errors have two common and sometimes overlapping causes: Error conditions and other exceptional circumstances. Confusion over which part of the program is responsible for freeing the memory. In this scenario, the memory in question is allocated to another pointer validly at some point after it has been freed. The original pointer to the freed memory is used again and points to somewhere within the new allocation. As the data is changed, it corrupts the validly used memory; this induces undefined behavior in the process. -If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,Unknown, +If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,unclassified, CWE-420,EN-Unprotected Alternate Channel (Type: Base),"The software protects a primary channel, but it does not use the same level of protection for an alternate channel. The use of previously-freed memory can have any number of adverse consequences, ranging from the corruption of valid data to the execution of arbitrary code, depending on the instantiation and timing of the flaw. The simplest way data corruption may occur involves the system's reuse of the freed memory. Use-after-free errors have two common and sometimes overlapping causes: Error conditions and other exceptional circumstances. Confusion over which part of the program is responsible for freeing the memory. In this scenario, the memory in question is allocated to another pointer validly at some point after it has been freed. The original pointer to the freed memory is used again and points to somewhere within the new allocation. As the data is changed, it corrupts the validly used memory; this induces undefined behavior in the process. -If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,Unknown, +If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,unclassified, CWE-421,EN-Race Condition During Access to Alternate Channel (Type: Base),"The product opens an alternate channel to communicate with an authorized user, but the channel is accessible to other actors. -This creates a race condition that allows an attacker to access the channel before the authorized user does.",,Unknown,"Discovering and Exploiting Named Pipe Security Flaws for Fun and Profit: http://www.blakewatts.com/namedpipepaper.html +This creates a race condition that allows an attacker to access the channel before the authorized user does.",,unclassified,"Discovering and Exploiting Named Pipe Security Flaws for Fun and Profit: http://www.blakewatts.com/namedpipepaper.html 24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205" CWE-422,EN-Unprotected Windows Messaging Channel (Shatter) (Type: Variant),"The software does not properly verify the source of a message in the Windows Messaging System while running at elevated privileges, creating an alternate channel through which an attacker can directly send a message to the product. -This creates a race condition that allows an attacker to access the channel before the authorized user does.",,Unknown,"Exploiting design flaws in the Win32 API for privilege escalation. Or... Shatter Attacks - How to break Windows: http://web.archive.org/web/20060115174629/http://security.tombom.co.uk/shatter.html +This creates a race condition that allows an attacker to access the channel before the authorized user does.",,unclassified,"Exploiting design flaws in the Win32 API for privilege escalation. Or... Shatter Attacks - How to break Windows: http://web.archive.org/web/20060115174629/http://security.tombom.co.uk/shatter.html The Art of Software Security Assessment: Chapter 2, ""Design Review."" Page 34. The Art of Software Security Assessment: Chapter 12, ""Shatter Attacks"", Page 694." CWE-423,EN-DEPRECATED (Duplicate): Proxied Trusted Channel (Type: Base),"This entry has been deprecated because it was a duplicate of CWE-441. All content has been transferred to CWE-441. -This creates a race condition that allows an attacker to access the channel before the authorized user does.",,Unknown, +This creates a race condition that allows an attacker to access the channel before the authorized user does.",,unclassified, CWE-424,EN-Improper Protection of Alternate Path (Type: Class),"The product does not sufficiently protect all possible paths that a user can take to access restricted functionality or resources. -This creates a race condition that allows an attacker to access the channel before the authorized user does.",,Unknown, +This creates a race condition that allows an attacker to access the channel before the authorized user does.",,unclassified, CWE-425,EN-Direct Request (Forced Browsing) (Type: Base),"The web application does not adequately enforce appropriate authorization on all restricted URLs, scripts, or files. -Web applications susceptible to direct request attacks often make the false assumption that such resources can only be reached through a given navigation path and so only apply authorization at certain points in the path.",,Unknown, +Web applications susceptible to direct request attacks often make the false assumption that such resources can only be reached through a given navigation path and so only apply authorization at certain points in the path.",,unclassified, CWE-427,EN-Uncontrolled Search Path Element (Type: Base),"The product uses a fixed or controlled search path to find resources, but one or more locations in that path can be under the control of unintended actors. Although this weakness can occur with any type of resource, it is frequently introduced when a product uses a directory search path to find executables or code libraries, but the path contains a directory that can be modified by an attacker, such as ""/tmp"" or the current working directory. In Windows-based systems, when the LoadLibrary or LoadLibraryEx function is called with a DLL name that does not contain a fully qualified path, the function follows a search order that includes two path elements that might be uncontrolled: the directory from which the program has been loaded the current working directory. In some cases, the attack can be conducted remotely, such as when SMB or WebDAV network shares are used. -In some Unix-based systems, a PATH might be created that contains an empty element, e.g. by splicing an empty variable into the PATH. This empty element can be interpreted as equivalent to the current working directory, which might be an untrusted search element.",,Unknown,"Double clicking on MS Office documents from Windows Explorer may execute arbitrary programs in some cases +In some Unix-based systems, a PATH might be created that contains an empty element, e.g. by splicing an empty variable into the PATH. This empty element can be interpreted as equivalent to the current working directory, which might be an untrusted search element.",,unclassified,"Double clicking on MS Office documents from Windows Explorer may execute arbitrary programs in some cases ACROS Security: Remote Binary Planting in Apple iTunes for Windows (ASPR #2010-08-18-1) Automatic Detection of Vulnerable Dynamic Component Loadings: http://www.cs.ucdavis.edu/research/tech-reports/2010/CSE-2010-2.pdf Dynamic-Link Library Search Order: http://msdn.microsoft.com/en-us/library/ms682586%28v=VS.85%29.aspx @@ -1318,402 +1318,402 @@ Insecure Library Loading Could Allow Remote Code Execution: http://www.microsoft Application DLL Load Hijacking: http://blog.rapid7.com/?p=5325 DLL Hijacking: Facts and Fiction: http://threatpost.com/en_us/blogs/dll-hijacking-facts-and-fiction-082610" CWE-428,EN-Unquoted Search Path or Element (Type: Base),"The product uses a search path that contains an unquoted element, in which the element contains whitespace or other separators. This can cause the product to access resources in a parent path. -If a malicious individual has access to the file system, it is possible to elevate privileges by inserting such a file as ""C:\Program.exe"" to be run by a privileged program making use of WinExec.",,Unknown,"The Art of Software Security Assessment: Chapter 11, ""Process Loading"", Page 654." +If a malicious individual has access to the file system, it is possible to elevate privileges by inserting such a file as ""C:\Program.exe"" to be run by a privileged program making use of WinExec.",,unclassified,"The Art of Software Security Assessment: Chapter 11, ""Process Loading"", Page 654." CWE-43,EN-Path Equivalence: filename.... (Multiple Trailing Dot) (Type: Variant),"A software system that accepts path input in the form of multiple trailing dot ('filedir....') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -If a malicious individual has access to the file system, it is possible to elevate privileges by inserting such a file as ""C:\Program.exe"" to be run by a privileged program making use of WinExec.",,Unknown, +If a malicious individual has access to the file system, it is possible to elevate privileges by inserting such a file as ""C:\Program.exe"" to be run by a privileged program making use of WinExec.",,unclassified, CWE-430,EN-Deployment of Wrong Handler (Type: Base),"The wrong ""handler"" is assigned to process an object. -An example of deploying the wrong handler would be calling a servlet to reveal source code of a .JSP file, or automatically ""determining"" type of the object even if it is contradictory to an explicitly specified type.",,Unknown,"The Art of Software Security Assessment: Chapter 3, ""File Handlers"", Page 74." +An example of deploying the wrong handler would be calling a servlet to reveal source code of a .JSP file, or automatically ""determining"" type of the object even if it is contradictory to an explicitly specified type.",,unclassified,"The Art of Software Security Assessment: Chapter 3, ""File Handlers"", Page 74." CWE-431,EN-Missing Handler (Type: Base),"A handler is not available or implemented. -When an exception is thrown and not caught, the process has given up an opportunity to decide if a given failure or event is worth a change in execution.",,Unknown,"The Art of Software Security Assessment: Chapter 3, ""File Handlers"", Page 74." +When an exception is thrown and not caught, the process has given up an opportunity to decide if a given failure or event is worth a change in execution.",,unclassified,"The Art of Software Security Assessment: Chapter 3, ""File Handlers"", Page 74." CWE-432,EN-Dangerous Signal Handler not Disabled During Sensitive Operations (Type: Base),"The application uses a signal handler that shares state with other signal handlers, but it does not properly mask or prevent those signal handlers from being invoked while the original signal handler is still running. -During the execution of a signal handler, it can be interrupted by another handler when a different signal is sent. If the two handlers share state - such as global variables - then an attacker can corrupt the state by sending another signal before the first handler has completed execution.",,Unknown, +During the execution of a signal handler, it can be interrupted by another handler when a different signal is sent. If the two handlers share state - such as global variables - then an attacker can corrupt the state by sending another signal before the first handler has completed execution.",,unclassified, CWE-433,EN-Unparsed Raw Web Content Delivery (Type: Variant),"The software stores raw content or supporting code under the web document root with an extension that is not specifically handled by the server. -If code is stored in a file with an extension such as "".inc"" or "".pl"", and the web server does not have a handler for that extension, then the server will likely send the contents of the file directly to the requester without the pre-processing that was expected. When that file contains sensitive information such as database credentials, this may allow the attacker to compromise the application or associated components.",,Unknown,"The Art of Software Security Assessment: Chapter 3, ""File Handlers"", Page 74." +If code is stored in a file with an extension such as "".inc"" or "".pl"", and the web server does not have a handler for that extension, then the server will likely send the contents of the file directly to the requester without the pre-processing that was expected. When that file contains sensitive information such as database credentials, this may allow the attacker to compromise the application or associated components.",,unclassified,"The Art of Software Security Assessment: Chapter 3, ""File Handlers"", Page 74." CWE-435,EN-Interaction Error (Type: Class),"An interaction error occurs when two entities work correctly when running independently, but they interact in unexpected ways when they are run together. -This could apply to products, systems, components, etc.",,Unknown, +This could apply to products, systems, components, etc.",,unclassified, CWE-436,EN-Interpretation Conflict (Type: Base),"Product A handles inputs or steps differently than Product B, which causes A to perform incorrect actions based on its perception of B's state. -This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,Unknown,"On Interpretation Conflict Vulnerabilities +This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified,"On Interpretation Conflict Vulnerabilities Insertion, Evasion, and Denial of Service: Eluding Network Intrusion Detection: http://www.insecure.org/stf/secnet_ids/secnet_ids.pdf 0x00 vs ASP file upload scripts: http://www.security-assessment.com/Whitepapers/0x00_vs_ASP_File_Uploads.pdf Poison NULL byte Re: Corsaire Security Advisory - Multiple vendor MIME RFC2047 encoding: http://marc.theaimsgroup.com/?l=bugtraq&m=109525864717484&w=2" CWE-437,EN-Incomplete Model of Endpoint Features (Type: Base),"A product acts as an intermediary or monitor between two or more endpoints, but it does not have a complete model of an endpoint's features, behaviors, or state, potentially causing the product to perform incorrect actions based on this incomplete model. -This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,Unknown, +This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified, CWE-439,EN-Behavioral Change in New Version or Environment (Type: Base),"A's behavior or functionality changes with a new version of A, or a new environment, which is not known (or manageable) by B. -This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,Unknown, +This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified, CWE-44,EN-Path Equivalence: file.name (Internal Dot) (Type: Variant),"A software system that accepts path input in the form of internal dot ('file.ordir') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,Unknown, +This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified, CWE-440,EN-Expected Behavior Violation (Type: Base),"A feature, API, or function being used by a product behaves differently than the product expects. -This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,Unknown, +This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified, CWE-441,EN-Unintended Proxy or Intermediary (Confused Deputy) (Type: Class),"The software receives a request, message, or directive from an upstream component, but the software does not sufficiently preserve the original source of the request before forwarding the request to an external actor that is outside of the software's control sphere. This causes the software to appear to be the source of the request, leading it to act as a proxy or other intermediary between the upstream component and the external actor. If an attacker cannot directly contact a target, but the software has access to the target, then the attacker can send a request to the software and have it be forwarded from the target. The request would appear to be coming from the software's system, not the attacker's system. As a result, the attacker can bypass access controls (such as firewalls) or hide the source of malicious requests, since the requests would not be coming directly from the attacker. Since proxy functionality and message-forwarding often serve a legitimate purpose, this issue only becomes a vulnerability when: The software runs with different privileges or on a different system, or otherwise has different levels of access than the upstream component; The attacker is prevented from making the request directly to the target; and -The attacker can create a request that the proxy does not explicitly intend to be forwarded on the behalf of the requester. Such a request might point to an unexpected hostname, port number, or service. Or, the request might be sent to an allowed service, but the request could contain disallowed directives, commands, or resources.",,Unknown,The Confused Deputy (or why capabilities might have been invented): http://www.cap-lore.com/CapTheory/ConfusedDeputy.html +The attacker can create a request that the proxy does not explicitly intend to be forwarded on the behalf of the requester. Such a request might point to an unexpected hostname, port number, or service. Or, the request might be sent to an allowed service, but the request could contain disallowed directives, commands, or resources.",,unclassified,The Confused Deputy (or why capabilities might have been invented): http://www.cap-lore.com/CapTheory/ConfusedDeputy.html CWE-443,EN-DEPRECATED (Duplicate): HTTP response splitting (Type: Base),"This weakness can be found at CWE-113. If an attacker cannot directly contact a target, but the software has access to the target, then the attacker can send a request to the software and have it be forwarded from the target. The request would appear to be coming from the software's system, not the attacker's system. As a result, the attacker can bypass access controls (such as firewalls) or hide the source of malicious requests, since the requests would not be coming directly from the attacker. Since proxy functionality and message-forwarding often serve a legitimate purpose, this issue only becomes a vulnerability when: The software runs with different privileges or on a different system, or otherwise has different levels of access than the upstream component; The attacker is prevented from making the request directly to the target; and -The attacker can create a request that the proxy does not explicitly intend to be forwarded on the behalf of the requester. Such a request might point to an unexpected hostname, port number, or service. Or, the request might be sent to an allowed service, but the request could contain disallowed directives, commands, or resources.",,Unknown, +The attacker can create a request that the proxy does not explicitly intend to be forwarded on the behalf of the requester. Such a request might point to an unexpected hostname, port number, or service. Or, the request might be sent to an allowed service, but the request could contain disallowed directives, commands, or resources.",,unclassified, CWE-444,EN-Inconsistent Interpretation of HTTP Requests (HTTP Request Smuggling) (Type: Base),"When malformed or abnormal HTTP requests are interpreted by one or more entities in the data flow between the user and the web server, such as a proxy or firewall, they can be interpreted inconsistently, allowing the attacker to ""smuggle"" a request to one device without the other device being aware of it. If an attacker cannot directly contact a target, but the software has access to the target, then the attacker can send a request to the software and have it be forwarded from the target. The request would appear to be coming from the software's system, not the attacker's system. As a result, the attacker can bypass access controls (such as firewalls) or hide the source of malicious requests, since the requests would not be coming directly from the attacker. Since proxy functionality and message-forwarding often serve a legitimate purpose, this issue only becomes a vulnerability when: The software runs with different privileges or on a different system, or otherwise has different levels of access than the upstream component; The attacker is prevented from making the request directly to the target; and -The attacker can create a request that the proxy does not explicitly intend to be forwarded on the behalf of the requester. Such a request might point to an unexpected hostname, port number, or service. Or, the request might be sent to an allowed service, but the request could contain disallowed directives, commands, or resources.",,Unknown,HTTP Request Smuggling: http://www.cgisecurity.com/lib/HTTP-Request-Smuggling.pdf +The attacker can create a request that the proxy does not explicitly intend to be forwarded on the behalf of the requester. Such a request might point to an unexpected hostname, port number, or service. Or, the request might be sent to an allowed service, but the request could contain disallowed directives, commands, or resources.",,unclassified,HTTP Request Smuggling: http://www.cgisecurity.com/lib/HTTP-Request-Smuggling.pdf CWE-446,EN-UI Discrepancy for Security Feature (Type: Base),"The user interface does not correctly enable or configure a security feature, but the interface provides feedback that causes the user to believe that the feature is in a secure state. -When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,Unknown, +When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, CWE-447,EN-Unimplemented or Unsupported Feature in UI (Type: Base),"A UI function for a security feature appears to be supported and gives feedback to the user that suggests that it is supported, but the underlying functionality is not implemented. -When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,Unknown, +When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, CWE-448,EN-Obsolete Feature in UI (Type: Base),"A UI function is obsolete and the product does not warn the user. -When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,Unknown, +When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, CWE-449,EN-The UI Performs the Wrong Action (Type: Base),"The UI performs the wrong action with respect to the user's request. -When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,Unknown, +When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, CWE-45,EN-Path Equivalence: file...name (Multiple Internal Dot) (Type: Variant),"A software system that accepts path input in the form of multiple internal dot ('file...dir') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,Unknown, +When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, CWE-450,EN-Multiple Interpretations of UI Input (Type: Base),"The UI has multiple interpretations of user input but does not prompt the user when it selects the less secure interpretation. -When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,Unknown, +When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, CWE-451,EN-UI Misrepresentation of Critical Information (Type: Base),"The UI does not properly represent critical information to the user, allowing the information - or its source - to be obscured or spoofed. This is often a component in phishing attacks. -When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,Unknown, +When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, CWE-453,EN-Insecure Default Variable Initialization (Type: Base),"The software, by default, initializes an internal variable with an insecure or less secure value than is possible. -When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,Unknown, +When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, CWE-454,EN-External Initialization of Trusted Variables or Data Stores (Type: Base),"The software initializes critical internal variables or data stores using inputs that can be modified by untrusted actors. -A software system should be reluctant to trust variables that have been initialized outside of its trust boundary, especially if they are initialized by users. They may have been initialized incorrectly. If an attacker can initialize the variable, then he/she can influence what the vulnerable system will do.",,Unknown, +A software system should be reluctant to trust variables that have been initialized outside of its trust boundary, especially if they are initialized by users. They may have been initialized incorrectly. If an attacker can initialize the variable, then he/she can influence what the vulnerable system will do.",,unclassified, CWE-455,EN-Non-exit on Failed Initialization (Type: Base),"The software does not exit or otherwise modify its operation when security-relevant errors occur during initialization, such as when a configuration file has a format error, which can cause the software to execute in a less secure fashion than intended by the administrator. -A software system should be reluctant to trust variables that have been initialized outside of its trust boundary, especially if they are initialized by users. They may have been initialized incorrectly. If an attacker can initialize the variable, then he/she can influence what the vulnerable system will do.",,Unknown, +A software system should be reluctant to trust variables that have been initialized outside of its trust boundary, especially if they are initialized by users. They may have been initialized incorrectly. If an attacker can initialize the variable, then he/she can influence what the vulnerable system will do.",,unclassified, CWE-456,EN-Missing Initialization of a Variable (Type: Base),"The software does not initialize critical variables, which causes the execution environment to use unexpected values. -A software system should be reluctant to trust variables that have been initialized outside of its trust boundary, especially if they are initialized by users. They may have been initialized incorrectly. If an attacker can initialize the variable, then he/she can influence what the vulnerable system will do.",,Unknown,"The Art of Software Security Assessment: Chapter 7, ""Variable Initialization"", Page 312." +A software system should be reluctant to trust variables that have been initialized outside of its trust boundary, especially if they are initialized by users. They may have been initialized incorrectly. If an attacker can initialize the variable, then he/she can influence what the vulnerable system will do.",,unclassified,"The Art of Software Security Assessment: Chapter 7, ""Variable Initialization"", Page 312." CWE-458,EN-DEPRECATED: Incorrect Initialization (Type: Base),"This weakness has been deprecated because its name and description did not match. The description duplicated CWE-454, while the name suggested a more abstract initialization problem. Please refer to CWE-665 for the more abstract problem. -In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,Unknown, +In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,unclassified, CWE-459,EN-Incomplete Cleanup (Type: Base),"The software does not properly ""clean up"" and remove temporary or supporting resources after they have been used. -In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,Unknown, +In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,unclassified, CWE-46,EN-Path Equivalence: filename (Trailing Space) (Type: Variant),"A software system that accepts path input in the form of trailing space ('filedir ') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,Unknown, +In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,unclassified, CWE-463,EN-Deletion of Data Structure Sentinel (Type: Base),"The accidental deletion of a data-structure sentinel can cause serious programming logic problems. -Often times data-structure sentinels are used to mark structure of the data structure. A common example of this is the null character at the end of strings. Another common example is linked lists which may contain a sentinel to mark the end of the list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the deletion or modification outside of some wrapper interface which provides safety.",,Unknown,"The Art of Software Security Assessment: Chapter 8, ""NUL-Termination Problems"", Page 452." +Often times data-structure sentinels are used to mark structure of the data structure. A common example of this is the null character at the end of strings. Another common example is linked lists which may contain a sentinel to mark the end of the list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the deletion or modification outside of some wrapper interface which provides safety.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""NUL-Termination Problems"", Page 452." CWE-466,EN-Return of Pointer Value Outside of Expected Range (Type: Base),"A function can return a pointer to memory that is outside of the buffer that the pointer is expected to reference. -Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,Unknown,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89" +Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89" CWE-47,EN-Path Equivalence: filename (Leading Space) (Type: Variant),"A software system that accepts path input in the form of leading space (' filedir') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,Unknown, +Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,unclassified, CWE-470,EN-Use of Externally-Controlled Input to Select Classes or Code (Unsafe Reflection) (Type: Base),"The application uses external input with reflection to select which classes or code to use, but it does not sufficiently prevent the input from selecting improper classes or code. -If the application uses external inputs to determine which class to instantiate or which method to invoke, then an attacker could supply values to select unexpected classes or methods. If this occurs, then the attacker could create control flow paths that were not intended by the developer. These paths could bypass authentication or access control checks, or otherwise cause the application to behave in an unexpected manner. This situation becomes a doomsday scenario if the attacker can upload files into a location that appears on the application's classpath (CWE-427) or add new entries to the application's classpath (CWE-426). Under either of these conditions, the attacker can use reflection to introduce new, malicious behavior into the application.",,Unknown, +If the application uses external inputs to determine which class to instantiate or which method to invoke, then an attacker could supply values to select unexpected classes or methods. If this occurs, then the attacker could create control flow paths that were not intended by the developer. These paths could bypass authentication or access control checks, or otherwise cause the application to behave in an unexpected manner. This situation becomes a doomsday scenario if the attacker can upload files into a location that appears on the application's classpath (CWE-427) or add new entries to the application's classpath (CWE-426). Under either of these conditions, the attacker can use reflection to introduce new, malicious behavior into the application.",,unclassified, CWE-471,EN-Modification of Assumed-Immutable Data (MAID) (Type: Base),"The software does not properly protect an assumed-immutable element from being modified by an attacker. -If the application uses external inputs to determine which class to instantiate or which method to invoke, then an attacker could supply values to select unexpected classes or methods. If this occurs, then the attacker could create control flow paths that were not intended by the developer. These paths could bypass authentication or access control checks, or otherwise cause the application to behave in an unexpected manner. This situation becomes a doomsday scenario if the attacker can upload files into a location that appears on the application's classpath (CWE-427) or add new entries to the application's classpath (CWE-426). Under either of these conditions, the attacker can use reflection to introduce new, malicious behavior into the application.",,Unknown, +If the application uses external inputs to determine which class to instantiate or which method to invoke, then an attacker could supply values to select unexpected classes or methods. If this occurs, then the attacker could create control flow paths that were not intended by the developer. These paths could bypass authentication or access control checks, or otherwise cause the application to behave in an unexpected manner. This situation becomes a doomsday scenario if the attacker can upload files into a location that appears on the application's classpath (CWE-427) or add new entries to the application's classpath (CWE-426). Under either of these conditions, the attacker can use reflection to introduce new, malicious behavior into the application.",,unclassified, CWE-472,EN-External Control of Assumed-Immutable Web Parameter (Type: Base),"The web application does not sufficiently verify inputs that are assumed to be immutable but are actually externally controllable, such as hidden form fields. If a web product does not properly protect assumed-immutable values from modification in hidden form fields, parameters, cookies, or URLs, this can lead to modification of critical data. Web applications often mistakenly make the assumption that data passed to the client in hidden fields or cookies is not susceptible to tampering. Improper validation of data that are user-controllable can lead to the application processing incorrect, and often malicious, input. -For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,Unknown,"24 Deadly Sins of Software Security: ""Sin 4: Use of Magic URLs, Predictable Cookies, and Hidden Form Fields."" Page 75 +For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 4: Use of Magic URLs, Predictable Cookies, and Hidden Form Fields."" Page 75 The Art of Software Security Assessment: Chapter 17, ""Embedding State in HTML and URLs"", Page 1032." CWE-473,EN-PHP External Variable Modification (Type: Variant),"A PHP application does not properly protect against the modification of variables from external sources, such as query parameters or cookies. This can expose the application to numerous weaknesses that would not exist otherwise. If a web product does not properly protect assumed-immutable values from modification in hidden form fields, parameters, cookies, or URLs, this can lead to modification of critical data. Web applications often mistakenly make the assumption that data passed to the client in hidden fields or cookies is not susceptible to tampering. Improper validation of data that are user-controllable can lead to the application processing incorrect, and often malicious, input. -For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,Unknown, +For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,unclassified, CWE-474,EN-Use of Function with Inconsistent Implementations (Type: Base),"The code uses a function that has inconsistent implementations across operating systems and versions, which might cause security-relevant portability problems. If a web product does not properly protect assumed-immutable values from modification in hidden form fields, parameters, cookies, or URLs, this can lead to modification of critical data. Web applications often mistakenly make the assumption that data passed to the client in hidden fields or cookies is not susceptible to tampering. Improper validation of data that are user-controllable can lead to the application processing incorrect, and often malicious, input. -For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,Unknown, +For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,unclassified, CWE-475,EN-Undefined Behavior for Input to API (Type: Base),"The behavior of this function is undefined unless its control parameter is set to a specific value. If a web product does not properly protect assumed-immutable values from modification in hidden form fields, parameters, cookies, or URLs, this can lead to modification of critical data. Web applications often mistakenly make the assumption that data passed to the client in hidden fields or cookies is not susceptible to tampering. Improper validation of data that are user-controllable can lead to the application processing incorrect, and often malicious, input. -For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,Unknown, +For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,unclassified, CWE-477,EN-Use of Obsolete Functions (Type: Base),"The code uses deprecated or obsolete functions, which suggests that the code has not been actively reviewed or maintained. -NULL pointer dereference issues can occur through a number of flaws, including race conditions, and simple programming omissions.",,Unknown, +NULL pointer dereference issues can occur through a number of flaws, including race conditions, and simple programming omissions.",,unclassified, CWE-478,EN-Missing Default Case in Switch Statement (Type: Variant),"The code does not have a default case in a switch statement, which might lead to complex logical errors and resultant weaknesses. -NULL pointer dereference issues can occur through a number of flaws, including race conditions, and simple programming omissions.",,Unknown,"The Art of Software Security Assessment: Chapter 7, ""Switch Statements"", Page 337." +NULL pointer dereference issues can occur through a number of flaws, including race conditions, and simple programming omissions.",,unclassified,"The Art of Software Security Assessment: Chapter 7, ""Switch Statements"", Page 337." CWE-48,EN-Path Equivalence: file name (Internal Whitespace) (Type: Variant),"A software system that accepts path input in the form of internal space ('file(SPACE)name') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. Non-reentrant functions are functions that cannot safely be called, interrupted, and then recalled before the first call has finished without resulting in memory corruption. This can lead to an unexpected system state an unpredictable results with a variety of potential consequences depending on context, including denial of service and code execution. -Many functions are not reentrant, but some of them can result in the corruption of memory if they are used in a signal handler. The function call syslog() is an example of this. In order to perform its functionality, it allocates a small amount of memory as ""scratch space."" If syslog() is suspended by a signal call and the signal handler calls syslog(), the memory used by both of these functions enters an undefined, and possibly, exploitable state. Implementations of malloc() and free() manage metadata in global structures in order to track which memory is allocated versus which memory is available, but they are non-reentrant. Simultaneous calls to these functions can cause corruption of the metadata.",,Unknown, +Many functions are not reentrant, but some of them can result in the corruption of memory if they are used in a signal handler. The function call syslog() is an example of this. In order to perform its functionality, it allocates a small amount of memory as ""scratch space."" If syslog() is suspended by a signal call and the signal handler calls syslog(), the memory used by both of these functions enters an undefined, and possibly, exploitable state. Implementations of malloc() and free() manage metadata in global structures in order to track which memory is allocated versus which memory is available, but they are non-reentrant. Simultaneous calls to these functions can cause corruption of the metadata.",,unclassified, CWE-485,EN-Insufficient Encapsulation (Type: Class),"The product does not sufficiently encapsulate critical data or functionality. -Encapsulation is about drawing strong boundaries. In a web browser that might mean ensuring that your mobile code cannot be abused by other mobile code. On the server it might mean differentiation between validated data and unvalidated data, between one user's data and another's, or between data users are allowed to see and data that they are not.",,Unknown, +Encapsulation is about drawing strong boundaries. In a web browser that might mean ensuring that your mobile code cannot be abused by other mobile code. On the server it might mean differentiation between validated data and unvalidated data, between one user's data and another's, or between data users are allowed to see and data that they are not.",,unclassified, CWE-488,EN-Exposure of Data Element to Wrong Session (Type: Variant),"The product does not sufficiently enforce boundaries between the states of different sessions, causing data to be provided to, or used by, the wrong session. Data can ""bleed"" from one session to another through member variables of singleton objects, such as Servlets, and objects from a shared pool. -In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,Unknown, +In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,unclassified, CWE-489,EN-Leftover Debug Code (Type: Base),"The application can be deployed with active debugging code that can create unintended entry points. Data can ""bleed"" from one session to another through member variables of singleton objects, such as Servlets, and objects from a shared pool. -In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,Unknown, +In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,unclassified, CWE-49,EN-Path Equivalence: filename/ (Trailing Slash) (Type: Variant),"A software system that accepts path input in the form of trailing slash ('filedir/') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. Data can ""bleed"" from one session to another through member variables of singleton objects, such as Servlets, and objects from a shared pool. -In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,Unknown, +In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,unclassified, CWE-491,EN-Public cloneable() Method Without Final (Object Hijack) (Type: Variant),"A class has a cloneable() method that is not declared final, which allows an object to be created without calling the constructor. This can cause the object to be in an unexpected state. Data can ""bleed"" from one session to another through member variables of singleton objects, such as Servlets, and objects from a shared pool. -In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,Unknown,"OWASP , Attack Category : Mobile code: object hijack: http://www.owasp.org/index.php/Mobile_code:_object_hijack" +In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,unclassified,"OWASP , Attack Category : Mobile code: object hijack: http://www.owasp.org/index.php/Mobile_code:_object_hijack" CWE-495,EN-Private Array-Typed Field Returned From A Public Method (Type: Variant),"The product has a method that is declared public, but returns a reference to a private array, which could then be modified in unexpected ways. -An attacker can execute malicious code by compromising the host server, performing DNS spoofing, or modifying the code in transit.",,Unknown, +An attacker can execute malicious code by compromising the host server, performing DNS spoofing, or modifying the code in transit.",,unclassified, CWE-496,EN-Public Data Assigned to Private Array-Typed Field (Type: Variant),"Assigning public data to a private array is equivalent to giving public access to the array. -An attacker can execute malicious code by compromising the host server, performing DNS spoofing, or modifying the code in transit.",,Unknown, +An attacker can execute malicious code by compromising the host server, performing DNS spoofing, or modifying the code in transit.",,unclassified, CWE-497,EN-Exposure of System Data to an Unauthorized Control Sphere (Type: Variant),"Exposing system data or debugging information helps an adversary learn about the system and form an attack plan. -An information exposure occurs when system data or debugging information leaves the program through an output stream or logging function that makes it accessible to unauthorized parties. An attacker can also cause errors to occur by submitting unusual requests to the web application. The response to these errors can reveal detailed system information, deny service, cause security mechanisms to fail, and crash the server. An attacker can use error messages that reveal technologies, operating systems, and product versions to tune the attack against known vulnerabilities in these technologies. An application may use diagnostic methods that provide significant implementation details such as stack traces as part of its error handling mechanism.",,Unknown, +An information exposure occurs when system data or debugging information leaves the program through an output stream or logging function that makes it accessible to unauthorized parties. An attacker can also cause errors to occur by submitting unusual requests to the web application. The response to these errors can reveal detailed system information, deny service, cause security mechanisms to fail, and crash the server. An attacker can use error messages that reveal technologies, operating systems, and product versions to tune the attack against known vulnerabilities in these technologies. An application may use diagnostic methods that provide significant implementation details such as stack traces as part of its error handling mechanism.",,unclassified, CWE-5,EN-J2EE Misconfiguration: Data Transmission Without Encryption (Type: Variant),"Information sent over a network can be compromised while in transit. An attacker may be able to read/modify the contents if the data are sent in plaintext or are weakly encrypted. -Serializable classes are effectively open classes since data cannot be hidden in them. Classes that do not explicitly deny serialization can be serialized by any other class, which can then in turn use the data stored inside it.",,Unknown, +Serializable classes are effectively open classes since data cannot be hidden in them. Classes that do not explicitly deny serialization can be serialized by any other class, which can then in turn use the data stored inside it.",,unclassified, CWE-50,EN-Path Equivalence: //multiple/leading/slash (Type: Variant),"A software system that accepts path input in the form of multiple leading slash ('//multiple/leading/slash') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -Serializable classes are effectively open classes since data cannot be hidden in them. Classes that do not explicitly deny serialization can be serialized by any other class, which can then in turn use the data stored inside it.",,Unknown, +Serializable classes are effectively open classes since data cannot be hidden in them. Classes that do not explicitly deny serialization can be serialized by any other class, which can then in turn use the data stored inside it.",,unclassified, CWE-501,EN-Trust Boundary Violation (Type: Base),"The product mixes trusted and untrusted data in the same data structure or structured message. -By combining trusted and untrusted data in the same data structure, it becomes easier for programmers to mistakenly trust unvalidated data.",,Unknown, +By combining trusted and untrusted data in the same data structure, it becomes easier for programmers to mistakenly trust unvalidated data.",,unclassified, CWE-506,EN-Embedded Malicious Code (Type: Class),"The application contains code that appears to be malicious in nature. -Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,Unknown, +Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified, CWE-507,EN-Trojan Horse (Type: Base),"The software appears to contain benign or useful functionality, but it also contains code that is hidden from normal operation that violates the intended security policy of the user or the system administrator. -Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,Unknown,"Writing Secure Code: Chapter 7, ""Viruses, Trojans, and Worms In a Nutshell"" Page 208" +Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified,"Writing Secure Code: Chapter 7, ""Viruses, Trojans, and Worms In a Nutshell"" Page 208" CWE-508,EN-Non-Replicating Malicious Code (Type: Base),"Non-replicating malicious code only resides on the target system or software that is attacked; it does not attempt to spread to other systems. -Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,Unknown, +Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified, CWE-509,EN-Replicating Malicious Code (Virus or Worm) (Type: Base),"Replicating malicious code, including viruses and worms, will attempt to attack other systems once it has successfully compromised the target system or software. -Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,Unknown, +Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified, CWE-51,EN-Path Equivalence: /multiple//internal/slash (Type: Variant),"A software system that accepts path input in the form of multiple internal slash ('/multiple//internal/slash/') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,Unknown, +Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified, CWE-510,EN-Trapdoor (Type: Base),"A trapdoor is a hidden piece of code that responds to a special input, allowing its user access to resources without passing through the normal security enforcement mechanism. -Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,Unknown, +Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified, CWE-511,EN-Logic/Time Bomb (Type: Base),"The software contains code that is designed to disrupt the legitimate operation of the software (or its environment) when a certain time passes, or when a certain logical condition is met. -When the time bomb or logic bomb is detonated, it may perform a denial of service such as crashing the system, deleting critical data, or degrading system response time. This bomb might be placed within either a replicating or non-replicating Trojan horse.",,Unknown,Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/ +When the time bomb or logic bomb is detonated, it may perform a denial of service such as crashing the system, deleting critical data, or degrading system response time. This bomb might be placed within either a replicating or non-replicating Trojan horse.",,unclassified,Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/ CWE-512,EN-Spyware (Type: Base),"The software collects personally identifiable information about a human user or the user's activities, but the software accesses this information using other resources besides itself, and it does not require that user's explicit approval or direct input into the software. -""Spyware"" is a commonly used term with many definitions and interpretations. In general, it is meant to software that collects information or installs functionality that human users might not allow if they were fully aware of the actions being taken by the software. For example, a user might expect that tax software would collect a social security number and include it when filing a tax return, but that same user would not expect gaming software to obtain the social security number from that tax software's data.",,Unknown, +""Spyware"" is a commonly used term with many definitions and interpretations. In general, it is meant to software that collects information or installs functionality that human users might not allow if they were fully aware of the actions being taken by the software. For example, a user might expect that tax software would collect a social security number and include it when filing a tax return, but that same user would not expect gaming software to obtain the social security number from that tax software's data.",,unclassified, CWE-514,EN-Covert Channel (Type: Class),"A covert channel is a path that can be used to transfer information in a way not intended by the system's designers. -Typically the system has not given authorization for the transmission and has no knowledge of its occurrence.",,Unknown, +Typically the system has not given authorization for the transmission and has no knowledge of its occurrence.",,unclassified, CWE-516,EN-DEPRECATED (Duplicate): Covert Timing Channel (Type: Base),"This weakness can be found at CWE-385. -Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,Unknown, +Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,unclassified, CWE-52,EN-Path Equivalence: /multiple/trailing/slash// (Type: Variant),"A software system that accepts path input in the form of multiple trailing slash ('/multiple/trailing/slash//') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,Unknown, +Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,unclassified, CWE-520,EN-.NET Misconfiguration: Use of Impersonation (Type: Variant),"Allowing a .NET application to run at potentially escalated levels of access to the underlying operating and file systems can be dangerous and result in various forms of attacks. -Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,Unknown, +Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,unclassified, CWE-521,EN-Weak Password Requirements (Type: Base),"The product does not require that users should have strong passwords, which makes it easier for attackers to compromise user accounts. -An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,Unknown,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" +An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" CWE-522,EN-Insufficiently Protected Credentials (Type: Base),"This weakness occurs when the application transmits or stores authentication credentials and uses an insecure method that is susceptible to unauthorized interception and/or retrieval. -An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,Unknown,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" +An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" CWE-523,EN-Unprotected Transport of Credentials (Type: Variant),"Login pages not using adequate measures to protect the user name and password while they are in transit from the client to the server. -An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,Unknown, +An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified, CWE-524,EN-Information Exposure Through Caching (Type: Variant),"The application uses a cache to maintain a pool of objects, threads, connections, pages, or passwords to minimize the time it takes to access them or the resources to which they connect. If implemented improperly, these caches can allow access to unauthorized information or cause a denial of service vulnerability. -An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,Unknown, +An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified, CWE-525,EN-Information Exposure Through Browser Caching (Type: Variant),"For each web page, the application should have an appropriate caching policy specifying the extent to which the page and its form fields should be cached. -An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,Unknown, +An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified, CWE-526,EN-Information Exposure Through Environmental Variables (Type: Variant),"Environmental variables may contain sensitive information about a remote server. -An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,Unknown, +An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified, CWE-527,EN-Exposure of CVS Repository to an Unauthorized Control Sphere (Type: Variant),"The product stores a CVS repository in a directory or other container that is accessible to actors outside of the intended control sphere. -Information contained within a CVS subdirectory on a web server or other server could be recovered by an attacker and used for malicious purposes. This information may include usernames, filenames, path root, and IP addresses.",,Unknown, +Information contained within a CVS subdirectory on a web server or other server could be recovered by an attacker and used for malicious purposes. This information may include usernames, filenames, path root, and IP addresses.",,unclassified, CWE-528,EN-Exposure of Core Dump File to an Unauthorized Control Sphere (Type: Variant),"The product generates a core dump file in a directory that is accessible to actors outside of the intended control sphere. -Information contained within a CVS subdirectory on a web server or other server could be recovered by an attacker and used for malicious purposes. This information may include usernames, filenames, path root, and IP addresses.",,Unknown, +Information contained within a CVS subdirectory on a web server or other server could be recovered by an attacker and used for malicious purposes. This information may include usernames, filenames, path root, and IP addresses.",,unclassified, CWE-529,EN-Exposure of Access Control List Files to an Unauthorized Control Sphere (Type: Variant),"The product stores access control list files in a directory or other container that is accessible to actors outside of the intended control sphere. -Exposure of these access control list files may give the attacker information about the configuration of the site or system. This information may then be used to bypass the intended security policy or identify trusted systems from which an attack can be launched.",,Unknown, +Exposure of these access control list files may give the attacker information about the configuration of the site or system. This information may then be used to bypass the intended security policy or identify trusted systems from which an attack can be launched.",,unclassified, CWE-53,EN-Path Equivalence: \multiple\\internal\backslash (Type: Variant),"A software system that accepts path input in the form of multiple internal backslash ('\multiple\trailing\\slash') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -Exposure of these access control list files may give the attacker information about the configuration of the site or system. This information may then be used to bypass the intended security policy or identify trusted systems from which an attack can be launched.",,Unknown, +Exposure of these access control list files may give the attacker information about the configuration of the site or system. This information may then be used to bypass the intended security policy or identify trusted systems from which an attack can be launched.",,unclassified, CWE-530,EN-Exposure of Backup File to an Unauthorized Control Sphere (Type: Variant),"A backup file is stored in a directory that is accessible to actors outside of the intended control sphere. -Often, old files are renamed with an extension such as .~bk to distinguish them from production files. The source code for old files that have been renamed in this manner and left in the webroot can often be retrieved. This renaming may have been performed automatically by the web server, or manually by the administrator.",,Unknown, +Often, old files are renamed with an extension such as .~bk to distinguish them from production files. The source code for old files that have been renamed in this manner and left in the webroot can often be retrieved. This renaming may have been performed automatically by the web server, or manually by the administrator.",,unclassified, CWE-531,EN-Information Exposure Through Test Code (Type: Variant),"Accessible test applications can pose a variety of security risks. Since developers or administrators rarely consider that someone besides themselves would even know about the existence of these applications, it is common for them to contain sensitive information or functions. -Often, old files are renamed with an extension such as .~bk to distinguish them from production files. The source code for old files that have been renamed in this manner and left in the webroot can often be retrieved. This renaming may have been performed automatically by the web server, or manually by the administrator.",,Unknown, +Often, old files are renamed with an extension such as .~bk to distinguish them from production files. The source code for old files that have been renamed in this manner and left in the webroot can often be retrieved. This renaming may have been performed automatically by the web server, or manually by the administrator.",,unclassified, CWE-533,EN-Information Exposure Through Server Log Files (Type: Variant),"A server.log file was found. This can give information on whatever application left the file. Usually this can give full path names and system information, and sometimes usernames and passwords. -While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,Unknown, +While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified, CWE-534,EN-Information Exposure Through Debug Log Files (Type: Variant),"The application does not sufficiently restrict access to a log file that is used for debugging. -While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,Unknown, +While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified, CWE-535,EN-Information Exposure Through Shell Error Message (Type: Variant),"A command shell error message indicates that there exists an unhandled exception in the web application code. In many cases, an attacker can leverage the conditions that cause these errors in order to gain unauthorized access to the system. -While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,Unknown, +While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified, CWE-536,EN-Information Exposure Through Servlet Runtime Error Message (Type: Variant),"A servlet error message indicates that there exists an unhandled exception in your web application code and may provide useful information to an attacker. -While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,Unknown, +While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified, CWE-537,EN-Information Exposure Through Java Runtime Error Message (Type: Variant),"In many cases, an attacker can leverage the conditions that cause unhandled exception errors in order to gain unauthorized access to the system. -While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,Unknown, +While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified, CWE-538,EN-File and Directory Information Exposure (Type: Base),"The product stores sensitive information in files or directories that are accessible to actors outside of the intended control sphere. -While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,Unknown,"24 Deadly Sins of Software Security: ""Sin 12: Information Leakage."" Page 191" +While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 12: Information Leakage."" Page 191" CWE-539,EN-Information Exposure Through Persistent Cookies (Type: Variant),"Persistent cookies are cookies that are stored on the browser's hard drive. This can cause security and privacy issues depending on the information stored in the cookie and how it is accessed. -Cookies are small bits of data that are sent by the web application but stored locally in the browser. This lets the application use the cookie to pass information between pages and store variable information. The web application controls what information is stored in a cookie and how it is used. Typical types of information stored in cookies are session Identifiers, personalization and customization information, and in rare cases even usernames to enable automated logins. There are two different types of cookies: session cookies and persistent cookies. Session cookies just live in the browser's memory, and are not stored anywhere, but persistent cookies are stored on the browser's hard drive.",,Unknown, +Cookies are small bits of data that are sent by the web application but stored locally in the browser. This lets the application use the cookie to pass information between pages and store variable information. The web application controls what information is stored in a cookie and how it is used. Typical types of information stored in cookies are session Identifiers, personalization and customization information, and in rare cases even usernames to enable automated logins. There are two different types of cookies: session cookies and persistent cookies. Session cookies just live in the browser's memory, and are not stored anywhere, but persistent cookies are stored on the browser's hard drive.",,unclassified, CWE-54,EN-Path Equivalence: filedir\ (Trailing Backslash) (Type: Variant),"A software system that accepts path input in the form of trailing backslash ('filedir\') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -Cookies are small bits of data that are sent by the web application but stored locally in the browser. This lets the application use the cookie to pass information between pages and store variable information. The web application controls what information is stored in a cookie and how it is used. Typical types of information stored in cookies are session Identifiers, personalization and customization information, and in rare cases even usernames to enable automated logins. There are two different types of cookies: session cookies and persistent cookies. Session cookies just live in the browser's memory, and are not stored anywhere, but persistent cookies are stored on the browser's hard drive.",,Unknown, +Cookies are small bits of data that are sent by the web application but stored locally in the browser. This lets the application use the cookie to pass information between pages and store variable information. The web application controls what information is stored in a cookie and how it is used. Typical types of information stored in cookies are session Identifiers, personalization and customization information, and in rare cases even usernames to enable automated logins. There are two different types of cookies: session cookies and persistent cookies. Session cookies just live in the browser's memory, and are not stored anywhere, but persistent cookies are stored on the browser's hard drive.",,unclassified, CWE-540,EN-Information Exposure Through Source Code (Type: Variant),"Source code on a web server often contains sensitive information and should generally not be accessible to users. -There are situations where it is critical to remove source code from an area or server. For example, obtaining Perl source code on a system allows an attacker to understand the logic of the script and extract extremely useful information such as code bugs or logins and passwords.",,Unknown, +There are situations where it is critical to remove source code from an area or server. For example, obtaining Perl source code on a system allows an attacker to understand the logic of the script and extract extremely useful information such as code bugs or logins and passwords.",,unclassified, CWE-541,EN-Information Exposure Through Include Source Code (Type: Variant),"If an include file source is accessible, the file can contain usernames and passwords, as well as sensitive information pertaining to the application and system. -There are situations where it is critical to remove source code from an area or server. For example, obtaining Perl source code on a system allows an attacker to understand the logic of the script and extract extremely useful information such as code bugs or logins and passwords.",,Unknown, +There are situations where it is critical to remove source code from an area or server. For example, obtaining Perl source code on a system allows an attacker to understand the logic of the script and extract extremely useful information such as code bugs or logins and passwords.",,unclassified, CWE-542,EN-Information Exposure Through Cleanup Log Files (Type: Variant),"The application does not properly protect or delete a log file related to cleanup. -There are situations where it is critical to remove source code from an area or server. For example, obtaining Perl source code on a system allows an attacker to understand the logic of the script and extract extremely useful information such as code bugs or logins and passwords.",,Unknown, +There are situations where it is critical to remove source code from an area or server. For example, obtaining Perl source code on a system allows an attacker to understand the logic of the script and extract extremely useful information such as code bugs or logins and passwords.",,unclassified, CWE-543,EN-Use of Singleton Pattern Without Synchronization in a Multithreaded Context (Type: Variant),"The software uses the singleton pattern when creating a resource within a multithreaded environment. -The use of a singleton pattern may not be thread-safe.",,Unknown,Thread-Specifc Storage for C/C++: http://www.cs.wustl.edu/~schmidt/PDF/TSS-pattern.pdf +The use of a singleton pattern may not be thread-safe.",,unclassified,Thread-Specifc Storage for C/C++: http://www.cs.wustl.edu/~schmidt/PDF/TSS-pattern.pdf CWE-544,EN-Missing Standardized Error Handling Mechanism (Type: Base),"The software does not use a standardized method for handling errors throughout the code, which might introduce inconsistent error handling and resultant weaknesses. -If the application handles error messages individually, on a one-by-one basis, this is likely to result in inconsistent error handling. The causes of errors may be lost. Also, detailed information about the causes of an error may be unintentionally returned to the user.",,Unknown, +If the application handles error messages individually, on a one-by-one basis, this is likely to result in inconsistent error handling. The causes of errors may be lost. Also, detailed information about the causes of an error may be unintentionally returned to the user.",,unclassified, CWE-545,EN-Use of Dynamic Class Loading (Type: Variant),"Dynamically loaded code has the potential to be malicious. -If the application handles error messages individually, on a one-by-one basis, this is likely to result in inconsistent error handling. The causes of errors may be lost. Also, detailed information about the causes of an error may be unintentionally returned to the user.",,Unknown, +If the application handles error messages individually, on a one-by-one basis, this is likely to result in inconsistent error handling. The causes of errors may be lost. Also, detailed information about the causes of an error may be unintentionally returned to the user.",,unclassified, CWE-546,EN-Suspicious Comment (Type: Variant),"The code contains comments that suggest the presence of bugs, incomplete functionality, or weaknesses. -Many suspicious comments, such as BUG, HACK, FIXME, LATER, LATER2, TODO, in the code indicate missing security functionality and checking. Others indicate code problems that programmers should fix, such as hard-coded variables, error handling, not using stored procedures, and performance issues.",,Unknown, +Many suspicious comments, such as BUG, HACK, FIXME, LATER, LATER2, TODO, in the code indicate missing security functionality and checking. Others indicate code problems that programmers should fix, such as hard-coded variables, error handling, not using stored procedures, and performance issues.",,unclassified, CWE-547,"EN-Use of Hard-coded, Security-relevant Constants (Type: Variant)","The program uses hard-coded constants instead of symbolic names for security-critical values, which increases the likelihood of mistakes during code maintenance or security policy change. -If the developer does not find all occurrences of the hard-coded constants, an incorrect policy decision may be made if one of the constants is not changed. Making changes to these values will require code changes that may be difficult or impossible once the system is released to the field. In addition, these hard-coded values may become available to attackers if the code is ever disclosed.",,Unknown, +If the developer does not find all occurrences of the hard-coded constants, an incorrect policy decision may be made if one of the constants is not changed. Making changes to these values will require code changes that may be difficult or impossible once the system is released to the field. In addition, these hard-coded values may become available to attackers if the code is ever disclosed.",,unclassified, CWE-548,EN-Information Exposure Through Directory Listing (Type: Variant),"A directory listing is inappropriately exposed, yielding potentially sensitive information to attackers. -A directory listing provides an attacker with the complete index of all the resources located inside of the directory. The specific risks and consequences vary depending on which files are listed and accessible.",,Unknown, +A directory listing provides an attacker with the complete index of all the resources located inside of the directory. The specific risks and consequences vary depending on which files are listed and accessible.",,unclassified, CWE-549,EN-Missing Password Field Masking (Type: Variant),"The software does not mask passwords during entry, increasing the potential for attackers to observe and capture passwords. -A directory listing provides an attacker with the complete index of all the resources located inside of the directory. The specific risks and consequences vary depending on which files are listed and accessible.",,Unknown,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" +A directory listing provides an attacker with the complete index of all the resources located inside of the directory. The specific risks and consequences vary depending on which files are listed and accessible.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" CWE-55,EN-Path Equivalence: /./ (Single Dot Directory) (Type: Variant),"A software system that accepts path input in the form of single dot directory exploit ('/./') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -A directory listing provides an attacker with the complete index of all the resources located inside of the directory. The specific risks and consequences vary depending on which files are listed and accessible.",,Unknown, +A directory listing provides an attacker with the complete index of all the resources located inside of the directory. The specific risks and consequences vary depending on which files are listed and accessible.",,unclassified, CWE-550,EN-Information Exposure Through Server Error Message (Type: Variant),"Certain conditions, such as network failure, will cause a server error message to be displayed. -While error messages in and of themselves are not dangerous, per se, it is what an attacker can glean from them that might cause eventual problems.",,Unknown, +While error messages in and of themselves are not dangerous, per se, it is what an attacker can glean from them that might cause eventual problems.",,unclassified, CWE-551,EN-Incorrect Behavior Order: Authorization Before Parsing and Canonicalization (Type: Base),"If a web server does not fully parse requested URLs before it examines them for authorization, it may be possible for an attacker to bypass authorization protection. -For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,Unknown, +For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,unclassified, CWE-552,EN-Files or Directories Accessible to External Parties (Type: Base),"Files or directories are accessible in the environment that should not be. -For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,Unknown, +For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,unclassified, CWE-553,EN-Command Shell in Externally Accessible Directory (Type: Variant),"A possible shell file exists in /cgi-bin/ or other accessible directories. This is extremely dangerous and can be used by an attacker to execute commands on the web server. -For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,Unknown, +For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,unclassified, CWE-554,EN-ASP.NET Misconfiguration: Not Using Input Validation Framework (Type: Variant),"The ASP.NET application does not use an input validation framework. -For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,Unknown, +For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,unclassified, CWE-555,EN-J2EE Misconfiguration: Plaintext Password in Configuration File (Type: Variant),"The J2EE application stores a plaintext password in a configuration file. -Storing a plaintext password in a configuration file allows anyone who can read the file to access the password-protected resource, making it an easy target for attackers.",,Unknown, +Storing a plaintext password in a configuration file allows anyone who can read the file to access the password-protected resource, making it an easy target for attackers.",,unclassified, CWE-556,EN-ASP.NET Misconfiguration: Use of Identity Impersonation (Type: Variant),"Configuring an ASP.NET application to run with impersonated credentials may give the application unnecessary privileges. -The use of impersonated credentials allows an ASP.NET application to run with either the privileges of the client on whose behalf it is executing or with arbitrary privileges granted in its configuration.",,Unknown, +The use of impersonated credentials allows an ASP.NET application to run with either the privileges of the client on whose behalf it is executing or with arbitrary privileges granted in its configuration.",,unclassified, CWE-558,EN-Use of getlogin() in Multithreaded Application (Type: Variant),"The application uses the getlogin() function in a multithreaded context, potentially causing it to return incorrect values. -The getlogin() function returns a pointer to a string that contains the name of the user associated with the calling process. The function is not reentrant, meaning that if it is called from another process, the contents are not locked out and the value of the string can be changed by another process. This makes it very risky to use because the username can be changed by other processes, so the results of the function cannot be trusted.",,Unknown, +The getlogin() function returns a pointer to a string that contains the name of the user associated with the calling process. The function is not reentrant, meaning that if it is called from another process, the contents are not locked out and the value of the string can be changed by another process. This makes it very risky to use because the username can be changed by other processes, so the results of the function cannot be trusted.",,unclassified, CWE-56,EN-Path Equivalence: filedir* (Wildcard) (Type: Variant),"A software system that accepts path input in the form of asterisk wildcard ('filedir*') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -The getlogin() function returns a pointer to a string that contains the name of the user associated with the calling process. The function is not reentrant, meaning that if it is called from another process, the contents are not locked out and the value of the string can be changed by another process. This makes it very risky to use because the username can be changed by other processes, so the results of the function cannot be trusted.",,Unknown, +The getlogin() function returns a pointer to a string that contains the name of the user associated with the calling process. The function is not reentrant, meaning that if it is called from another process, the contents are not locked out and the value of the string can be changed by another process. This makes it very risky to use because the username can be changed by other processes, so the results of the function cannot be trusted.",,unclassified, CWE-560,EN-Use of umask() with chmod-style Argument (Type: Variant),"The product calls umask() with an incorrect argument that is specified as if it is an argument to chmod(). -The getlogin() function returns a pointer to a string that contains the name of the user associated with the calling process. The function is not reentrant, meaning that if it is called from another process, the contents are not locked out and the value of the string can be changed by another process. This makes it very risky to use because the username can be changed by other processes, so the results of the function cannot be trusted.",,Unknown, +The getlogin() function returns a pointer to a string that contains the name of the user associated with the calling process. The function is not reentrant, meaning that if it is called from another process, the contents are not locked out and the value of the string can be changed by another process. This makes it very risky to use because the username can be changed by other processes, so the results of the function cannot be trusted.",,unclassified, CWE-561,EN-Dead Code (Type: Variant),"The software contains dead code, which can never be executed. -Dead code is source code that can never be executed in a running program. The surrounding code makes it impossible for a section of code to ever be executed.",,Unknown, +Dead code is source code that can never be executed in a running program. The surrounding code makes it impossible for a section of code to ever be executed.",,unclassified, CWE-562,EN-Return of Stack Variable Address (Type: Base),"A function returns the address of a stack variable, which will cause unintended program behavior, typically in the form of a crash. -Dead code is source code that can never be executed in a running program. The surrounding code makes it impossible for a section of code to ever be executed.",,Unknown, +Dead code is source code that can never be executed in a running program. The surrounding code makes it impossible for a section of code to ever be executed.",,unclassified, CWE-563,EN-Unused Variable (Type: Variant),"The variable's value is assigned but never used, making it a dead store. -It is likely that the variable is simply vestigial, but it is also possible that the unused variable points out a bug.",,Unknown, +It is likely that the variable is simply vestigial, but it is also possible that the unused variable points out a bug.",,unclassified, CWE-564,EN-SQL Injection: Hibernate (Type: Variant),"Using Hibernate to execute a dynamic SQL statement built with user-controlled input can allow an attacker to modify the statement's meaning or to execute arbitrary SQL commands. -It is likely that the variable is simply vestigial, but it is also possible that the unused variable points out a bug.",,Unknown, +It is likely that the variable is simply vestigial, but it is also possible that the unused variable points out a bug.",,unclassified, CWE-565,EN-Reliance on Cookies without Validation and Integrity Checking (Type: Base),"The application relies on the existence or values of cookies when performing security-critical operations, but it does not properly ensure that the setting is valid for the associated user. -Attackers can easily modify cookies, within the browser or by implementing the client-side code outside of the browser. Reliance on cookies without detailed validation and integrity checking can allow attackers to bypass authentication, conduct injection attacks such as SQL injection and cross-site scripting, or otherwise modify inputs in unexpected ways.",,Unknown, +Attackers can easily modify cookies, within the browser or by implementing the client-side code outside of the browser. Reliance on cookies without detailed validation and integrity checking can allow attackers to bypass authentication, conduct injection attacks such as SQL injection and cross-site scripting, or otherwise modify inputs in unexpected ways.",,unclassified, CWE-566,EN-Authorization Bypass Through User-Controlled SQL Primary Key (Type: Variant),"The software uses a database table that includes records that should not be accessible to an actor, but it executes a SQL statement with a primary key that can be controlled by that actor. When a user can set a primary key to any value, then the user can modify the key to point to unauthorized records. Database access control errors occur when: Data enters a program from an untrusted source. The data is used to specify the value of a primary key in a SQL query. -The untrusted source does not have the permissions to be able to access all rows in the associated table.",,Unknown, +The untrusted source does not have the permissions to be able to access all rows in the associated table.",,unclassified, CWE-567,EN-Unsynchronized Access to Shared Data in a Multithreaded Context (Type: Base),"The product does not properly synchronize shared data, such as static variables across threads, which can lead to undefined behavior and unpredictable data changes. Within servlets, shared static variables are not protected from concurrent access, but servlets are multithreaded. This is a typical programming mistake in J2EE applications, since the multithreading is handled by the framework. When a shared variable can be influenced by an attacker, one thread could wind up modifying the variable to contain data that is not valid for a different thread that is also using the data within the variable. -Note that this weakness is not unique to servlets.",,Unknown, +Note that this weakness is not unique to servlets.",,unclassified, CWE-568,EN-finalize() Method Without super.finalize() (Type: Variant),"The software contains a finalize() method that does not call super.finalize(). -The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,Unknown, +The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,unclassified, CWE-57,EN-Path Equivalence: fakedir/../realdir/filename (Type: Variant),"The software contains protection mechanisms to restrict access to 'realdir/filename', but it constructs pathnames using external input in the form of 'fakedir/../realdir/filename' that are not handled by those mechanisms. This allows attackers to perform unauthorized actions against the targeted file. -The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,Unknown, +The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,unclassified, CWE-570,EN-Expression is Always False (Type: Variant),"The software contains an expression that will always evaluate to false. -The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,Unknown, +The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,unclassified, CWE-571,EN-Expression is Always True (Type: Variant),"The software contains an expression that will always evaluate to true. -The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,Unknown, +The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,unclassified, CWE-572,EN-Call to Thread run() instead of start() (Type: Variant),"The program calls a thread's run() method instead of calling start(), which causes the code to run in the thread of the caller instead of the callee. -In most cases a direct call to a Thread object's run() method is a bug. The programmer intended to begin a new thread of control, but accidentally called run() instead of start(), so the run() method will execute in the caller's thread of control.",,Unknown, +In most cases a direct call to a Thread object's run() method is a bug. The programmer intended to begin a new thread of control, but accidentally called run() instead of start(), so the run() method will execute in the caller's thread of control.",,unclassified, CWE-573,EN-Improper Following of Specification by Caller (Type: Class),"The software does not follow or incorrectly follows the specifications as required by the implementation language, environment, framework, protocol, or platform. -When leveraging external functionality, such as an API, it is important that the caller does so in accordance with the requirements of the external functionality or else unintended behaviors may result, possibly leaving the system vulnerable to any number of exploits.",,Unknown, +When leveraging external functionality, such as an API, it is important that the caller does so in accordance with the requirements of the external functionality or else unintended behaviors may result, possibly leaving the system vulnerable to any number of exploits.",,unclassified, CWE-574,EN-EJB Bad Practices: Use of Synchronization Primitives (Type: Variant),"The program violates the Enterprise JavaBeans (EJB) specification by using thread synchronization primitives. -The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not use thread synchronization primitives to synchronize execution of multiple instances."" The specification justifies this requirement in the following way: ""This rule is required to ensure consistent runtime semantics because while some EJB containers may use a single JVM to execute all enterprise bean's instances, others may distribute the instances across multiple JVMs.""",,Unknown, +The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not use thread synchronization primitives to synchronize execution of multiple instances."" The specification justifies this requirement in the following way: ""This rule is required to ensure consistent runtime semantics because while some EJB containers may use a single JVM to execute all enterprise bean's instances, others may distribute the instances across multiple JVMs.""",,unclassified, CWE-575,EN-EJB Bad Practices: Use of AWT Swing (Type: Variant),"The program violates the Enterprise JavaBeans (EJB) specification by using AWT/Swing. -The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not use the AWT functionality to attempt to output information to a display, or to input information from a keyboard."" The specification justifies this requirement in the following way: ""Most servers do not allow direct interaction between an application program and a keyboard/display attached to the server system.""",,Unknown, +The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not use the AWT functionality to attempt to output information to a display, or to input information from a keyboard."" The specification justifies this requirement in the following way: ""Most servers do not allow direct interaction between an application program and a keyboard/display attached to the server system.""",,unclassified, CWE-576,EN-EJB Bad Practices: Use of Java I/O (Type: Variant),"The program violates the Enterprise JavaBeans (EJB) specification by using the java.io package. -The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not use the java.io package to attempt to access files and directories in the file system."" The specification justifies this requirement in the following way: ""The file system APIs are not well-suited for business components to access data. Business components should use a resource manager API, such as JDBC, to store data.""",,Unknown, +The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not use the java.io package to attempt to access files and directories in the file system."" The specification justifies this requirement in the following way: ""The file system APIs are not well-suited for business components to access data. Business components should use a resource manager API, such as JDBC, to store data.""",,unclassified, CWE-577,EN-EJB Bad Practices: Use of Sockets (Type: Variant),"The program violates the Enterprise JavaBeans (EJB) specification by using sockets. -The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not attempt to listen on a socket, accept connections on a socket, or use a socket for multicast."" The specification justifies this requirement in the following way: ""The EJB architecture allows an enterprise bean instance to be a network socket client, but it does not allow it to be a network server. Allowing the instance to become a network server would conflict with the basic function of the enterprise bean-- to serve the EJB clients.""",,Unknown, +The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not attempt to listen on a socket, accept connections on a socket, or use a socket for multicast."" The specification justifies this requirement in the following way: ""The EJB architecture allows an enterprise bean instance to be a network socket client, but it does not allow it to be a network server. Allowing the instance to become a network server would conflict with the basic function of the enterprise bean-- to serve the EJB clients.""",,unclassified, CWE-578,EN-EJB Bad Practices: Use of Class Loader (Type: Variant),"The program violates the Enterprise JavaBeans (EJB) specification by using the class loader. -The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""The enterprise bean must not attempt to create a class loader; obtain the current class loader; set the context class loader; set security manager; create a new security manager; stop the JVM; or change the input, output, and error streams."" The specification justifies this requirement in the following way: ""These functions are reserved for the EJB container. Allowing the enterprise bean to use these functions could compromise security and decrease the container's ability to properly manage the runtime environment.""",,Unknown, +The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""The enterprise bean must not attempt to create a class loader; obtain the current class loader; set the context class loader; set security manager; create a new security manager; stop the JVM; or change the input, output, and error streams."" The specification justifies this requirement in the following way: ""These functions are reserved for the EJB container. Allowing the enterprise bean to use these functions could compromise security and decrease the container's ability to properly manage the runtime environment.""",,unclassified, CWE-579,EN-J2EE Bad Practices: Non-serializable Object Stored in Session (Type: Variant),"The application stores a non-serializable object as an HttpSession attribute, which can hurt reliability. -The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""The enterprise bean must not attempt to create a class loader; obtain the current class loader; set the context class loader; set security manager; create a new security manager; stop the JVM; or change the input, output, and error streams."" The specification justifies this requirement in the following way: ""These functions are reserved for the EJB container. Allowing the enterprise bean to use these functions could compromise security and decrease the container's ability to properly manage the runtime environment.""",,Unknown, +The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""The enterprise bean must not attempt to create a class loader; obtain the current class loader; set the context class loader; set security manager; create a new security manager; stop the JVM; or change the input, output, and error streams."" The specification justifies this requirement in the following way: ""These functions are reserved for the EJB container. Allowing the enterprise bean to use these functions could compromise security and decrease the container's ability to properly manage the runtime environment.""",,unclassified, CWE-58,EN-Path Equivalence: Windows 8.3 Filename (Type: Variant),"The software contains a protection mechanism that restricts access to a long filename on a Windows operating system, but the software does not properly restrict access to the equivalent short ""8.3"" filename. -On later Windows operating systems, a file can have a ""long name"" and a short name that is compatible with older Windows file systems, with up to 8 characters in the filename and 3 characters for the extension. These ""8.3"" filenames, therefore, act as an alternate name for files with long names, so they are useful pathname equivalence manipulations.",,Unknown,"Writing Secure Code +On later Windows operating systems, a file can have a ""long name"" and a short name that is compatible with older Windows file systems, with up to 8 characters in the filename and 3 characters for the extension. These ""8.3"" filenames, therefore, act as an alternate name for files with long names, so they are useful pathname equivalence manipulations.",,unclassified,"Writing Secure Code The Art of Software Security Assessment: Chapter 11, ""DOS 8.3 Filenames"", Page 673." CWE-580,EN-clone() Method Without super.clone() (Type: Variant),"The software contains a clone() method that does not call super.clone() to obtain the new object. -All implementations of clone() should obtain the new object by calling super.clone(). If a class does not follow this convention, a subclass's clone() method will return an object of the wrong type.",,Unknown, +All implementations of clone() should obtain the new object by calling super.clone(). If a class does not follow this convention, a subclass's clone() method will return an object of the wrong type.",,unclassified, CWE-581,EN-Object Model Violation: Just One of Equals and Hashcode Defined (Type: Base),"The software does not maintain equal hashcodes for equal objects. -Java objects are expected to obey a number of invariants related to equality. One of these invariants is that equal objects must have equal hashcodes. In other words, if a.equals(b) == true then a.hashCode() == b.hashCode().",,Unknown, +Java objects are expected to obey a number of invariants related to equality. One of these invariants is that equal objects must have equal hashcodes. In other words, if a.equals(b) == true then a.hashCode() == b.hashCode().",,unclassified, CWE-582,"EN-Array Declared Public, Final, and Static (Type: Variant)","The program declares an array public, final, and static, which is not sufficient to prevent the array's contents from being modified. -Because arrays are mutable objects, the final constraint requires that the array object itself be assigned only once, but makes no guarantees about the values of the array elements. Since the array is public, a malicious program can change the values stored in the array. As such, in most cases an array declared public, final and static is a bug.",,Unknown, +Because arrays are mutable objects, the final constraint requires that the array object itself be assigned only once, but makes no guarantees about the values of the array elements. Since the array is public, a malicious program can change the values stored in the array. As such, in most cases an array declared public, final and static is a bug.",,unclassified, CWE-583,EN-finalize() Method Declared Public (Type: Variant),"The program violates secure coding principles for mobile code by declaring a finalize() method public. -A program should never call finalize explicitly, except to call super.finalize() inside an implementation of finalize(). In mobile code situations, the otherwise error prone practice of manual garbage collection can become a security threat if an attacker can maliciously invoke one of your finalize() methods because it is declared with public access.",,Unknown, +A program should never call finalize explicitly, except to call super.finalize() inside an implementation of finalize(). In mobile code situations, the otherwise error prone practice of manual garbage collection can become a security threat if an attacker can maliciously invoke one of your finalize() methods because it is declared with public access.",,unclassified, CWE-584,EN-Return Inside Finally Block (Type: Base),"The code has a return statement inside a finally block, which will cause any thrown exception in the try block to be discarded. -A program should never call finalize explicitly, except to call super.finalize() inside an implementation of finalize(). In mobile code situations, the otherwise error prone practice of manual garbage collection can become a security threat if an attacker can maliciously invoke one of your finalize() methods because it is declared with public access.",,Unknown, +A program should never call finalize explicitly, except to call super.finalize() inside an implementation of finalize(). In mobile code situations, the otherwise error prone practice of manual garbage collection can become a security threat if an attacker can maliciously invoke one of your finalize() methods because it is declared with public access.",,unclassified, CWE-585,EN-Empty Synchronized Block (Type: Variant),"The software contains an empty synchronized block. -An empty synchronized block does not actually accomplish any synchronization and may indicate a troubled section of code. An empty synchronized block can occur because code no longer needed within the synchronized block is commented out without removing the synchronized block.",,Unknown,Intrinsic Locks and Synchronization (in Java): http://java.sun.com/docs/books/tutorial/essential/concurrency/locksync.html +An empty synchronized block does not actually accomplish any synchronization and may indicate a troubled section of code. An empty synchronized block can occur because code no longer needed within the synchronized block is commented out without removing the synchronized block.",,unclassified,Intrinsic Locks and Synchronization (in Java): http://java.sun.com/docs/books/tutorial/essential/concurrency/locksync.html CWE-586,EN-Explicit Call to Finalize() (Type: Variant),"The software makes an explicit call to the finalize() method from outside the finalizer. -While the Java Language Specification allows an object's finalize() method to be called from outside the finalizer, doing so is usually a bad idea. For example, calling finalize() explicitly means that finalize() will be called more than once: the first time will be the explicit call and the last time will be the call that is made after the object is garbage collected.",,Unknown, +While the Java Language Specification allows an object's finalize() method to be called from outside the finalizer, doing so is usually a bad idea. For example, calling finalize() explicitly means that finalize() will be called more than once: the first time will be the explicit call and the last time will be the call that is made after the object is garbage collected.",,unclassified, CWE-587,EN-Assignment of a Fixed Address to a Pointer (Type: Base),"The software sets a pointer to a specific address other than NULL or 0. -Using a fixed address is not portable because that address will probably not be valid in all environments or platforms.",,Unknown, +Using a fixed address is not portable because that address will probably not be valid in all environments or platforms.",,unclassified, CWE-588,EN-Attempt to Access Child of a Non-structure Pointer (Type: Variant),"Casting a non-structure type to a structure type and accessing a field can lead to memory access errors or data corruption. -Using a fixed address is not portable because that address will probably not be valid in all environments or platforms.",,Unknown, +Using a fixed address is not portable because that address will probably not be valid in all environments or platforms.",,unclassified, CWE-589,EN-Call to Non-ubiquitous API (Type: Variant),"The software uses an API function that does not exist on all versions of the target platform. This could cause portability problems or inconsistencies that allow denial of service or other consequences. -Some functions that offer security features supported by the OS are not available on all versions of the OS in common use. Likewise, functions are often deprecated or made obsolete for security reasons and should not be used.",,Unknown, +Some functions that offer security features supported by the OS are not available on all versions of the OS in common use. Likewise, functions are often deprecated or made obsolete for security reasons and should not be used.",,unclassified, CWE-590,EN-Free of Memory not on the Heap (Type: Variant),"The application calls free() on a pointer to memory that was not allocated using associated heap allocation functions such as malloc(), calloc(), or realloc(). -When free() is called on an invalid pointer, the program's memory management data structures may become corrupted. This corruption can cause the program to crash or, in some circumstances, an attacker may be able to cause free() to operate on controllable memory locations to modify critical program variables or execute code.",,Unknown,Valgrind: http://valgrind.org/ +When free() is called on an invalid pointer, the program's memory management data structures may become corrupted. This corruption can cause the program to crash or, in some circumstances, an attacker may be able to cause free() to operate on controllable memory locations to modify critical program variables or execute code.",,unclassified,Valgrind: http://valgrind.org/ CWE-591,EN-Sensitive Data Storage in Improperly Locked Memory (Type: Variant),"The application stores sensitive data in memory that is not locked, or that has been incorrectly locked, which might cause the memory to be written to swap files on disk by the virtual memory manager. This can make the data more accessible to external actors. -On Windows systems the VirtualLock function can lock a page of memory to ensure that it will remain present in memory and not be swapped to disk. However, on older versions of Windows, such as 95, 98, or Me, the VirtualLock() function is only a stub and provides no protection. On POSIX systems the mlock() call ensures that a page will stay resident in memory but does not guarantee that the page will not appear in the swap. Therefore, it is unsuitable for use as a protection mechanism for sensitive data. Some platforms, in particular Linux, do make the guarantee that the page will not be swapped, but this is non-standard and is not portable. Calls to mlock() also require supervisor privilege. Return values for both of these calls must be checked to ensure that the lock operation was actually successful.",,Unknown, +On Windows systems the VirtualLock function can lock a page of memory to ensure that it will remain present in memory and not be swapped to disk. However, on older versions of Windows, such as 95, 98, or Me, the VirtualLock() function is only a stub and provides no protection. On POSIX systems the mlock() call ensures that a page will stay resident in memory but does not guarantee that the page will not appear in the swap. Therefore, it is unsuitable for use as a protection mechanism for sensitive data. Some platforms, in particular Linux, do make the guarantee that the page will not be swapped, but this is non-standard and is not portable. Calls to mlock() also require supervisor privilege. Return values for both of these calls must be checked to ensure that the lock operation was actually successful.",,unclassified, CWE-592,EN-Authentication Bypass Issues (Type: Class),"The software does not properly perform authentication, allowing it to be bypassed through various methods. -On Windows systems the VirtualLock function can lock a page of memory to ensure that it will remain present in memory and not be swapped to disk. However, on older versions of Windows, such as 95, 98, or Me, the VirtualLock() function is only a stub and provides no protection. On POSIX systems the mlock() call ensures that a page will stay resident in memory but does not guarantee that the page will not appear in the swap. Therefore, it is unsuitable for use as a protection mechanism for sensitive data. Some platforms, in particular Linux, do make the guarantee that the page will not be swapped, but this is non-standard and is not portable. Calls to mlock() also require supervisor privilege. Return values for both of these calls must be checked to ensure that the lock operation was actually successful.",,Unknown,"The Art of Software Security Assessment: Chapter 2, ""Untrustworthy Credentials"", Page 37." +On Windows systems the VirtualLock function can lock a page of memory to ensure that it will remain present in memory and not be swapped to disk. However, on older versions of Windows, such as 95, 98, or Me, the VirtualLock() function is only a stub and provides no protection. On POSIX systems the mlock() call ensures that a page will stay resident in memory but does not guarantee that the page will not appear in the swap. Therefore, it is unsuitable for use as a protection mechanism for sensitive data. Some platforms, in particular Linux, do make the guarantee that the page will not be swapped, but this is non-standard and is not portable. Calls to mlock() also require supervisor privilege. Return values for both of these calls must be checked to ensure that the lock operation was actually successful.",,unclassified,"The Art of Software Security Assessment: Chapter 2, ""Untrustworthy Credentials"", Page 37." CWE-593,EN-Authentication Bypass: OpenSSL CTX Object Modified after SSL Objects are Created (Type: Variant),"The software modifies the SSL context after connection creation has begun. -If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,Unknown, +If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,unclassified, CWE-594,EN-J2EE Framework: Saving Unserializable Objects to Disk (Type: Variant),"When the J2EE container attempts to write unserializable objects to disk there is no guarantee that the process will complete successfully. -If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,Unknown, +If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,unclassified, CWE-595,EN-Comparison of Object References Instead of Object Contents (Type: Base),"The program compares object references instead of the contents of the objects themselves, preventing it from detecting equivalent objects. -If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,Unknown, +If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,unclassified, CWE-596,EN-Incorrect Semantic Object Comparison (Type: Base),"The software does not correctly compare two objects based on their conceptual content. -If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,Unknown, +If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,unclassified, CWE-597,EN-Use of Wrong Operator in String Comparison (Type: Variant),"The product uses the wrong operator when comparing a string, such as using ""=="" when the equals() method should be used instead. -In Java, using == or != to compare two strings for equality actually compares two objects for equality, not their values. Chances are good that the two references will never be equal. While this weakness often only affects program correctness, if the equality is used for a security decision, it could be leveraged to affect program security.",,Unknown,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289." +In Java, using == or != to compare two strings for equality actually compares two objects for equality, not their values. Chances are good that the two references will never be equal. While this weakness often only affects program correctness, if the equality is used for a security decision, it could be leveraged to affect program security.",,unclassified,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289." CWE-598,EN-Information Exposure Through Query Strings in GET Request (Type: Variant),"The web application uses the GET method to process requests that contain sensitive information, which can expose that information through the browser's history, Referers, web logs, and other sources. -In Java, using == or != to compare two strings for equality actually compares two objects for equality, not their values. Chances are good that the two references will never be equal. While this weakness often only affects program correctness, if the equality is used for a security decision, it could be leveraged to affect program security.",,Unknown, +In Java, using == or != to compare two strings for equality actually compares two objects for equality, not their values. Chances are good that the two references will never be equal. While this weakness often only affects program correctness, if the equality is used for a security decision, it could be leveraged to affect program security.",,unclassified, CWE-599,EN-Missing Validation of OpenSSL Certificate (Type: Variant),"The software uses OpenSSL and trusts or uses a certificate without using the SSL_get_verify_result() function to ensure that the certificate satisfies all necessary security requirements. -This could allow an attacker to use an invalid certificate to claim to be a trusted host, use expired certificates, or conduct other attacks that could be detected if the certificate is properly validated.",,Unknown, +This could allow an attacker to use an invalid certificate to claim to be a trusted host, use expired certificates, or conduct other attacks that could be detected if the certificate is properly validated.",,unclassified, CWE-6,EN-J2EE Misconfiguration: Insufficient Session-ID Length (Type: Variant),"The J2EE application is configured to use an insufficient session ID length. -If an attacker can guess or steal a session ID, then he/she may be able to take over the user's session (called session hijacking). The number of possible session IDs increases with increased session ID length, making it more difficult to guess or steal a session ID.",,Unknown,No description: http://www.securiteam.com/securityreviews/5TP0F0UEVQ.html +If an attacker can guess or steal a session ID, then he/she may be able to take over the user's session (called session hijacking). The number of possible session IDs increases with increased session ID length, making it more difficult to guess or steal a session ID.",,unclassified,No description: http://www.securiteam.com/securityreviews/5TP0F0UEVQ.html CWE-600,EN-Uncaught Exception in Servlet (Type: Base),"The Servlet does not catch all exceptions, which may reveal sensitive debugging information. -When a Servlet throws an exception, the default error response the Servlet container sends back to the user typically includes debugging information. This information is of great value to an attacker. For example, a stack trace might show the attacker a malformed SQL query string, the type of database being used, and the version of the application container. This information enables the attacker to target known vulnerabilities in these components.",,Unknown, +When a Servlet throws an exception, the default error response the Servlet container sends back to the user typically includes debugging information. This information is of great value to an attacker. For example, a stack trace might show the attacker a malformed SQL query string, the type of database being used, and the version of the application container. This information enables the attacker to target known vulnerabilities in these components.",,unclassified, CWE-603,EN-Use of Client-Side Authentication (Type: Base),"A client/server product performs authentication within client code but not in server code, allowing server-side authentication to be bypassed via a modified client that omits the authentication check. -Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,Unknown,"The Art of Software Security Assessment: Chapter 2, ""Untrustworthy Credentials"", Page 37." +Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified,"The Art of Software Security Assessment: Chapter 2, ""Untrustworthy Credentials"", Page 37." CWE-605,EN-Multiple Binds to the Same Port (Type: Base),"When multiple sockets are allowed to bind to the same port, other services on that port may be stolen or spoofed. -Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,Unknown, +Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified, CWE-606,EN-Unchecked Input for Loop Condition (Type: Base),"The product does not properly check inputs that are used for loop conditions, potentially leading to a denial of service because of excessive looping. -Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,Unknown,"The Art of Software Security Assessment: Chapter 7, ""Looping Constructs"", Page 327." +Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified,"The Art of Software Security Assessment: Chapter 7, ""Looping Constructs"", Page 327." CWE-607,EN-Public Static Final Field References Mutable Object (Type: Variant),"A public or protected static final field references a mutable object, which allows the object to be changed by malicious code, or accidentally from another package. -Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,Unknown, +Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified, CWE-608,EN-Struts: Non-private Field in ActionForm Class (Type: Variant),"An ActionForm class contains a field that has not been declared private, which can be accessed without using a setter or getter. -Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,Unknown, +Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified, CWE-609,EN-Double-Checked Locking (Type: Base),"The program uses double-checked locking to access a resource without the overhead of explicit synchronization, but the locking is insufficient. -Double-checked locking refers to the situation where a programmer checks to see if a resource has been initialized, grabs a lock, checks again to see if the resource has been initialized, and then performs the initialization if it has not occurred yet. This should not be done, as is not guaranteed to work in all languages and on all architectures. In summary, other threads may not be operating inside the synchronous block and are not guaranteed to see the operations execute in the same order as they would appear inside the synchronous block.",,Unknown,"The ""Double-Checked Locking is Broken"" Declaration: http://www.cs.umd.edu/~pugh/java/memoryModel/DoubleCheckedLocking.html +Double-checked locking refers to the situation where a programmer checks to see if a resource has been initialized, grabs a lock, checks again to see if the resource has been initialized, and then performs the initialization if it has not occurred yet. This should not be done, as is not guaranteed to work in all languages and on all architectures. In summary, other threads may not be operating inside the synchronous block and are not guaranteed to see the operations execute in the same order as they would appear inside the synchronous block.",,unclassified,"The ""Double-Checked Locking is Broken"" Declaration: http://www.cs.umd.edu/~pugh/java/memoryModel/DoubleCheckedLocking.html JSR 133 (Java Memory Model) FAQ: http://www.cs.umd.edu/~pugh/java/memoryModel/jsr-133-faq.html#dcl The Art of Software Security Assessment: Chapter 13, ""Threading Vulnerabilities"", Page 815." CWE-610,EN-Externally Controlled Reference to a Resource in Another Sphere (Type: Class),"The product uses an externally controlled name or reference that resolves to a resource that is outside of the intended control sphere. -",,Unknown, +",,unclassified, CWE-611,EN-Improper Restriction of XML External Entity Reference (XXE) (Type: Variant),"The software processes an XML document that can contain XML entities with URIs that resolve to documents outside of the intended sphere of control, causing the product to embed incorrect documents into its output. XML documents optionally contain a Document Type Definition (DTD), which, among other features, enables the definition of XML entities. It is possible to define an entity by providing a substitution string in the form of a URI. The XML parser can access the contents of this URI and embed these contents back into the XML document for further processing. By submitting an XML file that defines an external entity with a file:// URI, an attacker can cause the processing application to read the contents of a local file. For example, a URI such as ""file:///c:/winnt/win.ini"" designates (in Windows) the file C:\Winnt\win.ini, or file:///etc/passwd designates the password file in Unix-based systems. Using URIs with other schemes such as http://, the attacker can force the application to make outgoing requests to servers that the attacker cannot reach directly, which can be used to bypass firewall restrictions or hide the source of attacks such as port scanning. -Once the content of the URI is read, it is fed back into the application that is processing the XML. This application may echo back the data (e.g. in an error message), thereby exposing the file contents.",,Unknown,"XML External Entity (XXE) Processing: https://www.owasp.org/index.php/XML_External_Entity_(XXE)_Processing +Once the content of the URI is read, it is fed back into the application that is processing the XML. This application may echo back the data (e.g. in an error message), thereby exposing the file contents.",,unclassified,"XML External Entity (XXE) Processing: https://www.owasp.org/index.php/XML_External_Entity_(XXE)_Processing XML External Entity Attacks (XXE): https://www.owasp.org/images/5/5d/XML_Exteral_Entity_Attack.pdf XXE (Xml eXternal Entity) Attack: http://www.securiteam.com/securitynews/6D0100A5PU.html XML External Entities (XXE) Attack: http://projects.webappsec.org/w/page/13247003/XML%20External%20Entities XML Denial of Service Attacks and Defenses: http://msdn.microsoft.com/en-us/magazine/ee335713.aspx Preventing XXE in PHP: http://websec.io/2012/08/27/Preventing-XEE-in-PHP.html" CWE-612,EN-Information Exposure Through Indexing of Private Data (Type: Variant),"The product performs an indexing routine against private documents, but does not sufficiently verify that the actors who can access the index also have the privileges to access the private documents. -When an indexing routine is applied against a group of private documents, and that index's results are available to outsiders who do not have access to those documents, then outsiders might be able to obtain sensitive information by conducting targeted searches. The risk is especially dangerous if search results include surrounding text that was not part of the search query. This issue can appear in search engines that are not configured (or implemented) to ignore critical files that should remain hidden; even without permissions to download these files directly, the remote user could read them.",,Unknown, +When an indexing routine is applied against a group of private documents, and that index's results are available to outsiders who do not have access to those documents, then outsiders might be able to obtain sensitive information by conducting targeted searches. The risk is especially dangerous if search results include surrounding text that was not part of the search query. This issue can appear in search engines that are not configured (or implemented) to ignore critical files that should remain hidden; even without permissions to download these files directly, the remote user could read them.",,unclassified, CWE-613,EN-Insufficient Session Expiration (Type: Base),"According to WASC, ""Insufficient Session Expiration is when a web site permits an attacker to reuse old session credentials or session IDs for authorization."" -When an indexing routine is applied against a group of private documents, and that index's results are available to outsiders who do not have access to those documents, then outsiders might be able to obtain sensitive information by conducting targeted searches. The risk is especially dangerous if search results include surrounding text that was not part of the search query. This issue can appear in search engines that are not configured (or implemented) to ignore critical files that should remain hidden; even without permissions to download these files directly, the remote user could read them.",,Unknown, +When an indexing routine is applied against a group of private documents, and that index's results are available to outsiders who do not have access to those documents, then outsiders might be able to obtain sensitive information by conducting targeted searches. The risk is especially dangerous if search results include surrounding text that was not part of the search query. This issue can appear in search engines that are not configured (or implemented) to ignore critical files that should remain hidden; even without permissions to download these files directly, the remote user could read them.",,unclassified, CWE-614,EN-Sensitive Cookie in HTTPS Session Without Secure Attribute (Type: Variant),"The Secure attribute for sensitive cookies in HTTPS sessions is not set, which could cause the user agent to send those cookies in plaintext over an HTTP session. -When an indexing routine is applied against a group of private documents, and that index's results are available to outsiders who do not have access to those documents, then outsiders might be able to obtain sensitive information by conducting targeted searches. The risk is especially dangerous if search results include surrounding text that was not part of the search query. This issue can appear in search engines that are not configured (or implemented) to ignore critical files that should remain hidden; even without permissions to download these files directly, the remote user could read them.",,Unknown, +When an indexing routine is applied against a group of private documents, and that index's results are available to outsiders who do not have access to those documents, then outsiders might be able to obtain sensitive information by conducting targeted searches. The risk is especially dangerous if search results include surrounding text that was not part of the search query. This issue can appear in search engines that are not configured (or implemented) to ignore critical files that should remain hidden; even without permissions to download these files directly, the remote user could read them.",,unclassified, CWE-615,EN-Information Exposure Through Comments (Type: Variant),"While adding general comments is very useful, some programmers tend to leave important data, such as: filenames related to the web application, old links or links which were not meant to be browsed by users, old code fragments, etc. -An attacker who finds these comments can map the application's structure and files, expose hidden parts of the site, and study the fragments of code to reverse engineer the application, which may help develop further attacks against the site.",,Unknown, +An attacker who finds these comments can map the application's structure and files, expose hidden parts of the site, and study the fragments of code to reverse engineer the application, which may help develop further attacks against the site.",,unclassified, CWE-616,EN-Incomplete Identification of Uploaded File Variables (PHP) (Type: Variant),"The PHP application uses an old method for processing uploaded files by referencing the four global variables that are set for each file (e.g. $varname, $varname_size, $varname_name, $varname_type). These variables could be overwritten by attackers, causing the application to process unauthorized files. -These global variables could be overwritten by POST requests, cookies, or other methods of populating or overwriting these variables. This could be used to read or process arbitrary files by providing values such as ""/etc/passwd"".",,Unknown,"A Study in Scarlet - section 5, ""File Upload""" +These global variables could be overwritten by POST requests, cookies, or other methods of populating or overwriting these variables. This could be used to read or process arbitrary files by providing values such as ""/etc/passwd"".",,unclassified,"A Study in Scarlet - section 5, ""File Upload""" CWE-617,EN-Reachable Assertion (Type: Variant),"The product contains an assert() or similar statement that can be triggered by an attacker, which leads to an application exit or other behavior that is more severe than necessary. -For example, if a server handles multiple simultaneous connections, and an assert() occurs in one single connection that causes all other connections to be dropped, this is a reachable assertion that leads to a denial of service.",,Unknown, +For example, if a server handles multiple simultaneous connections, and an assert() occurs in one single connection that causes all other connections to be dropped, this is a reachable assertion that leads to a denial of service.",,unclassified, CWE-618,EN-Exposed Unsafe ActiveX Method (Type: Base),"An ActiveX control is intended for use in a web browser, but it exposes dangerous methods that perform actions that are outside of the browser's security model (e.g. the zone or domain). -ActiveX controls can exercise far greater control over the operating system than typical Java or javascript. Exposed methods can be subject to various vulnerabilities, depending on the implemented behaviors of those methods, and whether input validation is performed on the provided arguments. If there is no integrity checking or origin validation, this method could be invoked by attackers.",,Unknown,"No description: http://msdn.microsoft.com/workshop/components/activex/safety.asp +ActiveX controls can exercise far greater control over the operating system than typical Java or javascript. Exposed methods can be subject to various vulnerabilities, depending on the implemented behaviors of those methods, and whether input validation is performed on the provided arguments. If there is no integrity checking or origin validation, this method could be invoked by attackers.",,unclassified,"No description: http://msdn.microsoft.com/workshop/components/activex/safety.asp No description: http://msdn.microsoft.com/workshop/components/activex/security.asp The Art of Software Security Assessment: Chapter 12, ""ActiveX Security"", Page 749." CWE-619,EN-Dangling Database Cursor (Cursor Injection) (Type: Base),"If a database cursor is not closed properly, then it could become accessible to other users while retaining the same privileges that were originally assigned, leaving the cursor ""dangling."" -For example, an improper dangling cursor could arise from unhandled exceptions. The impact of the issue depends on the cursor's role, but SQL injection attacks are commonly possible.",,Unknown,"The Oracle Hacker's Handbook +For example, an improper dangling cursor could arise from unhandled exceptions. The impact of the issue depends on the cursor's role, but SQL injection attacks are commonly possible.",,unclassified,"The Oracle Hacker's Handbook Cursor Injection: http://www.databasesecurity.com/dbsec/cursor-injection.pdf" CWE-62,EN-UNIX Hard Link (Type: Variant),"The software, when opening a file or directory, does not sufficiently account for when the name is associated with a hard link to a target that is outside of the intended control sphere. This could allow an attacker to cause the software to operate on unauthorized files. -Failure for a system to check for hard links can result in vulnerability to different types of attacks. For example, an attacker can escalate their privileges if a file used by a privileged program is replaced with a hard link to a sensitive file (e.g. /etc/passwd). When the process opens the file, the attacker can assume the privileges of that process.",,Unknown,"The Art of Software Security Assessment: Chapter 9, ""Hard Links"", Page 518." +Failure for a system to check for hard links can result in vulnerability to different types of attacks. For example, an attacker can escalate their privileges if a file used by a privileged program is replaced with a hard link to a sensitive file (e.g. /etc/passwd). When the process opens the file, the attacker can assume the privileges of that process.",,unclassified,"The Art of Software Security Assessment: Chapter 9, ""Hard Links"", Page 518." CWE-620,EN-Unverified Password Change (Type: Variant),"When setting a new password for a user, the product does not require knowledge of the original password, or using another form of authentication. -This could be used by an attacker to change passwords for another user, thus gaining the privileges associated with that user.",,Unknown,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" +This could be used by an attacker to change passwords for another user, thus gaining the privileges associated with that user.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" CWE-621,EN-Variable Extraction Error (Type: Base),"The product uses external input to determine the names of variables into which information is extracted, without verifying that the names of the specified variables are valid. This could cause the program to overwrite unintended variables. -For example, in PHP, calling extract() or import_request_variables() without the proper arguments could allow arbitrary global variables to be overwritten, including superglobals. Similar functionality might be possible in other interpreted languages, including custom languages.",,Unknown, +For example, in PHP, calling extract() or import_request_variables() without the proper arguments could allow arbitrary global variables to be overwritten, including superglobals. Similar functionality might be possible in other interpreted languages, including custom languages.",,unclassified, CWE-622,EN-Improper Validation of Function Hook Arguments (Type: Variant),"A product adds hooks to user-accessible API functions, but does not properly validate the arguments. This could lead to resultant vulnerabilities. -Such hooks can be used in defensive software that runs with privileges, such as anti-virus or firewall, which hooks kernel calls. When the arguments are not validated, they could be used to bypass the protection scheme or attack the product itself.",,Unknown, +Such hooks can be used in defensive software that runs with privileges, such as anti-virus or firewall, which hooks kernel calls. When the arguments are not validated, they could be used to bypass the protection scheme or attack the product itself.",,unclassified, CWE-623,EN-Unsafe ActiveX Control Marked Safe For Scripting (Type: Variant),"An ActiveX control is intended for restricted use, but it has been marked as safe-for-scripting. -This might allow attackers to use dangerous functionality via a web page that accesses the control, which can lead to different resultant vulnerabilities, depending on the control's behavior.",,Unknown,"No description: http://msdn.microsoft.com/workshop/components/activex/safety.asp +This might allow attackers to use dangerous functionality via a web page that accesses the control, which can lead to different resultant vulnerabilities, depending on the control's behavior.",,unclassified,"No description: http://msdn.microsoft.com/workshop/components/activex/safety.asp No description: http://msdn.microsoft.com/workshop/components/activex/security.asp No description: http://support.microsoft.com/kb/240797 Writing Secure Code: Chapter 16, ""What ActiveX Components Are Safe for Initialization and Safe for Scripting?"" Page 510 The Art of Software Security Assessment: Chapter 12, ""ActiveX Security"", Page 749." CWE-624,EN-Executable Regular Expression Error (Type: Base),"The product uses a regular expression that either (1) contains an executable component with user-controlled inputs, or (2) allows a user to enable execution by inserting pattern modifiers. -Case (2) is possible in the PHP preg_replace() function, and possibly in other languages when a user-controlled input is inserted into a string that is later parsed as a regular expression.",,Unknown, +Case (2) is possible in the PHP preg_replace() function, and possibly in other languages when a user-controlled input is inserted into a string that is later parsed as a regular expression.",,unclassified, CWE-625,EN-Permissive Regular Expression (Type: Base),"The product uses a regular expression that does not sufficiently restrict the set of allowed values. This effectively causes the regexp to accept substrings that match the pattern, which produces a partial comparison to the target. In some cases, this can lead to other weaknesses. Common errors include: not identifying the beginning and end of the target string using wildcards instead of acceptable character ranges -others",,Unknown,"The Art of Software Security Assessment: Chapter 8, ""Character Stripping Vulnerabilities"", Page 437." +others",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Character Stripping Vulnerabilities"", Page 437." CWE-626,EN-Null Byte Interaction Error (Poison Null Byte) (Type: Variant),"The product does not properly handle null bytes or NUL characters when passing data between different representations or components. -A null byte (NUL character) can have different meanings across representations or languages. For example, it is a string terminator in standard C libraries, but Perl and PHP strings do not treat it as a terminator. When two representations are crossed - such as when Perl or PHP invokes underlying C functionality - this can produce an interaction error with unexpected results. Similar issues have been reported for ASP. Other interpreters written in C might also be affected.",,Unknown,"Poison NULL byte: http://insecure.org/news/P55-07.txt +A null byte (NUL character) can have different meanings across representations or languages. For example, it is a string terminator in standard C libraries, but Perl and PHP strings do not treat it as a terminator. When two representations are crossed - such as when Perl or PHP invokes underlying C functionality - this can produce an interaction error with unexpected results. Similar issues have been reported for ASP. Other interpreters written in C might also be affected.",,unclassified,"Poison NULL byte: http://insecure.org/news/P55-07.txt 0x00 vs ASP file upload scripts: http://www.security-assessment.com/Whitepapers/0x00_vs_ASP_File_Uploads.pdf ShAnKaR: multiple PHP application poison NULL byte vulnerability: http://seclists.org/fulldisclosure/2006/Sep/0185.html" CWE-627,EN-Dynamic Variable Evaluation (Type: Base),"In a language where the user can influence the name of a variable at runtime, if the variable names are not controlled, an attacker can read or write to arbitrary variables, or access arbitrary functions. -The resultant vulnerabilities depend on the behavior of the application, both at the crossover point and in any control/data flow that is reachable by the related variables or functions.",,Unknown,"Dynamic Evaluation Vulnerabilities in PHP applications: http://seclists.org/fulldisclosure/2006/May/0035.html +The resultant vulnerabilities depend on the behavior of the application, both at the crossover point and in any control/data flow that is reachable by the related variables or functions.",,unclassified,"Dynamic Evaluation Vulnerabilities in PHP applications: http://seclists.org/fulldisclosure/2006/May/0035.html A Study In Scarlet: Exploiting Common Vulnerabilities in PHP Applications: http://www.securereality.com.au/studyinscarlet.txt" CWE-628,EN-Function Call with Incorrectly Specified Arguments (Type: Base),"The product calls a function, procedure, or routine with arguments that are not correctly specified, leading to always-incorrect behavior and resultant weaknesses. There are multiple ways in which this weakness can be introduced, including: @@ -1721,148 +1721,148 @@ the wrong variable or reference; an incorrect number of arguments; incorrect order of arguments; wrong type of arguments; or -wrong value.",,Unknown, +wrong value.",,unclassified, CWE-636,EN-Not Failing Securely (Failing Open) (Type: Class),"When the product encounters an error condition or failure, its design requires it to fall back to a state that is less secure than other options that are available, such as selecting the weakest encryption algorithm or using the most permissive access control restrictions. -By entering a less secure state, the product inherits the weaknesses associated with that state, making it easier to compromise. At the least, it causes administrators to have a false sense of security. This weakness typically occurs as a result of wanting to ""fail functional"" to minimize administration and support costs, instead of ""failing safe.""",,Unknown,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ +By entering a less secure state, the product inherits the weaknesses associated with that state, making it easier to compromise. At the least, it causes administrators to have a false sense of security. This weakness typically occurs as a result of wanting to ""fail functional"" to minimize administration and support costs, instead of ""failing safe.""",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ Failing Securely: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/349.html" CWE-637,EN-Unnecessary Complexity in Protection Mechanism (Not Using Economy of Mechanism) (Type: Class),"The software uses a more complex mechanism than necessary, which could lead to resultant weaknesses when the mechanism is not correctly understood, modeled, configured, implemented, or used. -Security mechanisms should be as simple as possible. Complex security mechanisms may engender partial implementations and compatibility problems, with resulting mismatches in assumptions and implemented security. A corollary of this principle is that data specifications should be as simple as possible, because complex data specifications result in complex validation code. Complex tasks and systems may also need to be guarded by complex security checks, so simple systems should be preferred.",,Unknown,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ +Security mechanisms should be as simple as possible. Complex security mechanisms may engender partial implementations and compatibility problems, with resulting mismatches in assumptions and implemented security. A corollary of this principle is that data specifications should be as simple as possible, because complex data specifications result in complex validation code. Complex tasks and systems may also need to be guarded by complex security checks, so simple systems should be preferred.",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ Economy of Mechanism: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/348.html" CWE-638,EN-Not Using Complete Mediation (Type: Class),"The software does not perform access checks on a resource every time the resource is accessed by an entity, which can create resultant weaknesses if that entity's rights or privileges change over time. -",,Unknown,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ +",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ Complete Mediation: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/346.html" CWE-65,EN-Windows Hard Link (Type: Variant),"The software, when opening a file or directory, does not sufficiently handle when the name is associated with a hard link to a target that is outside of the intended control sphere. This could allow an attacker to cause the software to operate on unauthorized files. -Failure for a system to check for hard links can result in vulnerability to different types of attacks. For example, an attacker can escalate their privileges if a file used by a privileged program is replaced with a hard link to a sensitive file (e.g. AUTOEXEC.BAT). When the process opens the file, the attacker can assume the privileges of that process, or prevent the program from accurately processing data.",,Unknown,"The Art of Software Security Assessment: Chapter 11, ""Links"", Page 676." +Failure for a system to check for hard links can result in vulnerability to different types of attacks. For example, an attacker can escalate their privileges if a file used by a privileged program is replaced with a hard link to a sensitive file (e.g. AUTOEXEC.BAT). When the process opens the file, the attacker can assume the privileges of that process, or prevent the program from accurately processing data.",,unclassified,"The Art of Software Security Assessment: Chapter 11, ""Links"", Page 676." CWE-651,EN-Information Exposure Through WSDL File (Type: Variant),"The Web services architecture may require exposing a WSDL file that contains information on the publicly accessible services and how callers of these services should interact with them (e.g. what parameters they expect and what types they return). An information exposure may occur if any of the following apply: The WSDL file is accessible to a wider audience than intended. The WSDL file contains information on the methods/services that should not be publicly accessible or information about deprecated methods. This problem is made more likely due to the WSDL often being automatically generated from the code. -Information in the WSDL file helps guess names/locations of methods/resources that should not be publicly accessible.",,Unknown, +Information in the WSDL file helps guess names/locations of methods/resources that should not be publicly accessible.",,unclassified, CWE-653,EN-Insufficient Compartmentalization (Type: Base),"The product does not sufficiently compartmentalize functionality or processes that require different privilege levels, rights, or permissions. -When a weakness occurs in functionality that is accessible by lower-privileged users, then without strong boundaries, an attack might extend the scope of the damage to higher-privileged users.",,Unknown,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ +When a weakness occurs in functionality that is accessible by lower-privileged users, then without strong boundaries, an attack might extend the scope of the damage to higher-privileged users.",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ Separation of Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/357.html" CWE-654,EN-Reliance on a Single Factor in a Security Decision (Type: Base),"A protection mechanism relies exclusively, or to a large extent, on the evaluation of a single condition or the integrity of a single object or entity in order to make a decision about granting access to restricted resources or functionality. -When a weakness occurs in functionality that is accessible by lower-privileged users, then without strong boundaries, an attack might extend the scope of the damage to higher-privileged users.",,Unknown,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ +When a weakness occurs in functionality that is accessible by lower-privileged users, then without strong boundaries, an attack might extend the scope of the damage to higher-privileged users.",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ Separation of Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/357.html" CWE-655,EN-Insufficient Psychological Acceptability (Type: Base),"The software has a protection mechanism that is too difficult or inconvenient to use, encouraging non-malicious users to disable or bypass the mechanism, whether by accident or on purpose. -When a weakness occurs in functionality that is accessible by lower-privileged users, then without strong boundaries, an attack might extend the scope of the damage to higher-privileged users.",,Unknown,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ +When a weakness occurs in functionality that is accessible by lower-privileged users, then without strong boundaries, an attack might extend the scope of the damage to higher-privileged users.",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ Psychological Acceptability: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/354.html Usability of Security: A Case Study: http://reports-archive.adm.cs.cmu.edu/anon/1998/CMU-CS-98-155.pdf 24 Deadly Sins of Software Security: ""Sin 14: Poor Usability."" Page 217" CWE-656,EN-Reliance on Security Through Obscurity (Type: Base),"The software uses a protection mechanism whose strength depends heavily on its obscurity, such that knowledge of its algorithms or key data is sufficient to defeat the mechanism. -This reliance on ""security through obscurity"" can produce resultant weaknesses if an attacker is able to reverse engineer the inner workings of the mechanism. Note that obscurity can be one small part of defense in depth, since it can create more work for an attacker; however, it is a significant risk if used as the primary means of protection.",,Unknown,"RFC: 793, TRANSMISSION CONTROL PROTOCOL: http://www.ietf.org/rfc/rfc0793.txt +This reliance on ""security through obscurity"" can produce resultant weaknesses if an attacker is able to reverse engineer the inner workings of the mechanism. Note that obscurity can be one small part of defense in depth, since it can create more work for an attacker; however, it is a significant risk if used as the primary means of protection.",,unclassified,"RFC: 793, TRANSMISSION CONTROL PROTOCOL: http://www.ietf.org/rfc/rfc0793.txt The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ Never Assuming that Your Secrets Are Safe: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/352.html" CWE-657,EN-Violation of Secure Design Principles (Type: Class),"The product violates well-established principles for secure design. -This can introduce resultant weaknesses or make it easier for developers to introduce related weaknesses during implementation. Because code is centered around design, it can be resource-intensive to fix design problems.",,Unknown,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ +This can introduce resultant weaknesses or make it easier for developers to introduce related weaknesses during implementation. Because code is centered around design, it can be resource-intensive to fix design problems.",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ Design Principles: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/358.html" CWE-66,EN-Improper Handling of File Names that Identify Virtual Resources (Type: Base),"The product does not handle or incorrectly handles a file name that identifies a ""virtual"" resource that is not directly specified within the directory that is associated with the file name, causing the product to perform file-based operations on a resource that is not a file. -Virtual file names are represented like normal file names, but they are effectively aliases for other resources that do not behave like normal files. Depending on their functionality, they could be alternate entities. They are not necessarily listed in directories.",,Unknown, +Virtual file names are represented like normal file names, but they are effectively aliases for other resources that do not behave like normal files. Depending on their functionality, they could be alternate entities. They are not necessarily listed in directories.",,unclassified, CWE-662,EN-Improper Synchronization (Type: Base),"The software attempts to use a shared resource in an exclusive manner, but does not prevent or incorrectly prevents use of the resource by another thread or process. -Virtual file names are represented like normal file names, but they are effectively aliases for other resources that do not behave like normal files. Depending on their functionality, they could be alternate entities. They are not necessarily listed in directories.",,Unknown, +Virtual file names are represented like normal file names, but they are effectively aliases for other resources that do not behave like normal files. Depending on their functionality, they could be alternate entities. They are not necessarily listed in directories.",,unclassified, CWE-663,EN-Use of a Non-reentrant Function in a Concurrent Context (Type: Base),"The software calls a non-reentrant function in a concurrent context in which a competing code sequence (e.g. thread or signal handler) may have an opportunity to call the same function or otherwise influence its state. -Virtual file names are represented like normal file names, but they are effectively aliases for other resources that do not behave like normal files. Depending on their functionality, they could be alternate entities. They are not necessarily listed in directories.",,Unknown,"Java Concurrency API: http://java.sun.com/j2se/1.5.0/docs/api/java/util/concurrent/locks/ReentrantLock.html +Virtual file names are represented like normal file names, but they are effectively aliases for other resources that do not behave like normal files. Depending on their functionality, they could be alternate entities. They are not necessarily listed in directories.",,unclassified,"Java Concurrency API: http://java.sun.com/j2se/1.5.0/docs/api/java/util/concurrent/locks/ReentrantLock.html Use reentrant functions for safer signal handling: http://www.ibm.com/developerworks/linux/library/l-reent.html" CWE-664,EN-Improper Control of a Resource Through its Lifetime (Type: Class),"The software does not maintain or incorrectly maintains control over a resource throughout its lifetime of creation, use, and release. Resources often have explicit instructions on how to be created, used and destroyed. When software does not follow these instructions, it can lead to unexpected behaviors and potentially exploitable states. -Even without explicit instructions, various principles are expected to be adhered to, such as ""Do not use an object until after its creation is complete,"" or ""do not use an object after it has been slated for destruction.""",,Unknown, +Even without explicit instructions, various principles are expected to be adhered to, such as ""Do not use an object until after its creation is complete,"" or ""do not use an object after it has been slated for destruction.""",,unclassified, CWE-666,EN-Operation on Resource in Wrong Phase of Lifetime (Type: Base),"The software performs an operation on a resource at the wrong phase of the resource's lifecycle, which can lead to unexpected behaviors. -When a developer wants to initialize, use or release a resource, it is important to follow the specifications outlined for how to operate on that resource and to ensure that the resource is in the expected state. In this case, the software wants to perform a normally valid operation, initialization, use or release, on a resource when it is in the incorrect phase of its lifetime.",,Unknown, +When a developer wants to initialize, use or release a resource, it is important to follow the specifications outlined for how to operate on that resource and to ensure that the resource is in the expected state. In this case, the software wants to perform a normally valid operation, initialization, use or release, on a resource when it is in the incorrect phase of its lifetime.",,unclassified, CWE-667,EN-Improper Locking (Type: Base),"The software does not properly acquire a lock on a resource, or it does not properly release a lock on a resource, leading to unexpected resource state changes and behaviors. -When a developer wants to initialize, use or release a resource, it is important to follow the specifications outlined for how to operate on that resource and to ensure that the resource is in the expected state. In this case, the software wants to perform a normally valid operation, initialization, use or release, on a resource when it is in the incorrect phase of its lifetime.",,Unknown, +When a developer wants to initialize, use or release a resource, it is important to follow the specifications outlined for how to operate on that resource and to ensure that the resource is in the expected state. In this case, the software wants to perform a normally valid operation, initialization, use or release, on a resource when it is in the incorrect phase of its lifetime.",,unclassified, CWE-668,EN-Exposure of Resource to Wrong Sphere (Type: Class),"The product exposes a resource to the wrong control sphere, providing unintended actors with inappropriate access to the resource. Resources such as files and directories may be inadvertently exposed through mechanisms such as insecure permissions, or when a program accidentally operates on the wrong object. For example, a program may intend that private files can only be provided to a specific user. This effectively defines a control sphere that is intended to prevent attackers from accessing these private files. If the file permissions are insecure, then parties other than the user will be able to access those files. A separate control sphere might effectively require that the user can only access the private files, but not any other files on the system. If the program does not ensure that the user is only requesting private files, then the user might be able to access other files on the system. -In either case, the end result is that a resource has been exposed to the wrong party.",,Unknown, +In either case, the end result is that a resource has been exposed to the wrong party.",,unclassified, CWE-669,EN-Incorrect Resource Transfer Between Spheres (Type: Class),"The product does not properly transfer a resource/behavior to another sphere, or improperly imports a resource/behavior from another sphere, in a manner that provides unintended control over that resource. Resources such as files and directories may be inadvertently exposed through mechanisms such as insecure permissions, or when a program accidentally operates on the wrong object. For example, a program may intend that private files can only be provided to a specific user. This effectively defines a control sphere that is intended to prevent attackers from accessing these private files. If the file permissions are insecure, then parties other than the user will be able to access those files. A separate control sphere might effectively require that the user can only access the private files, but not any other files on the system. If the program does not ensure that the user is only requesting private files, then the user might be able to access other files on the system. -In either case, the end result is that a resource has been exposed to the wrong party.",,Unknown, +In either case, the end result is that a resource has been exposed to the wrong party.",,unclassified, CWE-670,EN-Always-Incorrect Control Flow Implementation (Type: Class),"The code contains a control flow path that does not reflect the algorithm that the path is intended to implement, leading to incorrect behavior any time this path is navigated. -This weakness captures cases in which a particular code segment is always incorrect with respect to the algorithm that it is implementing. For example, if a C programmer intends to include multiple statements in a single block but does not include the enclosing braces (CWE-483), then the logic is always incorrect. This issue is in contrast to most weaknesses in which the code usually behaves correctly, except when it is externally manipulated in malicious ways.",,Unknown, +This weakness captures cases in which a particular code segment is always incorrect with respect to the algorithm that it is implementing. For example, if a C programmer intends to include multiple statements in a single block but does not include the enclosing braces (CWE-483), then the logic is always incorrect. This issue is in contrast to most weaknesses in which the code usually behaves correctly, except when it is externally manipulated in malicious ways.",,unclassified, CWE-671,EN-Lack of Administrator Control over Security (Type: Class),"The product uses security features in a way that prevents the product's administrator from tailoring security settings to reflect the environment in which the product is being used. This introduces resultant weaknesses or prevents it from operating at a level of security that is desired by the administrator. -If the product's administrator does not have the ability to manage security-related decisions at all times, then protecting the product from outside threats - including the product's developer - can become impossible. For example, a hard-coded account name and password cannot be changed by the administrator, thus exposing that product to attacks that the administrator can not prevent.",,Unknown, +If the product's administrator does not have the ability to manage security-related decisions at all times, then protecting the product from outside threats - including the product's developer - can become impossible. For example, a hard-coded account name and password cannot be changed by the administrator, thus exposing that product to attacks that the administrator can not prevent.",,unclassified, CWE-672,EN-Operation on a Resource after Expiration or Release (Type: Base),"The software uses, accesses, or otherwise operates on a resource after that resource has been expired, released, or revoked. -If the product's administrator does not have the ability to manage security-related decisions at all times, then protecting the product from outside threats - including the product's developer - can become impossible. For example, a hard-coded account name and password cannot be changed by the administrator, thus exposing that product to attacks that the administrator can not prevent.",,Unknown, +If the product's administrator does not have the ability to manage security-related decisions at all times, then protecting the product from outside threats - including the product's developer - can become impossible. For example, a hard-coded account name and password cannot be changed by the administrator, thus exposing that product to attacks that the administrator can not prevent.",,unclassified, CWE-673,EN-External Influence of Sphere Definition (Type: Class),"The product does not prevent the definition of control spheres from external actors. -Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,Unknown, +Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,unclassified, CWE-674,EN-Uncontrolled Recursion (Type: Base),"The product does not properly control the amount of recursion that takes place, which consumes excessive resources, such as allocated memory or the program stack. -Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,Unknown, +Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,unclassified, CWE-675,EN-Duplicate Operations on Resource (Type: Class),"The product performs the same operation on a resource two or more times, when the operation should only be applied once. -Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,Unknown, +Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,unclassified, CWE-683,EN-Function Call With Incorrect Order of Arguments (Type: Variant),"The software calls a function, procedure, or routine, but the caller specifies the arguments in an incorrect order, leading to resultant weaknesses. -While this weakness might be caught by the compiler in some languages, it can occur more frequently in cases in which the called function accepts variable numbers or types of arguments, such as format strings in C. It also can occur in languages or environments that do not enforce strong typing.",,Unknown, +While this weakness might be caught by the compiler in some languages, it can occur more frequently in cases in which the called function accepts variable numbers or types of arguments, such as format strings in C. It also can occur in languages or environments that do not enforce strong typing.",,unclassified, CWE-684,EN-Incorrect Provision of Specified Functionality (Type: Base),"The code does not function according to its published specifications, potentially leading to incorrect usage. -When providing functionality to an external party, it is important that the software behaves in accordance with the details specified. When requirements of nuances are not documented, the functionality may produce unintended behaviors for the caller, possibly leading to an exploitable state.",,Unknown, +When providing functionality to an external party, it is important that the software behaves in accordance with the details specified. When requirements of nuances are not documented, the functionality may produce unintended behaviors for the caller, possibly leading to an exploitable state.",,unclassified, CWE-685,EN-Function Call With Incorrect Number of Arguments (Type: Variant),"The software calls a function, procedure, or routine, but the caller specifies too many arguments, or too few arguments, which may lead to undefined behavior and resultant weaknesses. -When providing functionality to an external party, it is important that the software behaves in accordance with the details specified. When requirements of nuances are not documented, the functionality may produce unintended behaviors for the caller, possibly leading to an exploitable state.",,Unknown, +When providing functionality to an external party, it is important that the software behaves in accordance with the details specified. When requirements of nuances are not documented, the functionality may produce unintended behaviors for the caller, possibly leading to an exploitable state.",,unclassified, CWE-686,EN-Function Call With Incorrect Argument Type (Type: Variant),"The software calls a function, procedure, or routine, but the caller specifies an argument that is the wrong data type, which may lead to resultant weaknesses. -This weakness is most likely to occur in loosely typed languages, or in strongly typed languages in which the types of variable arguments cannot be enforced at compilation time, or where there is implicit casting.",,Unknown, +This weakness is most likely to occur in loosely typed languages, or in strongly typed languages in which the types of variable arguments cannot be enforced at compilation time, or where there is implicit casting.",,unclassified, CWE-687,EN-Function Call With Incorrectly Specified Argument Value (Type: Variant),"The software calls a function, procedure, or routine, but the caller specifies an argument that contains the wrong value, which may lead to resultant weaknesses. -This weakness is most likely to occur in loosely typed languages, or in strongly typed languages in which the types of variable arguments cannot be enforced at compilation time, or where there is implicit casting.",,Unknown, +This weakness is most likely to occur in loosely typed languages, or in strongly typed languages in which the types of variable arguments cannot be enforced at compilation time, or where there is implicit casting.",,unclassified, CWE-688,EN-Function Call With Incorrect Variable or Reference as Argument (Type: Variant),"The software calls a function, procedure, or routine, but the caller specifies the wrong variable or reference as one of the arguments, which may lead to undefined behavior and resultant weaknesses. -This weakness is most likely to occur in loosely typed languages, or in strongly typed languages in which the types of variable arguments cannot be enforced at compilation time, or where there is implicit casting.",,Unknown, +This weakness is most likely to occur in loosely typed languages, or in strongly typed languages in which the types of variable arguments cannot be enforced at compilation time, or where there is implicit casting.",,unclassified, CWE-69,EN-Improper Handling of Windows ::DATA Alternate Data Stream (Type: Variant),"The software does not properly prevent access to, or detect usage of, alternate data streams (ADS). -An attacker can use an ADS to hide information about a file (e.g. size, the name of the process) from a system or file browser tools such as Windows Explorer and 'dir' at the command line utility. Alternately, the attacker might be able to bypass intended access restrictions for the associated data fork.",,Unknown,"Windows NTFS Alternate Data Streams: http://www.securityfocus.com/infocus/1822 +An attacker can use an ADS to hide information about a file (e.g. size, the name of the process) from a system or file browser tools such as Windows Explorer and 'dir' at the command line utility. Alternately, the attacker might be able to bypass intended access restrictions for the associated data fork.",,unclassified,"Windows NTFS Alternate Data Streams: http://www.securityfocus.com/infocus/1822 Writing Secure Code" CWE-691,EN-Insufficient Control Flow Management (Type: Class),"The code does not sufficiently manage its control flow during execution, creating conditions in which the control flow can be modified in unexpected ways. -An attacker can use an ADS to hide information about a file (e.g. size, the name of the process) from a system or file browser tools such as Windows Explorer and 'dir' at the command line utility. Alternately, the attacker might be able to bypass intended access restrictions for the associated data fork.",,Unknown, +An attacker can use an ADS to hide information about a file (e.g. size, the name of the process) from a system or file browser tools such as Windows Explorer and 'dir' at the command line utility. Alternately, the attacker might be able to bypass intended access restrictions for the associated data fork.",,unclassified, CWE-693,EN-Protection Mechanism Failure (Type: Class),"The product does not use or incorrectly uses a protection mechanism that provides sufficient defense against directed attacks against the product. -This weakness covers three distinct situations. A ""missing"" protection mechanism occurs when the application does not define any mechanism against a certain class of attack. An ""insufficient"" protection mechanism might provide some defenses - for example, against the most common attacks - but it does not protect against everything that is intended. Finally, an ""ignored"" mechanism occurs when a mechanism is available and in active use within the product, but the developer has not applied it in some code path.",,Unknown, +This weakness covers three distinct situations. A ""missing"" protection mechanism occurs when the application does not define any mechanism against a certain class of attack. An ""insufficient"" protection mechanism might provide some defenses - for example, against the most common attacks - but it does not protect against everything that is intended. Finally, an ""ignored"" mechanism occurs when a mechanism is available and in active use within the product, but the developer has not applied it in some code path.",,unclassified, CWE-694,EN-Use of Multiple Resources with Duplicate Identifier (Type: Base),"The software uses multiple resources that can have the same identifier, in a context in which unique identifiers are required. -If the software assumes that each resource has a unique identifier, the software could operate on the wrong resource if attackers can cause multiple resources to be associated with the same identifier.",,Unknown, +If the software assumes that each resource has a unique identifier, the software could operate on the wrong resource if attackers can cause multiple resources to be associated with the same identifier.",,unclassified, CWE-695,EN-Use of Low-Level Functionality (Type: Base),"The software uses low-level functionality that is explicitly prohibited by the framework or specification under which the software is supposed to operate. -The use of low-level functionality can violate the specification in unexpected ways that effectively disable built-in protection mechanisms, introduce exploitable inconsistencies, or otherwise expose the functionality to attack.",,Unknown, +The use of low-level functionality can violate the specification in unexpected ways that effectively disable built-in protection mechanisms, introduce exploitable inconsistencies, or otherwise expose the functionality to attack.",,unclassified, CWE-696,EN-Incorrect Behavior Order (Type: Class),"The software performs multiple related behaviors, but the behaviors are performed in the wrong order in ways which may produce resultant weaknesses. -The use of low-level functionality can violate the specification in unexpected ways that effectively disable built-in protection mechanisms, introduce exploitable inconsistencies, or otherwise expose the functionality to attack.",,Unknown, +The use of low-level functionality can violate the specification in unexpected ways that effectively disable built-in protection mechanisms, introduce exploitable inconsistencies, or otherwise expose the functionality to attack.",,unclassified, CWE-697,EN-Insufficient Comparison (Type: Class),"The software compares two entities in a security-relevant context, but the comparison is insufficient, which may lead to resultant weaknesses. This weakness class covers several possibilities: the comparison checks one factor incorrectly; -the comparison should consider multiple factors, but it does not check some of those factors at all.",,Unknown, +the comparison should consider multiple factors, but it does not check some of those factors at all.",,unclassified, CWE-698,EN-Execution After Redirect (EAR) (Type: Base),"The web application sends a redirect to another location, but instead of exiting, it executes additional code. This weakness class covers several possibilities: the comparison checks one factor incorrectly; -the comparison should consider multiple factors, but it does not check some of those factors at all.",,Unknown,Fear the EAR: Discovering and Mitigating Execution After Redirect Vulnerabilities: http://cs.ucsb.edu/~bboe/public/pubs/fear-the-ear-ccs2011.pdf +the comparison should consider multiple factors, but it does not check some of those factors at all.",,unclassified,Fear the EAR: Discovering and Mitigating Execution After Redirect Vulnerabilities: http://cs.ucsb.edu/~bboe/public/pubs/fear-the-ear-ccs2011.pdf CWE-7,EN-J2EE Misconfiguration: Missing Custom Error Page (Type: Variant),"The default error page of a web application should not display sensitive information about the software system. -A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,Unknown,19 Deadly Sins of Software Security +A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified,19 Deadly Sins of Software Security CWE-703,EN-Improper Check or Handling of Exceptional Conditions (Type: Class),"The software does not properly anticipate or handle exceptional conditions that rarely occur during normal operation of the software. -A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,Unknown,"A Taxonomy of Security Faults in the UNIX Operating System: http://ftp.cerias.purdue.edu/pub/papers/taimur-aslam/aslam-taxonomy-msthesis.pdf +A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified,"A Taxonomy of Security Faults in the UNIX Operating System: http://ftp.cerias.purdue.edu/pub/papers/taimur-aslam/aslam-taxonomy-msthesis.pdf Use of A Taxonomy of Security Faults: http://csrc.nist.gov/nissc/1996/papers/NISSC96/paper057/PAPER.PDF 24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143" CWE-704,EN-Incorrect Type Conversion or Cast (Type: Class),"The software does not correctly convert an object, resource or structure from one type to a different type. -A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,Unknown, +A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified, CWE-705,EN-Incorrect Control Flow Scoping (Type: Class),"The software does not properly return control flow to the proper location after it has completed a task or detected an unusual condition. -A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,Unknown, +A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified, CWE-706,EN-Use of Incorrectly-Resolved Name or Reference (Type: Class),"The software uses a name or reference to access a resource, but the name/reference resolves to a resource that is outside of the intended control sphere. -A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,Unknown, +A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified, CWE-707,EN-Improper Enforcement of Message or Data Structure (Type: Class),"The software does not enforce or incorrectly enforces that structured messages or data are well-formed before being read from an upstream component or sent to a downstream component. If a message is malformed it may cause the message to be incorrectly interpreted. -This weakness typically applies in cases where the product prepares a control message that another process must act on, such as a command or query, and malicious input that was intended as data, can enter the control plane instead. However, this weakness also applies to more general cases where there are not always control implications.",,Unknown, +This weakness typically applies in cases where the product prepares a control message that another process must act on, such as a command or query, and malicious input that was intended as data, can enter the control plane instead. However, this weakness also applies to more general cases where there are not always control implications.",,unclassified, CWE-708,EN-Incorrect Ownership Assignment (Type: Base),"The software assigns an owner to a resource, but the owner is outside of the intended control sphere. -This may allow the resource to be manipulated by actors outside of the intended control sphere.",,Unknown, +This may allow the resource to be manipulated by actors outside of the intended control sphere.",,unclassified, CWE-71,EN-Apple .DS_Store (Type: Variant),"Software operating in a MAC OS environment, where .DS_Store is in effect, must carefully manage hard links, otherwise an attacker may be able to leverage a hard link from .DS_Store to overwrite arbitrary files and gain privileges. -This may allow the resource to be manipulated by actors outside of the intended control sphere.",,Unknown, +This may allow the resource to be manipulated by actors outside of the intended control sphere.",,unclassified, CWE-710,EN-Coding Standards Violation (Type: Class),"The software does not follow certain coding rules for development, which can lead to resultant weaknesses or increase the severity of the associated vulnerabilities. -This may allow the resource to be manipulated by actors outside of the intended control sphere.",,Unknown, +This may allow the resource to be manipulated by actors outside of the intended control sphere.",,unclassified, CWE-72,EN-Improper Handling of Apple HFS+ Alternate Data Stream Path (Type: Variant),"The software does not properly handle special paths that may identify the data or resource fork of a file on the HFS+ file system. -If the software chooses actions to take based on the file name, then if an attacker provides the data or resource fork, the software may take unexpected actions. Further, if the software intends to restrict access to a file, then an attacker might still be able to bypass intended access restrictions by requesting the data or resource fork for that file.",,Unknown,No description: http://docs.info.apple.com/article.html?artnum=300422 +If the software chooses actions to take based on the file name, then if an attacker provides the data or resource fork, the software may take unexpected actions. Further, if the software intends to restrict access to a file, then an attacker might still be able to bypass intended access restrictions by requesting the data or resource fork for that file.",,unclassified,No description: http://docs.info.apple.com/article.html?artnum=300422 CWE-733,EN-Compiler Optimization Removal or Modification of Security-critical Code (Type: Base),"The developer builds a security-critical protection mechanism into the software but the compiler optimizes the program such that the mechanism is removed or modified. -When a resource is given a permissions setting that provides access to a wider range of actors than required, it could lead to the exposure of sensitive information, or the modification of that resource by unintended parties. This is especially dangerous when the resource is related to program configuration, execution or sensitive user data.",,Unknown,"Writing Secure Code: Chapter 9, ""A Compiler Optimization Caveat"" Page 322" +When a resource is given a permissions setting that provides access to a wider range of actors than required, it could lead to the exposure of sensitive information, or the modification of that resource by unintended parties. This is especially dangerous when the resource is related to program configuration, execution or sensitive user data.",,unclassified,"Writing Secure Code: Chapter 9, ""A Compiler Optimization Caveat"" Page 322" CWE-75,EN-Failure to Sanitize Special Elements into a Different Plane (Special Element Injection) (Type: Class),"The software does not adequately filter user-controlled input for special elements with control implications. This weakness can lead to a wide variety of resultant weaknesses, depending on the behavior of the exposed method. It can apply to any number of technologies and approaches, such as ActiveX controls, Java functions, IOCTLs, and so on. The exposure can occur in a few different ways: 1) The function/method was never intended to be exposed to outside actors. -2) The function/method was only intended to be accessible to a limited set of actors, such as Internet-based access from a single web site.",,Unknown, +2) The function/method was only intended to be accessible to a limited set of actors, such as Internet-based access from a single web site.",,unclassified, CWE-756,EN-Missing Custom Error Page (Type: Class),"The software does not return custom error pages to the user, possibly exposing sensitive information. The programmer may assume that certain events or conditions will never occur or do not need to be worried about, such as low memory conditions, lack of access to resources due to restrictive permissions, or misbehaving clients or components. However, attackers may intentionally trigger these unusual conditions, thus violating the programmer's assumptions, possibly introducing instability, incorrect behavior, or a vulnerability. -Note that this entry is not exclusively about the use of exceptions and exception handling, which are mechanisms for both checking and handling unusual or unexpected conditions.",,Unknown, +Note that this entry is not exclusively about the use of exceptions and exception handling, which are mechanisms for both checking and handling unusual or unexpected conditions.",,unclassified, CWE-757,EN-Selection of Less-Secure Algorithm During Negotiation (Algorithm Downgrade) (Type: Class),"A protocol or its implementation supports interaction between multiple actors and allows those actors to negotiate which algorithm should be used as a protection mechanism such as encryption or authentication, but it does not select the strongest algorithm that is available to both parties. -When a security mechanism can be forced to downgrade to use a less secure algorithm, this can make it easier for attackers to compromise the software by exploiting weaker algorithm. The victim might not be aware that the less secure algorithm is being used. For example, if an attacker can force a communications channel to use cleartext instead of strongly-encrypted data, then the attacker could read the channel by sniffing, instead of going through extra effort of trying to decrypt the data using brute force techniques.",,Unknown, +When a security mechanism can be forced to downgrade to use a less secure algorithm, this can make it easier for attackers to compromise the software by exploiting weaker algorithm. The victim might not be aware that the less secure algorithm is being used. For example, if an attacker can force a communications channel to use cleartext instead of strongly-encrypted data, then the attacker could read the channel by sniffing, instead of going through extra effort of trying to decrypt the data using brute force techniques.",,unclassified, CWE-758,"EN-Reliance on Undefined, Unspecified, or Implementation-Defined Behavior (Type: Class)","The software uses an API function, data structure, or other entity in a way that relies on properties that are not always guaranteed to hold for that entity. -This can lead to resultant weaknesses when the required properties change, such as when the software is ported to a different platform or if an interaction error (CWE-435) occurs.",,Unknown, +This can lead to resultant weaknesses when the required properties change, such as when the software is ported to a different platform or if an interaction error (CWE-435) occurs.",,unclassified, CWE-759,EN-Use of a One-Way Hash without a Salt (Type: Base),"The software uses a one-way cryptographic hash against an input that should not be reversible, such as a password, but the software does not also use a salt as part of the input. This makes it easier for attackers to pre-compute the hash value using dictionary attack techniques such as rainbow tables. -It should be noted that, despite common perceptions, the use of a good salt with a hash does not sufficiently increase the effort for an attacker who is targeting an individual password, or who has a large amount of computing resources available, such as with cloud-based services or specialized, inexpensive hardware. Offline password cracking can still be effective if the hash function is not expensive to compute; many cryptographic functions are designed to be efficient and can be vulnerable to attacks using massive computing resources, even if the hash is cryptographically strong. The use of a salt only slightly increases the computing requirements for an attacker compared to other strategies such as adaptive hash functions. See CWE-916 for more details.",,Unknown,"bcrypt: http://bcrypt.sourceforge.net/ +It should be noted that, despite common perceptions, the use of a good salt with a hash does not sufficiently increase the effort for an attacker who is targeting an individual password, or who has a large amount of computing resources available, such as with cloud-based services or specialized, inexpensive hardware. Offline password cracking can still be effective if the hash function is not expensive to compute; many cryptographic functions are designed to be efficient and can be vulnerable to attacks using massive computing resources, even if the hash is cryptographically strong. The use of a salt only slightly increases the computing requirements for an attacker compared to other strategies such as adaptive hash functions. See CWE-916 for more details.",,unclassified,"bcrypt: http://bcrypt.sourceforge.net/ Tarsnap - The scrypt key derivation function and encryption utility: http://www.tarsnap.com/scrypt.html RFC2898 - PKCS #5: Password-Based Cryptography Specification Version 2.0: http://tools.ietf.org/html/rfc2898 How Companies Can Beef Up Password Security (interview with Thomas H. Ptacek): http://krebsonsecurity.com/2012/06/how-companies-can-beef-up-password-security/ @@ -1881,7 +1881,7 @@ Our password hashing has no clothes: http://www.troyhunt.com/2012/06/our-passwor Should we really use bcrypt/scrypt?: http://www.analyticalengine.net/2012/06/should-we-really-use-bcryptscrypt/" CWE-760,EN-Use of a One-Way Hash with a Predictable Salt (Type: Base),"The software uses a one-way cryptographic hash against an input that should not be reversible, such as a password, but the software uses a predictable salt as part of the input. This makes it easier for attackers to pre-compute the hash value using dictionary attack techniques such as rainbow tables, effectively disabling the protection that an unpredictable salt would provide. -It should be noted that, despite common perceptions, the use of a good salt with a hash does not sufficiently increase the effort for an attacker who is targeting an individual password, or who has a large amount of computing resources available, such as with cloud-based services or specialized, inexpensive hardware. Offline password cracking can still be effective if the hash function is not expensive to compute; many cryptographic functions are designed to be efficient and can be vulnerable to attacks using massive computing resources, even if the hash is cryptographically strong. The use of a salt only slightly increases the computing requirements for an attacker compared to other strategies such as adaptive hash functions. See CWE-916 for more details.",,Unknown,"bcrypt: http://bcrypt.sourceforge.net/ +It should be noted that, despite common perceptions, the use of a good salt with a hash does not sufficiently increase the effort for an attacker who is targeting an individual password, or who has a large amount of computing resources available, such as with cloud-based services or specialized, inexpensive hardware. Offline password cracking can still be effective if the hash function is not expensive to compute; many cryptographic functions are designed to be efficient and can be vulnerable to attacks using massive computing resources, even if the hash is cryptographically strong. The use of a salt only slightly increases the computing requirements for an attacker compared to other strategies such as adaptive hash functions. See CWE-916 for more details.",,unclassified,"bcrypt: http://bcrypt.sourceforge.net/ Tarsnap - The scrypt key derivation function and encryption utility: http://www.tarsnap.com/scrypt.html RFC2898 - PKCS #5: Password-Based Cryptography Specification Version 2.0: http://tools.ietf.org/html/rfc2898 How Companies Can Beef Up Password Security (interview with Thomas H. Ptacek): http://krebsonsecurity.com/2012/06/how-companies-can-beef-up-password-security/ @@ -1900,25 +1900,25 @@ Our password hashing has no clothes: http://www.troyhunt.com/2012/06/our-passwor Should we really use bcrypt/scrypt?: http://www.analyticalengine.net/2012/06/should-we-really-use-bcryptscrypt/" CWE-761,EN-Free of Pointer not at Start of Buffer (Type: Variant),"The application calls free() on a pointer to a memory resource that was allocated on the heap, but the pointer is not at the start of the buffer. This can cause the application to crash, or in some cases, modify critical program variables or execute code. -This weakness often occurs when the memory is allocated explicitly on the heap with one of the malloc() family functions and free() is called, but pointer arithmetic has caused the pointer to be in the interior or end of the buffer.",,Unknown,"boost C++ Library Smart Pointers: http://www.boost.org/doc/libs/1_38_0/libs/smart_ptr/smart_ptr.htm +This weakness often occurs when the memory is allocated explicitly on the heap with one of the malloc() family functions and free() is called, but pointer arithmetic has caused the pointer to be in the interior or end of the buffer.",,unclassified,"boost C++ Library Smart Pointers: http://www.boost.org/doc/libs/1_38_0/libs/smart_ptr/smart_ptr.htm Valgrind: http://valgrind.org/" CWE-763,EN-Release of Invalid Pointer or Reference (Type: Base),"The application attempts to return a memory resource to the system, but calls the wrong release function or calls the appropriate release function incorrectly. This weakness can take several forms, such as: The memory was allocated, explicitly or implicitly, via one memory management method and deallocated using a different, non-compatible function (CWE-762). -The function calls or memory management routines chosen are appropriate, however they are used incorrectly, such as in CWE-761.",,Unknown,"boost C++ Library Smart Pointers: http://www.boost.org/doc/libs/1_38_0/libs/smart_ptr/smart_ptr.htm +The function calls or memory management routines chosen are appropriate, however they are used incorrectly, such as in CWE-761.",,unclassified,"boost C++ Library Smart Pointers: http://www.boost.org/doc/libs/1_38_0/libs/smart_ptr/smart_ptr.htm Valgrind: http://valgrind.org/" CWE-764,EN-Multiple Locks of a Critical Resource (Type: Variant),"The software locks a critical resource more times than intended, leading to an unexpected state in the system. -When software is operating in a concurrent environment and repeatedly locks a critical resource, the consequences will vary based on the type of lock, the lock's implementation, and the resource being protected. In some situations such as with semaphores, the resources are pooled and extra locking calls will reduce the size of the total available pool, possibly leading to degraded performance or a denial of service. If this can be triggered by an attacker, it will be similar to an unrestricted lock (CWE-412). In the context of a binary lock, it is likely that any duplicate locking attempts will never succeed since the lock is already held and progress may not be possible.",,Unknown, +When software is operating in a concurrent environment and repeatedly locks a critical resource, the consequences will vary based on the type of lock, the lock's implementation, and the resource being protected. In some situations such as with semaphores, the resources are pooled and extra locking calls will reduce the size of the total available pool, possibly leading to degraded performance or a denial of service. If this can be triggered by an attacker, it will be similar to an unrestricted lock (CWE-412). In the context of a binary lock, it is likely that any duplicate locking attempts will never succeed since the lock is already held and progress may not be possible.",,unclassified, CWE-765,EN-Multiple Unlocks of a Critical Resource (Type: Variant),"The software unlocks a critical resource more times than intended, leading to an unexpected state in the system. -When software is operating in a concurrent environment and repeatedly unlocks a critical resource, the consequences will vary based on the type of lock, the lock's implementation, and the resource being protected. In some situations such as with semaphores, the resources are pooled and extra calls to unlock will increase the count for the number of available resources, likely resulting in a crash or unpredictable behavior when the system nears capacity.",,Unknown, +When software is operating in a concurrent environment and repeatedly unlocks a critical resource, the consequences will vary based on the type of lock, the lock's implementation, and the resource being protected. In some situations such as with semaphores, the resources are pooled and extra calls to unlock will increase the count for the number of available resources, likely resulting in a crash or unpredictable behavior when the system nears capacity.",,unclassified, CWE-785,EN-Use of Path Manipulation Function without Maximum-sized Buffer (Type: Variant),"The software invokes a function for normalizing paths or file names, but it provides an output buffer that is smaller than the maximum possible size, such as PATH_MAX. -Passing an inadequately-sized output buffer to a path manipulation function can result in a buffer overflow. Such functions include realpath(), readlink(), PathAppend(), and others.",,Unknown, +Passing an inadequately-sized output buffer to a path manipulation function can result in a buffer overflow. Such functions include realpath(), readlink(), PathAppend(), and others.",,unclassified, CWE-786,EN-Access of Memory Location Before Start of Buffer (Type: Base),"The software reads or writes to a buffer using an index or pointer that references a memory location prior to the beginning of the buffer. -This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used.",,Unknown, +This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used.",,unclassified, CWE-787,EN-Out-of-bounds Write (Type: Base),"The software writes data past the end, or before the beginning, of the intended buffer. -This typically occurs when the pointer or its index is incremented or decremented to a position beyond the bounds of the buffer or when pointer arithmetic results in a position outside of the valid memory location to name a few. This may result in corruption of sensitive information, a crash, or code execution among other things.",,Unknown, +This typically occurs when the pointer or its index is incremented or decremented to a position beyond the bounds of the buffer or when pointer arithmetic results in a position outside of the valid memory location to name a few. This may result in corruption of sensitive information, a crash, or code execution among other things.",,unclassified, CWE-788,EN-Access of Memory Location After End of Buffer (Type: Base),"The software reads or writes to a buffer using an index or pointer that references a memory location after the end of the buffer. -This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. These problems may be resultant from missing sentinel values (CWE-463) or trusting a user-influenced input length variable.",,Unknown, +This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. These problems may be resultant from missing sentinel values (CWE-463) or trusting a user-influenced input length variable.",,unclassified, CWE-790,EN-Improper Filtering of Special Elements (Type: Class),"The software receives data from an upstream component, but does not filter or incorrectly filters special elements before sending it to a downstream component. Cross-site scripting (XSS) vulnerabilities occur when: 1. Untrusted data enters a web application, typically from a web request. @@ -1932,7 +1932,7 @@ The server reads data directly from the HTTP request and reflects it back in the The application stores dangerous data in a database, message forum, visitor log, or other trusted data store. At a later time, the dangerous data is subsequently read back into the application and included in dynamic content. From an attacker's perspective, the optimal place to inject malicious content is in an area that is displayed to either many users or particularly interesting users. Interesting users typically have elevated privileges in the application or interact with sensitive data that is valuable to the attacker. If one of these users executes malicious content, the attacker may be able to perform privileged operations on behalf of the user or gain access to sensitive data belonging to the user. For example, the attacker might inject XSS into a log message, which might not be handled properly when an administrator views the logs. In DOM-based XSS, the client performs the injection of XSS into the page; in the other types, the server performs the injection. DOM-based XSS generally involves server-controlled, trusted script that is sent to the client, such as Javascript that performs sanity checks on a form before the user submits it. If the server-supplied script processes user-supplied data and then injects it back into the web page (such as with dynamic HTML), then DOM-based XSS is possible. Once the malicious script is injected, the attacker can perform a variety of malicious activities. The attacker could transfer private information, such as cookies that may include session information, from the victim's machine to the attacker. The attacker could send malicious requests to a web site on behalf of the victim, which could be especially dangerous to the site if the victim has administrator privileges to manage that site. Phishing attacks could be used to emulate trusted web sites and trick the victim into entering a password, allowing the attacker to compromise the victim's account on that web site. Finally, the script could exploit a vulnerability in the web browser itself possibly taking over the victim's machine, sometimes referred to as ""drive-by hacking."" -In many cases, the attack can be launched without the victim even being aware of it. Even with careful users, attackers frequently use a variety of methods to encode the malicious portion of the attack, such as URL encoding or Unicode, so the request looks less suspicious.",,Unknown, +In many cases, the attack can be launched without the victim even being aware of it. Even with careful users, attackers frequently use a variety of methods to encode the malicious portion of the attack, such as URL encoding or Unicode, so the request looks less suspicious.",,unclassified, CWE-791,EN-Incomplete Filtering of Special Elements (Type: Base),"The software receives data from an upstream component, but does not completely filter special elements before sending it to a downstream component. Cross-site scripting (XSS) vulnerabilities occur when: 1. Untrusted data enters a web application, typically from a web request. @@ -1946,66 +1946,66 @@ The server reads data directly from the HTTP request and reflects it back in the The application stores dangerous data in a database, message forum, visitor log, or other trusted data store. At a later time, the dangerous data is subsequently read back into the application and included in dynamic content. From an attacker's perspective, the optimal place to inject malicious content is in an area that is displayed to either many users or particularly interesting users. Interesting users typically have elevated privileges in the application or interact with sensitive data that is valuable to the attacker. If one of these users executes malicious content, the attacker may be able to perform privileged operations on behalf of the user or gain access to sensitive data belonging to the user. For example, the attacker might inject XSS into a log message, which might not be handled properly when an administrator views the logs. In DOM-based XSS, the client performs the injection of XSS into the page; in the other types, the server performs the injection. DOM-based XSS generally involves server-controlled, trusted script that is sent to the client, such as Javascript that performs sanity checks on a form before the user submits it. If the server-supplied script processes user-supplied data and then injects it back into the web page (such as with dynamic HTML), then DOM-based XSS is possible. Once the malicious script is injected, the attacker can perform a variety of malicious activities. The attacker could transfer private information, such as cookies that may include session information, from the victim's machine to the attacker. The attacker could send malicious requests to a web site on behalf of the victim, which could be especially dangerous to the site if the victim has administrator privileges to manage that site. Phishing attacks could be used to emulate trusted web sites and trick the victim into entering a password, allowing the attacker to compromise the victim's account on that web site. Finally, the script could exploit a vulnerability in the web browser itself possibly taking over the victim's machine, sometimes referred to as ""drive-by hacking."" -In many cases, the attack can be launched without the victim even being aware of it. Even with careful users, attackers frequently use a variety of methods to encode the malicious portion of the attack, such as URL encoding or Unicode, so the request looks less suspicious.",,Unknown, +In many cases, the attack can be launched without the victim even being aware of it. Even with careful users, attackers frequently use a variety of methods to encode the malicious portion of the attack, such as URL encoding or Unicode, so the request looks less suspicious.",,unclassified, CWE-792,EN-Incomplete Filtering of One or More Instances of Special Elements (Type: Variant),"The software receives data from an upstream component, but does not completely filter one or more instances of special elements before sending it to a downstream component. Incomplete filtering of this nature involves either only filtering a single instance of a special element when more exist, or -not filtering all instances or all elements where multiple special elements exist.",,Unknown, +not filtering all instances or all elements where multiple special elements exist.",,unclassified, CWE-793,EN-Only Filtering One Instance of a Special Element (Type: Variant),"The software receives data from an upstream component, but only filters a single instance of a special element before sending it to a downstream component. -Incomplete filtering of this nature may be location-dependent, as in only the first or last element is filtered.",,Unknown, +Incomplete filtering of this nature may be location-dependent, as in only the first or last element is filtered.",,unclassified, CWE-794,EN-Incomplete Filtering of Multiple Instances of Special Elements (Type: Variant),"The software receives data from an upstream component, but does not filter all instances of a special element before sending it to a downstream component. Incomplete filtering of this nature may be applied to sequential elements (special elements that appear next to each other) or -non-sequential elements (special elements that appear multiple times in different locations).",,Unknown, +non-sequential elements (special elements that appear multiple times in different locations).",,unclassified, CWE-795,EN-Only Filtering Special Elements at a Specified Location (Type: Base),"The software receives data from an upstream component, but only accounts for special elements at a specified location, thereby missing remaining special elements that may exist before sending it to a downstream component. A filter might only account for instances of special elements when they occur: relative to a marker (e.g. ""at the beginning/end of string; the second argument""), or at an absolute position (e.g. ""byte number 10""). -This may leave special elements in the data that did not match the filter position, but still may be dangerous.",,Unknown, +This may leave special elements in the data that did not match the filter position, but still may be dangerous.",,unclassified, CWE-796,EN-Only Filtering Special Elements Relative to a Marker (Type: Variant),"The software receives data from an upstream component, but only accounts for special elements positioned relative to a marker (e.g. ""at the beginning/end of a string; the second argument""), thereby missing remaining special elements that may exist before sending it to a downstream component. A filter might only account for instances of special elements when they occur: relative to a marker (e.g. ""at the beginning/end of string; the second argument""), or at an absolute position (e.g. ""byte number 10""). -This may leave special elements in the data that did not match the filter position, but still may be dangerous.",,Unknown, +This may leave special elements in the data that did not match the filter position, but still may be dangerous.",,unclassified, CWE-797,EN-Only Filtering Special Elements at an Absolute Position (Type: Variant),"The software receives data from an upstream component, but only accounts for special elements at an absolute position (e.g. ""byte number 10""), thereby missing remaining special elements that may exist before sending it to a downstream component. A filter might only account for instances of special elements when they occur: relative to a marker (e.g. ""at the beginning/end of string; the second argument""), or at an absolute position (e.g. ""byte number 10""). -This may leave special elements in the data that did not match the filter position, but still may be dangerous.",,Unknown, +This may leave special elements in the data that did not match the filter position, but still may be dangerous.",,unclassified, CWE-799,EN-Improper Control of Interaction Frequency (Type: Class),"The software does not properly limit the number or frequency of interactions that it has with an actor, such as the number of incoming requests. -This can allow the actor to perform actions more frequently than expected. The actor could be a human or an automated process such as a virus or bot. This could be used to cause a denial of service, compromise program logic (such as limiting humans to a single vote), or other consequences. For example, an authentication routine might not limit the number of times an attacker can guess a password. Or, a web site might conduct a poll but only expect humans to vote a maximum of once a day.",,Unknown,Insufficient Anti-automation: http://projects.webappsec.org/Insufficient+Anti-automation +This can allow the actor to perform actions more frequently than expected. The actor could be a human or an automated process such as a virus or bot. This could be used to cause a denial of service, compromise program logic (such as limiting humans to a single vote), or other consequences. For example, an authentication routine might not limit the number of times an attacker can guess a password. Or, a web site might conduct a poll but only expect humans to vote a maximum of once a day.",,unclassified,Insufficient Anti-automation: http://projects.webappsec.org/Insufficient+Anti-automation CWE-8,EN-J2EE Misconfiguration: Entity Bean Declared Remote (Type: Variant),"When an application exposes a remote interface for an entity bean, it might also expose methods that get or set the bean's data. These methods could be leveraged to read sensitive information, or to change data in ways that violate the application's expectations, potentially leading to other vulnerabilities. -This can allow the actor to perform actions more frequently than expected. The actor could be a human or an automated process such as a virus or bot. This could be used to cause a denial of service, compromise program logic (such as limiting humans to a single vote), or other consequences. For example, an authentication routine might not limit the number of times an attacker can guess a password. Or, a web site might conduct a poll but only expect humans to vote a maximum of once a day.",,Unknown, +This can allow the actor to perform actions more frequently than expected. The actor could be a human or an automated process such as a virus or bot. This could be used to cause a denial of service, compromise program logic (such as limiting humans to a single vote), or other consequences. For example, an authentication routine might not limit the number of times an attacker can guess a password. Or, a web site might conduct a poll but only expect humans to vote a maximum of once a day.",,unclassified, CWE-81,EN-Improper Neutralization of Script in an Error Message Web Page (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special characters that could be interpreted as web-scripting elements when they are sent to an error page. Error pages may include customized 403 Forbidden or 404 Not Found pages. -When an attacker can trigger an error that contains unneutralized input, then cross-site scripting attacks may be possible.",,Unknown,"24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183" +When an attacker can trigger an error that contains unneutralized input, then cross-site scripting attacks may be possible.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183" CWE-82,EN-Improper Neutralization of Script in Attributes of IMG Tags in a Web Page (Type: Variant),"The web application does not neutralize or incorrectly neutralizes scripting elements within attributes of HTML IMG tags, such as the src attribute. -Attackers can embed XSS exploits into the values for IMG attributes (e.g. SRC) that is streamed and then executed in a victim's browser. Note that when the page is loaded into a user's browsers, the exploit will automatically execute.",,Unknown, +Attackers can embed XSS exploits into the values for IMG attributes (e.g. SRC) that is streamed and then executed in a victim's browser. Note that when the page is loaded into a user's browsers, the exploit will automatically execute.",,unclassified, CWE-820,EN-Missing Synchronization (Type: Base),"The software utilizes a shared resource in a concurrent manner but does not attempt to synchronize access to the resource. -If access to a shared resource is not synchronized, then the resource may not be in a state that is expected by the software. This might lead to unexpected or insecure behaviors, especially if an attacker can influence the shared resource.",,Unknown, +If access to a shared resource is not synchronized, then the resource may not be in a state that is expected by the software. This might lead to unexpected or insecure behaviors, especially if an attacker can influence the shared resource.",,unclassified, CWE-821,EN-Incorrect Synchronization (Type: Base),"The software utilizes a shared resource in a concurrent manner but it does not correctly synchronize access to the resource. -If access to a shared resource is not correctly synchronized, then the resource may not be in a state that is expected by the software. This might lead to unexpected or insecure behaviors, especially if an attacker can influence the shared resource.",,Unknown, +If access to a shared resource is not correctly synchronized, then the resource may not be in a state that is expected by the software. This might lead to unexpected or insecure behaviors, especially if an attacker can influence the shared resource.",,unclassified, CWE-822,EN-Untrusted Pointer Dereference (Type: Base),"The program obtains a value from an untrusted source, converts this value to a pointer, and dereferences the resulting pointer. An attacker can supply a pointer for memory locations that the program is not expecting. If the pointer is dereferenced for a write operation, the attack might allow modification of critical program state variables, cause a crash, or execute code. If the dereferencing operation is for a read, then the attack might allow reading of sensitive data, cause a crash, or set a program variable to an unexpected value (since the value will be read from an unexpected memory location). There are several variants of this weakness, including but not necessarily limited to: The untrusted value is directly invoked as a function call. In OS kernels or drivers where there is a boundary between ""userland"" and privileged memory spaces, an untrusted pointer might enter through an API or system call (see CWE-781 for one such example). -Inadvertently accepting the value from an untrusted control sphere when it did not have to be accepted as input at all. This might occur when the code was originally developed to be run by a single user in a non-networked environment, and the code is then ported to or otherwise exposed to a networked environment.",,Unknown, +Inadvertently accepting the value from an untrusted control sphere when it did not have to be accepted as input at all. This might occur when the code was originally developed to be run by a single user in a non-networked environment, and the code is then ported to or otherwise exposed to a networked environment.",,unclassified, CWE-823,EN-Use of Out-of-range Pointer Offset (Type: Base),"The program performs pointer arithmetic on a valid pointer, but it uses an offset that can point outside of the intended range of valid memory locations for the resulting pointer. While a pointer can contain a reference to any arbitrary memory location, a program typically only intends to use the pointer to access limited portions of memory, such as contiguous memory used to access an individual array. Programs may use offsets in order to access fields or sub-elements stored within structured data. The offset might be out-of-range if it comes from an untrusted source, is the result of an incorrect calculation, or occurs because of another error. -If an attacker can control or influence the offset so that it points outside of the intended boundaries of the structure, then the attacker may be able to read or write to memory locations that are used elsewhere in the program. As a result, the attack might change the state of the software as accessed through program variables, cause a crash or instable behavior, and possibly lead to code execution.",,Unknown,"The Art of Software Security Assessment: Chapter 6, ""Pointer Arithmetic"", Page 277." +If an attacker can control or influence the offset so that it points outside of the intended boundaries of the structure, then the attacker may be able to read or write to memory locations that are used elsewhere in the program. As a result, the attack might change the state of the software as accessed through program variables, cause a crash or instable behavior, and possibly lead to code execution.",,unclassified,"The Art of Software Security Assessment: Chapter 6, ""Pointer Arithmetic"", Page 277." CWE-824,EN-Access of Uninitialized Pointer (Type: Base),"The program accesses or uses a pointer that has not been initialized. If the pointer contains an uninitialized value, then the value might not point to a valid memory location. This could cause the program to read from or write to unexpected memory locations, leading to a denial of service. If the uninitialized pointer is used as a function call, then arbitrary functions could be invoked. If an attacker can influence the portion of uninitialized memory that is contained in the pointer, this weakness could be leveraged to execute code or perform other attacks. -Depending on memory layout, associated memory management behaviors, and program operation, the attacker might be able to influence the contents of the uninitialized pointer, thus gaining more fine-grained control of the memory location to be accessed.",,Unknown,"The Art of Software Security Assessment: Chapter 7, ""Variable Initialization"", Page 312." +Depending on memory layout, associated memory management behaviors, and program operation, the attacker might be able to influence the contents of the uninitialized pointer, thus gaining more fine-grained control of the memory location to be accessed.",,unclassified,"The Art of Software Security Assessment: Chapter 7, ""Variable Initialization"", Page 312." CWE-825,EN-Expired Pointer Dereference (Type: Base),"The program dereferences a pointer that contains a location for memory that was previously valid, but is no longer valid. -When a program releases memory, but it maintains a pointer to that memory, then the memory might be re-allocated at a later time. If the original pointer is accessed to read or write data, then this could cause the program to read or modify data that is in use by a different function or process. Depending on how the newly-allocated memory is used, this could lead to a denial of service, information exposure, or code execution.",,Unknown, +When a program releases memory, but it maintains a pointer to that memory, then the memory might be re-allocated at a later time. If the original pointer is accessed to read or write data, then this could cause the program to read or modify data that is in use by a different function or process. Depending on how the newly-allocated memory is used, this could lead to a denial of service, information exposure, or code execution.",,unclassified, CWE-826,EN-Premature Release of Resource During Expected Lifetime (Type: Base),"The program releases a resource that is still intended to be used by the program itself or another actor. This weakness focuses on errors in which the program should not release a resource, but performs the release anyway. This is different than a weakness in which the program releases a resource at the appropriate time, but it maintains a reference to the resource, which it later accesses. For this weaknesses, the resource should still be valid upon the subsequent access. -When a program releases a resource that is still being used, it is possible that operations will still be taken on this resource, which may have been repurposed in the meantime, leading to issues similar to CWE-825. Consequences may include denial of service, information exposure, or code execution.",,Unknown, +When a program releases a resource that is still being used, it is possible that operations will still be taken on this resource, which may have been repurposed in the meantime, leading to issues similar to CWE-825. Consequences may include denial of service, information exposure, or code execution.",,unclassified, CWE-827,EN-Improper Control of Document Type Definition (Type: Base),"The software does not restrict a reference to a Document Type Definition (DTD) to the intended control sphere. This might allow attackers to reference arbitrary DTDs, possibly causing the software to expose files, consume excessive system resources, or execute arbitrary http requests on behalf of the attacker. As DTDs are processed, they might try to read or include files on the machine performing the parsing. If an attacker is able to control the DTD, then the attacker might be able to specify sensitive resources or requests or provide malicious content. -For example, the SOAP specification prohibits SOAP messages from containing DTDs.",,Unknown,Apache CXF Security Advisory (CVE-2010-2076): http://svn.apache.org/repos/asf/cxf/trunk/security/CVE-2010-2076.pdf +For example, the SOAP specification prohibits SOAP messages from containing DTDs.",,unclassified,Apache CXF Security Advisory (CVE-2010-2076): http://svn.apache.org/repos/asf/cxf/trunk/security/CVE-2010-2076.pdf CWE-828,EN-Signal Handler with Functionality that is not Asynchronous-Safe (Type: Base),"The software defines a signal handler that contains code sequences that are not asynchronous-safe, i.e., the functionality is not reentrant, or it can be interrupted. This can lead to an unexpected system state with a variety of potential consequences depending on context, including denial of service and code execution. Signal handlers are typically intended to interrupt normal functionality of a program, or even other signals, in order to notify the process of an event. When a signal handler uses global or static variables, or invokes functions that ultimately depend on such state or its associated metadata, then it could corrupt system state that is being used by normal functionality. This could subject the program to race conditions or other weaknesses that allow an attacker to cause the program state to be corrupted. While denial of service is frequently the consequence, in some cases this weakness could be leveraged for code execution. @@ -2014,45 +2014,45 @@ Invocation of non-reentrant functions from within the handler. One example is ma Code sequences (not necessarily function calls) contain non-atomic use of global variables, or associated metadata or structures, that can be accessed by other functionality of the program, including other signal handlers. Frequently, the same function is registered to handle multiple signals. The signal handler function is intended to run at most one time, but instead it can be invoked multiple times. This could happen by repeated delivery of the same signal, or by delivery of different signals that have the same handler function (CWE-831). Note that in some environments or contexts, it might be possible for the signal handler to be interrupted itself. -If both a signal handler and the normal behavior of the software have to operate on the same set of state variables, and a signal is received in the middle of the normal execution's modifications of those variables, the variables may be in an incorrect or corrupt state during signal handler execution, and possibly still incorrect or corrupt upon return.",,Unknown,"Delivering Signals for Fun and Profit: http://lcamtuf.coredump.cx/signals.txt +If both a signal handler and the normal behavior of the software have to operate on the same set of state variables, and a signal is received in the middle of the normal execution's modifications of those variables, the variables may be in an incorrect or corrupt state during signal handler execution, and possibly still incorrect or corrupt upon return.",,unclassified,"Delivering Signals for Fun and Profit: http://lcamtuf.coredump.cx/signals.txt Race Condition: Signal Handling: http://www.fortify.com/vulncat/en/vulncat/cpp/race_condition_signal_handling.html" CWE-829,EN-Inclusion of Functionality from Untrusted Control Sphere (Type: Class),"The software imports, requires, or includes executable functionality (such as a library) from a source that is outside of the intended control sphere. When including third-party functionality, such as a web widget, library, or other source of functionality, the software must effectively trust that functionality. Without sufficient protection mechanisms, the functionality could be malicious in nature (either by coming from an untrusted source, being spoofed, or being modified in transit from a trusted source). The functionality might also contain its own weaknesses, or grant access to additional functionality and state information that should be kept private to the base system, such as system state information, sensitive application data, or the DOM of a web application. -This might lead to many different consequences depending on the included functionality, but some examples include injection of malware, information exposure by granting excessive privileges or permissions to the untrusted functionality, DOM-based XSS vulnerabilities, stealing user's cookies, or open redirect to malware (CWE-601).",,Unknown,"OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI +This might lead to many different consequences depending on the included functionality, but some examples include injection of malware, information exposure by granting excessive privileges or permissions to the untrusted functionality, DOM-based XSS vulnerabilities, stealing user's cookies, or open redirect to malware (CWE-601).",,unclassified,"OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html" CWE-83,EN-Improper Neutralization of Script in Attributes in a Web Page (Type: Variant),"The software does not neutralize or incorrectly neutralizes ""javascript:"" or other URIs from dangerous attributes within tags, such as onmouseover, onload, onerror, or style. When including third-party functionality, such as a web widget, library, or other source of functionality, the software must effectively trust that functionality. Without sufficient protection mechanisms, the functionality could be malicious in nature (either by coming from an untrusted source, being spoofed, or being modified in transit from a trusted source). The functionality might also contain its own weaknesses, or grant access to additional functionality and state information that should be kept private to the base system, such as system state information, sensitive application data, or the DOM of a web application. -This might lead to many different consequences depending on the included functionality, but some examples include injection of malware, information exposure by granting excessive privileges or permissions to the untrusted functionality, DOM-based XSS vulnerabilities, stealing user's cookies, or open redirect to malware (CWE-601).",,Unknown, +This might lead to many different consequences depending on the included functionality, but some examples include injection of malware, information exposure by granting excessive privileges or permissions to the untrusted functionality, DOM-based XSS vulnerabilities, stealing user's cookies, or open redirect to malware (CWE-601).",,unclassified, CWE-830,EN-Inclusion of Web Functionality from an Untrusted Source (Type: Base),"The software includes web functionality (such as a web widget) from another domain, which causes it to operate within the domain of the software, potentially granting total access and control of the software to the untrusted source. Including third party functionality in a web-based environment is risky, especially if the source of the functionality is untrusted. Even if the third party is a trusted source, the software may still be exposed to attacks and malicious behavior if that trusted source is compromised, or if the code is modified in transmission from the third party to the software. This weakness is common in ""mashup"" development on the web, which may include source functionality from other domains. For example, Javascript-based web widgets may be inserted by using ' - diff --git a/server/www/scripts/config/services/config.js b/server/www/scripts/config/services/config.js index 99ea4fc8463..028d76b5276 100644 --- a/server/www/scripts/config/services/config.js +++ b/server/www/scripts/config/services/config.js @@ -3,17 +3,15 @@ // See the file 'doc/LICENSE' for the license information angular.module('faradayApp') - .factory('configSrv', ['BASEURL', '$http', function(BASEURL, $http) { + .factory('configSrv', ['$http', function($http) { var p = $http.get('config/config.json') .then(function(conf) { configSrv.faraday_version = conf.data.ver; - configSrv.license_db = conf.data.lic_db; }); configSrv = { faraday_version: null, - license_db: null, promise: p } diff --git a/server/www/scripts/licenses/providers/license.js b/server/www/scripts/licenses/providers/license.js index c7449582259..83236901a22 100644 --- a/server/www/scripts/licenses/providers/license.js +++ b/server/www/scripts/licenses/providers/license.js @@ -6,6 +6,7 @@ angular.module('faradayApp') .factory('License', ['BASEURL', 'configSrv', '$http', '$q', function(BASEURL, configSrv, $http, $q) { function License(data) { + var APIURL = BASEURL + "_api/v2/"; var now = new Date(), date = now.getTime() / 1000.0; @@ -59,7 +60,7 @@ angular.module('faradayApp') configSrv.promise .then(function() { - var url = BASEURL + configSrv.license_db + "/" + self._id + "?rev=" + self._rev; + var url = APIURL + "licenses/" + self._id; $http.delete(url) .then(function(resp) { @@ -79,7 +80,7 @@ angular.module('faradayApp') configSrv.promise .then(function() { - var url = BASEURL + configSrv.license_db + "/" + self._id; + var url = APIURL + "licenses/" + self._id; $http.put(url, data) .then(function(res) { @@ -105,7 +106,7 @@ angular.module('faradayApp') configSrv.promise .then(function() { - var url = BASEURL + configSrv.license_db; + var url = APIURL + "licenses/"; $http.post(url, self) .then(function(data) { diff --git a/server/www/scripts/licenses/providers/licenses.js b/server/www/scripts/licenses/providers/licenses.js index a0f21008c50..d9559094ca6 100644 --- a/server/www/scripts/licenses/providers/licenses.js +++ b/server/www/scripts/licenses/providers/licenses.js @@ -7,6 +7,7 @@ angular.module('faradayApp') ['License', 'BASEURL', 'configSrv', '$http', '$q', function(License, BASEURL, configSrv, $http, $q) { var licensesManager = {}; + var APIURL = BASEURL + "_api/v2/"; licensesManager.licenses = []; @@ -29,50 +30,6 @@ angular.module('faradayApp') "Other" ]; - licensesManager.DBExists = function() { - var deferred = $q.defer(), - self = this; - - configSrv.promise - .then(function() { - var url = BASEURL + configSrv.license_db; - - $http.head(url) - .then(function(resp) { - // status 200 - DB exists! - deferred.resolve(true); - }, function(resp) { - // status 404 - DB doesn't exist - deferred.resolve(false); - }); - }, function() { - deferred.reject("Unable to fetch licenses database name."); - }); - - return deferred.promise; - }; - - licensesManager.createDB = function() { - var deferred = $q.defer(), - self = this; - - configSrv.promise - .then(function() { - var url = BASEURL + configSrv.license_db; - - $http.put(url) - .then(function(resp) { - deferred.resolve(true); - }, function(resp) { - deferred.reject(resp); - }); - }, function() { - deferred.reject("Unable to fetch licenses database name."); - }); - - return deferred.promise; - }; - licensesManager.create = function(data) { var deferred = $q.defer(), self = this; @@ -123,7 +80,7 @@ angular.module('faradayApp') configSrv.promise .then(function() { - var url = BASEURL + configSrv.license_db + "/_all_docs?include_docs=true"; + var url = APIURL + "licenses/"; $http.get(url) .then(function(res) { From 13e23e4814c7c62bff4016a5fdd4359e06c455d6 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 12 Dec 2017 18:44:51 -0300 Subject: [PATCH 0644/1506] Fixed a bug when updating a host --- persistence/server/models.py | 2 +- persistence/server/server.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index 586c30354d3..fb42ca48733 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -351,7 +351,7 @@ def update_host(workspace_name, host): Return the server's json response as a dictionary. """ host_properties = get_host_properties(host) - return server.update_host(workspace_name, **host_properties) + return server.update_host(workspace_name, host.getID(), **host_properties) @_ignore_in_changes diff --git a/persistence/server/server.py b/persistence/server/server.py index d70c01eeb30..c69b9258ddb 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -879,7 +879,7 @@ def create_host(workspace_name, command_id, ip, os, default_gateway=None, description=description, type="Host") -def update_host(workspace_name, id, name, os, default_gateway, +def update_host(workspace_name, id, ip, os, default_gateway="", description="", metadata=None, owned=False, owner="", parent=None): """Updates a host. @@ -903,7 +903,7 @@ def update_host(workspace_name, id, name, os, default_gateway, """ return _update_in_server(workspace_name, id, - name=name, os=os, + ip=ip, os=os, default_gateway=default_gateway, owned=owned, metadata=metadata, From 6223acf35428997d98398752d4831f6e4a0b8823 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 12 Dec 2017 18:45:27 -0300 Subject: [PATCH 0645/1506] PEP8 fixes --- model/api.py | 1 + persistence/server/utils.py | 21 ++++++++++++++------- plugins/repo/nmap/plugin.py | 2 -- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/model/api.py b/model/api.py index e569b82094f..2e61b9ba2f5 100644 --- a/model/api.py +++ b/model/api.py @@ -443,6 +443,7 @@ def delEvidence(file_path): #------------------------------------------------------------------------------- # MISC APIS #------------------------------------------------------------------------------- + def log(msg ,level = "INFO"): """ This api will log the text in the GUI console without the level diff --git a/persistence/server/utils.py b/persistence/server/utils.py index 3a0d2f2218e..dd7ad579a44 100644 --- a/persistence/server/utils.py +++ b/persistence/server/utils.py @@ -9,6 +9,7 @@ import hashlib from persistence.server.server_io_exceptions import MoreThanOneObjectFoundByID + def force_unique(lst): """Takes a list and return its only member if the list len is 1, None if list is empty or raises an MoreThanOneObjectFoundByID error @@ -30,15 +31,17 @@ def get_object_properties(obj): metadata = metadata.toDict() return { - 'name': obj.getName(), - 'description': obj.getDescription(), - 'metadata': metadata, - 'owned': obj.isOwned(), - 'owner': obj.getOwner() - } + 'name': obj.getName(), + 'description': obj.getDescription(), + 'metadata': metadata, + 'owned': obj.isOwned(), + 'owner': obj.getOwner() + } + def get_host_properties(host): - host_dict = {'os': host.getOS(), + host_dict = { + 'os': host.getOS(), } if host.getDefaultGateway(): host['default_gateway'] = host.getDefaultGateway() @@ -78,6 +81,7 @@ def get_vuln_properties(vuln): vuln_dict.update(get_object_properties(vuln)) return vuln_dict + def get_vuln_web_properties(vuln_web): vuln_web_dict = { 'method': vuln_web.getMethod(), @@ -94,6 +98,7 @@ def get_vuln_web_properties(vuln_web): vuln_web_dict.update(get_vuln_properties(vuln_web)) return vuln_web_dict + def get_note_properties(note): note_dict = { 'text': note.getText(), @@ -103,12 +108,14 @@ def get_note_properties(note): note_dict.update(get_object_properties(note)) return note_dict + def get_credential_properties(credential): cred_dict = {'username': credential.getUsername(), 'password': credential.getPassword()} cred_dict.update(get_object_properties(credential)) return cred_dict + def get_command_properties(command): return { 'command': command.command, diff --git a/plugins/repo/nmap/plugin.py b/plugins/repo/nmap/plugin.py index d8364dcd7a2..6df4c6c8443 100644 --- a/plugins/repo/nmap/plugin.py +++ b/plugins/repo/nmap/plugin.py @@ -371,8 +371,6 @@ def parse_output(self, output): return m2 return [] - - def __init__(self, script_node): self.node = script_node From 40a10962ccae590451486beb1b33e8e1fde7b446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 13 Dec 2017 16:56:09 -0300 Subject: [PATCH 0646/1506] Improve status report API response time and command importer I added command related properties to the model as column properties, so I don't have to do this logic in the schema or in the view. Now that I can undefer this field, it improves performances of the status report API (in my workspace, by 4X). Also improved the way of creating CommandObject's --- server/api/modules/vulns.py | 38 ++++-------- server/importer.py | 32 +++++----- server/models.py | 77 ++++++++++++++++++------- test_cases/factories.py | 1 + test_cases/models/test_vulnerability.py | 63 +++++++++++++++++++- 5 files changed, 146 insertions(+), 65 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 9ef9fcf057f..3ee6f6918a2 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -12,7 +12,7 @@ from marshmallow import Schema, fields, post_load, ValidationError from marshmallow.validate import OneOf from sqlalchemy import and_ -from sqlalchemy.orm import joinedload, selectin_polymorphic +from sqlalchemy.orm import joinedload, selectin_polymorphic, undefer from sqlalchemy.orm.exc import NoResultFound from depot.manager import DepotManager @@ -79,26 +79,8 @@ class CustomMetadataSchema(MetadataSchema): """ Implements command_id and creator logic """ - command_id = fields.Method('get_command_id', dump_only=True) - creator = fields.Method('get_command_tool', dump_only=True) - - def get_command_obj(self, obj): - # TODO migration: this will cause an undiagnosed npusone performance - # issue. We should use column properties or something like that - command_obj = CommandObject.query.filter_by( - object_type='vulnerability', object_id=obj.id, - workspace_id=obj.workspace_id).first() - return command_obj - - def get_command_id(self, obj): - command_obj = self.get_command_obj(obj) - if command_obj: - return command_obj.command_id - - def get_command_tool(self, obj): - command_obj = self.get_command_obj(obj) - if command_obj: - return command_obj.command.tool + command_id = fields.Integer(dump_only=True, attribute='creator_command_id') + creator = fields.String(dump_only=True, attribute='creator_command_tool') class VulnerabilitySchema(AutoSchema): @@ -308,17 +290,15 @@ def filter(self): :returns: Filtered SQLALchemy query """ + # TODO migration: this can became a normal filter instead of a custom + # one, since now we can use creator_command_id command_id = request.args.get('command_id') - if command_id: - self.query = self.query.join( - CommandObject, and_( - VulnerabilityWeb.id == CommandObject.object_id, - CommandObject.object_type == 'vulnerability')) - query = super(VulnerabilityFilterSet, self).filter() if command_id: - query = query.filter(CommandObject.command_id == int(command_id)) + # query = query.filter(CommandObject.command_id == int(command_id)) + query = query.filter(VulnerabilityGeneric.creator_command_id == + int(command_id)) # TODO migration: handle invalid int() return query @@ -388,6 +368,8 @@ def _get_eagerloaded_query(self, *args, **kwargs): .joinedload(Service.host) .joinedload(Host.hostnames), + undefer(VulnerabilityGeneric.creator_command_id), + undefer(VulnerabilityGeneric.creator_command_tool), joinedload(VulnerabilityGeneric.evidence), joinedload(VulnerabilityGeneric.tags), ] diff --git a/server/importer.py b/server/importer.py index 48a0979b29b..4b39b3ad13f 100644 --- a/server/importer.py +++ b/server/importer.py @@ -31,6 +31,7 @@ from server.models import ( db, Command, + CommandObject, Comment, Credential, ExecutiveReport, @@ -54,7 +55,6 @@ VulnerabilityWeb, Workspace, File, - log_command_object_found, ) from server.utils import invalid_chars from server.utils.database import get_or_create @@ -259,8 +259,9 @@ def update_command_tools(workspace, command_tool_map, id_map): missing_tool_count = Command.query.filter_by( workspace=workspace, tool="unknown").count() if missing_tool_count: - logger.warn("Couldn't find the tool name of {} commands".format( - missing_tool_count)) + logger.warn("Couldn't find the tool name of {} commands in " + "workspace {}".format( + missing_tool_count, workspace.name)) class EntityNotFound(Exception): @@ -384,8 +385,9 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation for host, created in hosts: # we update or set other host attributes in this cycle # Ticket #3387: if the 'os' field is None, we default to 'unknown - if command: - log_command_object_found(command, host, created) + if command and created: + session.flush() + CommandObject.create(host, command) if not document.get('os'): document['os'] = 'unknown' @@ -444,8 +446,9 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation parent_id = document['_id'].split('.')[0] for relational_parent_id in couchdb_relational_map[parent_id]: host, created = get_or_create(session, Host, id=relational_parent_id) - if command: - log_command_object_found(command, host, created) + if command and created: + session.flush() + CommandObject.create(host, command) ports = document.get('ports') if len(ports) > 2: logger.warn('More than one port found in services!') @@ -480,8 +483,9 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation service.status = status_mapper.get(couchdb_status, 'open') service.version = document.get('version') service.workspace = workspace - if command: - log_command_object_found(command, service, created) + if command and created: + session.flush() + CommandObject.create(service, command) yield service @@ -557,8 +561,9 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation vulnerability.impact_availability = document.get('impact', {}).get('availability') vulnerability.impact_confidentiality = document.get('impact', {}).get('confidentiality') vulnerability.impact_integrity = document.get('impact', {}).get('integrity') - if command: - log_command_object_found(command, vulnerability, created) + if command and created: + session.flush() + CommandObject.create(vulnerability, command) if document['type'] == 'VulnerabilityWeb': vulnerability.query_string = document.get('query') vulnerability.request = document.get('request') @@ -747,8 +752,9 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation credential.description = document.get('description', None) credential.name = document.get('name', None) credential.workspace = workspace - if command: - log_command_object_found(command, credential, created) + if command and created: + session.flush() + CommandObject.create(credential, command) yield credential diff --git a/server/models.py b/server/models.py index f4986d34896..04ae2dcb0b7 100644 --- a/server/models.py +++ b/server/models.py @@ -21,6 +21,7 @@ from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import backref, relationship, undefer from sqlalchemy.sql import select, text, table +from sqlalchemy.sql.expression import asc, join from sqlalchemy import func from sqlalchemy.orm import ( backref, @@ -465,7 +466,7 @@ class CommandObject(db.Model): # We are currently using the column property created. however to avoid losing information # we also store the a boolean to know if at the moment of created the object related to the # Command was created. - created_persistent = Column(Boolean, default=False) + created_persistent = Column(Boolean, nullable=False) __table_args__ = ( UniqueConstraint('object_id', 'object_type', 'command_id', 'workspace_id', @@ -476,14 +477,44 @@ class CommandObject(db.Model): def parent(self): return self.command + @classmethod + def create(cls, obj, command, add_to_session=True, **kwargs): + co = cls(obj, workspace=command.workspace, command=command, + created_persistent=True, **kwargs) + if add_to_session: + db.session.add(co) + return co + + def __init__(self, object_=None, **kwargs): + + if object_ is not None: + assert 'object_type' not in kwargs + assert 'object_id' not in kwargs + object_type = object_.__tablename__ + if object_type is None: + if object_.__class__.__name__ in ['Vulnerability', + 'VulnerabilityWeb', + 'VulnerabilityCode']: + object_type = 'vulnerability' + else: + raise RuntimeError("Unknown table for object: {}".format( + object_)) + + # db.session.flush() + assert object_.id is not None, "object must have an ID. Try " \ + "flushing the session" + kwargs['object_id'] = object_.id + kwargs['object_type'] = object_type + return super(CommandObject, self).__init__(**kwargs) + def _make_created_objects_sum(object_type_filter): where_conditions = ["command_object.object_type= '%s'" % object_type_filter] where_conditions.append("command_object.command_id = command.id") where_conditions.append("command_object.workspace_id = command.workspace_id") return column_property( - select([func.sum(CommandObject.created)]).\ - select_from(table('command_object')). \ + select([func.sum(CommandObject.created)]). + select_from(table('command_object')). where(text(' and '.join(where_conditions))) ) @@ -612,6 +643,27 @@ class VulnerabilityGeneric(VulnerabilityABC): collection_class=set, ) + creator_command_id = column_property( + select([CommandObject.command_id]) + .where(CommandObject.object_type == 'vulnerability') + .where(text('command_object.object_id = vulnerability.id')) + .where(CommandObject.workspace_id == workspace_id) + .order_by(asc(CommandObject.create_date)) + .limit(1), + deferred=True) + + creator_command_tool = column_property( + select([Command.tool]) + .select_from(join(Command, CommandObject, + Command.id == CommandObject.command_id)) + .where(CommandObject.object_type == 'vulnerability') + .where(text('command_object.object_id = vulnerability.id')) + .where(CommandObject.workspace_id == workspace_id) + .order_by(asc(CommandObject.create_date)) + .limit(1), + deferred=True + ) + __mapper_args__ = { 'polymorphic_on': type } @@ -1317,24 +1369,5 @@ def parent(self): vulnerability_uniqueness.execute_if(dialect='postgresql') ) - -def log_command_object_found(command, object, created): - object_type = object.__tablename__ - if object.__class__.__name__ in ['Vulnerability', 'VulnerabilityWeb', 'VulnerabilityCode']: - object_type = 'vulnerability' - - db.session.flush() - log, log_created = get_or_create( - db.session, - CommandObject, - command=command, - object_id=object.id, - object_type=object_type, - workspace=object.workspace, - ) - if not log_created: - # without this if, multiple executions of the importer will write this attribute with False - log.created_persistent = created - # We have to import this after all models are defined import server.events diff --git a/test_cases/factories.py b/test_cases/factories.py index afe82e33c0b..f93d7522839 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -261,6 +261,7 @@ class Meta: class CommandObjectFactory(FaradayFactory): workspace = factory.SubFactory(WorkspaceFactory) + created_persistent = False class Meta: model = CommandObject diff --git a/test_cases/models/test_vulnerability.py b/test_cases/models/test_vulnerability.py index fa6a44868bd..11e62a9133a 100644 --- a/test_cases/models/test_vulnerability.py +++ b/test_cases/models/test_vulnerability.py @@ -1,9 +1,10 @@ import pytest from server.models import ( - Reference, - ReferenceVulnerabilityAssociation, + CommandObject, PolicyViolation, PolicyViolationVulnerabilityAssociation, + Reference, + ReferenceVulnerabilityAssociation, ) @@ -181,3 +182,61 @@ class TestPolicyViolations(TestReferences): model = PolicyViolation intermediate_model = PolicyViolationVulnerabilityAssociation intermediate_field = 'policy_violation' + + +class TestCommandProperties: + def test_no_creator_command(self, vulnerability, session): + session.commit() + assert vulnerability.creator_command_id is None + assert vulnerability.creator_command_tool is None + + def test_creator_command(self, workspace, vulnerability, + empty_command_factory, session): + command = empty_command_factory.create(workspace=workspace) + session.flush() + assert vulnerability.id is not None + assert command.id is not None + assert vulnerability.workspace is workspace + assert command.workspace is workspace + + CommandObject.create( + vulnerability, + command + ) + for _ in range(5): + new_command = empty_command_factory.create(workspace=workspace) + session.flush() + session.add(CommandObject(vulnerability, + command=new_command, + workspace=workspace, + created_persistent=False)) + session.commit() + assert vulnerability.creator_command_id == command.id + assert vulnerability.creator_command_tool == command.tool + + def test_different_object_type(self, vulnerability, workspace, + empty_command_factory, session): + command = empty_command_factory.create(workspace=workspace) + session.flush() + invalid_co = CommandObject.create(vulnerability, command) + invalid_co.object_type = 'host' + session.add(invalid_co) + session.commit() + + command = empty_command_factory.create(workspace=workspace) + session.flush() + + CommandObject.create( + vulnerability, + command + ) + for _ in range(5): + new_command = empty_command_factory.create(workspace=workspace) + session.flush() + session.add(CommandObject(vulnerability, + command=new_command, + workspace=workspace, + created_persistent=False)) + session.commit() + assert vulnerability.creator_command_id == command.id + assert vulnerability.creator_command_tool == command.tool From c48dd73c8b64957dfff263957b242d7a0e61d868 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 13 Dec 2017 18:14:48 -0300 Subject: [PATCH 0647/1506] Set command_id and creator to dump_only This is done by default in marshmallow 3, but not in marshmallow 2 --- server/schemas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/schemas.py b/server/schemas.py index 7b5d11c407d..c7394f618fc 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -109,9 +109,9 @@ def _add_to_schema(self, field_name, schema): class MetadataSchema(Schema): - command_id = fields.Function(lambda x: None) + command_id = fields.Function(lambda x: None, dump_only=True) - creator = fields.Function(lambda x: '') + creator = fields.Function(lambda x: '', dump_only=True) owner = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator') create_time = JSTimestampField(attribute='create_date', dump_only=True) From ec3fb869d1777d88f5fd9afc4776e8c300c6dbdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 13 Dec 2017 18:28:00 -0300 Subject: [PATCH 0648/1506] Fix host tests Include mac field --- test_cases/test_api_hosts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 489ed7336dc..f62c7f3a706 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -406,7 +406,9 @@ def test_update_host(self, test_client, session): u'hostnames': [], u'id': host.id, u'ip': u'10.31.112.21', - u'metadata': {u'command_id': None, + u'mac': None, + u'metadata': { + u'command_id': None, u'create_time': int(time.mktime(updated_host.create_date.timetuple())) * 1000, u'creator': u'', u'owner': host.creator.username, From 6187bd14db40f2c93c60c7a31941a5bcf2c36d66 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 13 Dec 2017 18:34:18 -0300 Subject: [PATCH 0649/1506] Add command_id on all update models with test included --- managers/mapper_manager.py | 5 +- model/controller.py | 18 +-- persistence/server/models.py | 18 +-- persistence/server/server.py | 31 ++-- persistence/server/utils.py | 2 +- server/api/base.py | 52 ++++--- test_cases/test_managers_mapper_manager.py | 150 +++++++++++++++++++ test_cases/test_persistence_server_models.py | 0 8 files changed, 223 insertions(+), 53 deletions(-) create mode 100644 test_cases/test_managers_mapper_manager.py create mode 100644 test_cases/test_persistence_server_models.py diff --git a/managers/mapper_manager.py b/managers/mapper_manager.py index 658ce69514e..b97e2c85302 100644 --- a/managers/mapper_manager.py +++ b/managers/mapper_manager.py @@ -3,7 +3,6 @@ Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) See the file 'doc/LICENSE' for the license information ''' -from requests import Session from persistence.server.models import create_object, get_object, update_object, delete_object # NOTE: This class is intended to be instantiated by the @@ -28,8 +27,8 @@ def save(self, obj, command_id=None): return saved_raw_obj['_id'] return False - def update(self, obj): - if update_object(self.workspace_name, obj.class_signature, obj): + def update(self, obj, command_id=None): + if update_object(self.workspace_name, obj.class_signature, obj, command_id): return True return False diff --git a/model/controller.py b/model/controller.py index 1f2ca014b88..b8776acc79c 100644 --- a/model/controller.py +++ b/model/controller.py @@ -387,20 +387,20 @@ def __addPendingAction(self, *args): new_action = args self._pending_actions.put(new_action) - def addUpdate(self, old_object, new_object): + def addUpdate(self, old_object, new_object, command_id): # Returns True if the update was resolved without user interaction try: - mergeAction = old_object.addUpdate(new_object) + mergeAction = old_object.addUpdate(new_object, command_id) if mergeAction: if old_object not in self.objects_with_updates: self.objects_with_updates.append(old_object) notifier.conflictUpdate(1) return False - except: + except Exception as ex: api.devlog("(%s).addUpdate(%s, %s) - failed" % (self, old_object, new_object)) return False - self.mappers_manager.update(old_object) + self.mappers_manager.update(old_object, command_id) notifier.editHost(old_object) return True @@ -415,9 +415,9 @@ def _save_new_object(self, new_object, command_id): notifier.addObject(new_object) return res - def _handle_conflict(self, old_obj, new_obj): + def _handle_conflict(self, old_obj, new_obj, command_id): if not old_obj.needs_merge(new_obj): return True - return self.addUpdate(old_obj, new_obj) + return self.addUpdate(old_obj, new_obj, command_id) def __add(self, new_obj, command_id=None, *args): """ @@ -433,11 +433,11 @@ def __add(self, new_obj, command_id=None, *args): except ConflictInDatabase as conflict: old_obj = new_obj.__class__(conflict.answer.json()['object'], new_obj._workspace_name) new_obj.setID(old_obj.getID()) - return self._handle_conflict(old_obj, new_obj) + return self._handle_conflict(old_obj, new_obj, command_id) - def __edit(self, obj, *args, **kwargs): + def __edit(self, obj, command_id=None, *args, **kwargs): obj.updateAttributes(*args, **kwargs) - self.mappers_manager.update(obj) + self.mappers_manager.update(obj, command_id) notifier.editHost(obj) return True diff --git a/persistence/server/models.py b/persistence/server/models.py index fb42ca48733..411144ea22e 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -345,13 +345,13 @@ def create_host(workspace_name, host, command_id): @_ignore_in_changes -def update_host(workspace_name, host): +def update_host(workspace_name, host, command_id): """Take a workspace_name and a host object and update it in the sever. Return the server's json response as a dictionary. """ host_properties = get_host_properties(host) - return server.update_host(workspace_name, host.getID(), **host_properties) + return server.update_host(workspace_name, command_id, host.getID(), **host_properties) @_ignore_in_changes @@ -370,7 +370,7 @@ def update_service(workspace_name, service): Return the server's json response as a dictionary. """ service_properties = get_service_properties(service) - return server.update_service(workspace_name, **service_properties) + return server.update_service(workspace_name, service.getID(), **service_properties) @_ignore_in_changes @@ -390,7 +390,7 @@ def update_vuln(workspace_name, vuln): Return the server's json response as a dictionary. """ vuln_properties = get_vuln_properties(vuln) - return server.update_vuln(workspace_name, **vuln_properties) + return server.update_vuln(workspace_name, vuln.getID(), **vuln_properties) @_ignore_in_changes @@ -410,7 +410,7 @@ def update_vuln_web(workspace_name, vuln_web): Return the server's json response as a dictionary. """ vuln_web_properties = get_vuln_web_properties(vuln_web) - return server.update_vuln_web(workspace_name, **vuln_web_properties) + return server.update_vuln_web(workspace_name, vuln_web.getID(), **vuln_web_properties) @_ignore_in_changes @@ -428,7 +428,7 @@ def update_note(workspace_name, note): Return the server's json response as a dictionary. """ note_properties = get_note_properties(note) - return server.update_note(workspace_name, **note_properties) + return server.update_note(workspace_name, note.getID(), **note_properties) @_ignore_in_changes @@ -446,7 +446,7 @@ def update_credential(workspace_name, credential): Return the server's json response as a dictionary. """ credential_properties = get_credential_properties(credential) - return server.update_credential(workspace_name, **credential_properties) + return server.update_credential(workspace_name, credential.getID(), **credential_properties) @_ignore_in_changes @@ -498,7 +498,7 @@ def create_object(workspace_name, object_signature, obj, command_id): return appropiate_function(workspace_name, obj, command_id) -def update_object(workspace_name, object_signature, obj): +def update_object(workspace_name, object_signature, obj, command_id): """Given a workspace name, an object_signature as string and obj, a Faraday object, update that object on the server. @@ -521,7 +521,7 @@ def update_object(workspace_name, object_signature, obj): except KeyError: raise WrongObjectSignature(object_signature) - return appropiate_function(workspace_name, obj) + return appropiate_function(workspace_name, obj, command_id) def create_workspace(workspace_name, description, start_date, finish_date, diff --git a/persistence/server/server.py b/persistence/server/server.py index c69b9258ddb..2aa9358e9a7 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -118,11 +118,15 @@ def _create_server_post_url(workspace_name, obj_type, command_id): return post_url -def _create_server_put_url(workspace_name, obj_type, obj_id): +def _create_server_put_url(workspace_name, obj_type, obj_id, command_id): server_api_url = _create_server_api_url() object_end_point_name = OBJECT_TYPE_END_POINT_MAPPER[obj_type] - pust_url = '{0}/ws/{1}/{2}/{3}/'.format(server_api_url, workspace_name, object_end_point_name, obj_id) - return pust_url + put_url = '{0}/ws/{1}/{2}/{3}/'.format(server_api_url, workspace_name, object_end_point_name, obj_id) + if command_id: + get_params = {'command_id': command_id} + put_url += '?' + urlencode(get_params) + print(put_url) + return put_url def _create_server_delete_url(workspace_name, object_id): @@ -302,7 +306,6 @@ def _save_to_server(workspace_name, **params): """ :param workspace_name: - :param command_id: :param params: :return: """ @@ -310,7 +313,7 @@ def _save_to_server(workspace_name, **params): return _post(post_url, update=False, expected_response=201, **params) def _update_in_server(workspace_name, faraday_object_id, **params): - put_url = _create_server_put_url(workspace_name, params['type'], faraday_object_id) + put_url = _create_server_put_url(workspace_name, params['type'], faraday_object_id, params.get('command_id', None)) return _put(put_url, expected_response=200, **params) def _save_db_to_server(db_name, **params): @@ -879,7 +882,7 @@ def create_host(workspace_name, command_id, ip, os, default_gateway=None, description=description, type="Host") -def update_host(workspace_name, id, ip, os, default_gateway="", +def update_host(workspace_name, command_id, id, ip, os, default_gateway="", description="", metadata=None, owned=False, owner="", parent=None): """Updates a host. @@ -903,6 +906,7 @@ def update_host(workspace_name, id, ip, os, default_gateway="", """ return _update_in_server(workspace_name, id, + command_id=command_id, ip=ip, os=os, default_gateway=default_gateway, owned=owned, @@ -949,7 +953,7 @@ def create_service(workspace_name, command_id, name, description, ports, parent, type="Service", metadata=metadata) -def update_service(workspace_name, id, name, description, ports, +def update_service(workspace_name, command_id, id, name, description, ports, owned=False, owner="", protocol="", status="", version="", metadata=None): """Creates a service. @@ -973,6 +977,7 @@ def update_service(workspace_name, id, name, description, ports, """ return _update_in_server(workspace_name, id, + command_id=command_id, name=name, description=description, ports=ports, @@ -1032,7 +1037,7 @@ def create_vuln(workspace_name, command_id, name, description, parent, parent_ty metadata=metadata, policyviolations=policyviolations) -def update_vuln(workspace_name, id, name, description, owned=None, owner="", +def update_vuln(workspace_name, command_id, id, name, description, owned=None, owner="", confirmed=False, data="", refs=None, severity="info", resolution="", desc="", metadata=None, status=None, policyviolations=[]): """Updates a vuln. @@ -1061,6 +1066,7 @@ def update_vuln(workspace_name, id, name, description, owned=None, owner="", """ return _update_in_server(workspace_name, id, + command_id=command_id, name=name, description=description, owned=owned, @@ -1138,7 +1144,7 @@ def create_vuln_web(workspace_name, command_id, name, description, owned=None, o type='VulnerabilityWeb', policyviolations=policyviolations) -def update_vuln_web(workspace_name, id, name, description, owned=None, owner="", +def update_vuln_web(workspace_name, command_id, id, name, description, owned=None, owner="", confirmed=False, data="", refs=None, severity="info", resolution="", desc="", metadata=None, method=None, params="", path=None, pname=None, query=None, request=None, response=None, category="", website=None, @@ -1176,6 +1182,7 @@ def update_vuln_web(workspace_name, id, name, description, owned=None, owner="", """ return _update_in_server(workspace_name, id, + command_id=command_id, name=name, description=description, owned=owned, @@ -1230,7 +1237,7 @@ def create_note(workspace_name, command_id, object_type, object_id, name, text, type="Note", metadata=metadata) -def update_note(workspace_name, id, name, text, owned=None, owner="", +def update_note(workspace_name, command_id, id, name, text, owned=None, owner="", description="", metadata=None): """Updates a note. @@ -1250,6 +1257,7 @@ def update_note(workspace_name, id, name, text, owned=None, owner="", """ return _update_in_server(workspace_name, id, + command_id=command_id, name=name, description=description, owned=owned, @@ -1289,7 +1297,7 @@ def create_credential(workspace_name, command_id, name, username, password, password=password, type="Cred") -def update_credential(workspace_name, id, name, username, password, +def update_credential(workspace_name, command_id, id, name, username, password, owned=None, owner="", description="", metadata=None): """Updates a credential. @@ -1310,6 +1318,7 @@ def update_credential(workspace_name, id, name, username, password, """ return _update_in_server(workspace_name, id, + command_id=command_id, name=name, description=description, owned=owned, diff --git a/persistence/server/utils.py b/persistence/server/utils.py index dd7ad579a44..9cc3220a142 100644 --- a/persistence/server/utils.py +++ b/persistence/server/utils.py @@ -44,7 +44,7 @@ def get_host_properties(host): 'os': host.getOS(), } if host.getDefaultGateway(): - host['default_gateway'] = host.getDefaultGateway() + host_dict['default_gateway'] = host.getDefaultGateway() host_dict.update(get_object_properties(host)) # name was removed from host and changed to ip ip = host_dict.pop('name') diff --git a/server/api/base.py b/server/api/base.py index 1f5c9e9eb64..98c0f2805f6 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -428,26 +428,17 @@ def _perform_create(self, data, **kwargs): return obj -class CreateWorkspacedMixin(CreateMixin): - """Add POST // route""" - - def _perform_create(self, data, workspace_name): - assert not db.session.new - workspace = self._get_workspace(workspace_name) - obj = self.model_class(**data) - obj.workspace = workspace - # assert not db.session.new - with db.session.no_autoflush: - # Required because _validate_uniqueness does a select. Doing this - # outside a no_autoflush block would result in a premature create. - self._validate_uniqueness(obj) - db.session.add(obj) - db.session.commit() - self._set_command_id(obj) - return obj +class CommandMixing(): + """ + Created the command obj to log model activity after a command + execution via the api (ex. from plugins) + This will use GET parameter command_id. + NOTE: GET parameters are also available in POST requests + """ - def _set_command_id(self, obj): + def _set_command_id(self, obj, created): try: + # validates the data type from user input. command_id = int(flask.request.args.get('command_id', None)) except TypeError: command_id = None @@ -461,12 +452,31 @@ def _set_command_id(self, obj): object_type=obj.__class__.__name__, command=command, workspace=obj.workspace, - created_persistent=True + created_persistent=created ) db.session.add(command) db.session.add(command_object) +class CreateWorkspacedMixin(CreateMixin, CommandMixing): + """Add POST // route""" + + def _perform_create(self, data, workspace_name): + assert not db.session.new + workspace = self._get_workspace(workspace_name) + obj = self.model_class(**data) + obj.workspace = workspace + # assert not db.session.new + with db.session.no_autoflush: + # Required because _validate_uniqueness does a select. Doing this + # outside a no_autoflush block would result in a premature create. + self._validate_uniqueness(obj) + db.session.add(obj) + db.session.commit() + self._set_command_id(obj, True) + return obj + + class UpdateMixin(object): """Add PUT /// route""" @@ -492,13 +502,15 @@ def _perform_update(self, object_id, obj): db.session.commit() -class UpdateWorkspacedMixin(UpdateMixin): +class UpdateWorkspacedMixin(UpdateMixin, CommandMixing): """Add PUT // route""" def _perform_update(self, object_id, obj, workspace_name): assert not db.session.new with db.session.no_autoflush: obj.workspace = self._get_workspace(workspace_name) + + self._set_command_id(obj, False) return super(UpdateWorkspacedMixin, self)._perform_update( object_id, obj) diff --git a/test_cases/test_managers_mapper_manager.py b/test_cases/test_managers_mapper_manager.py new file mode 100644 index 00000000000..7d067456a8d --- /dev/null +++ b/test_cases/test_managers_mapper_manager.py @@ -0,0 +1,150 @@ +from functools import partial +from managers.mapper_manager import MapperManager +from persistence.server.models import Host +import persistence.server.server +from test_cases.factories import WorkspaceFactory, CommandFactory, HostFactory + +# OBJ_DATA is like a fixture. +# We use it to test all model classes. +# to add more tests you need to add items in the list or more objects in the dict. +OBJ_DATA = { + Host: { + 'factory': HostFactory, + 'api_end_point': 'hosts', + 'data': { + '_id': 1, + 'name': '192.168.0.20', + 'description': 'My computer', + 'default_gateway': '192.168.0.1', + 'os': 'Debian', + 'owned': False, + 'owner': 'leo' + }, + 'expected_payload': { + 'command_id': None, + 'default_gateway': '192.168.0.1', + 'description': 'My computer', + 'ip': '192.168.0.20', + 'os': 'Debian', + 'owned': False, + 'owner': 'leo', + 'parent': None, + 'type': 'Host' + }, + } +} + +class TestMapperManager(): + + + def test_save_without_command(self, monkeypatch, session): + workspace = WorkspaceFactory.create(name='test') + session.commit() + mapper_manager = MapperManager() + mapper_manager.createMappers(workspace.name) + for obj_class, test_data in OBJ_DATA.items(): + raw_data = test_data['data'] + + def mock_server_post(test_data, post_url, update=False, expected_response=201, **params): + assert post_url == 'http://localhost:5985/_api/v2/ws/test/{0}/'.format(test_data['api_end_point']) + assert expected_response == 201 + assert update == False + metadata = params.pop('metadata') + assert metadata['owner'] == 'leo' + assert params == test_data['expected_payload'] + return { + 'id': 1, + 'ok': True, + 'rev': '' + } + + monkeypatch.setattr(persistence.server.server, '_post', partial(mock_server_post, test_data)) + obj = obj_class(raw_data, workspace.name) + mapper_manager.save(obj) + + def test_save_with_command(self, monkeypatch, session): + workspace = WorkspaceFactory.create(name='test') + command = CommandFactory.create(workspace=workspace) + session.commit() + mapper_manager = MapperManager() + mapper_manager.createMappers(workspace.name) + for obj_class, test_data in OBJ_DATA.items(): + raw_data = test_data['data'] + + def mock_server_post(test_data, post_url, update=False, expected_response=201, **params): + assert post_url == 'http://localhost:5985/_api/v2/ws/test/{0}/?command_id={1}'.format(test_data['api_end_point'], params['command_id']) + assert expected_response == 201 + assert update == False + metadata = params.pop('metadata') + assert metadata['owner'] == 'leo' + params.pop('command_id') + test_data['expected_payload'].pop('command_id') + assert params == test_data['expected_payload'] + return { + 'id': 1, + 'ok': True, + 'rev': '' + } + + monkeypatch.setattr(persistence.server.server, '_post', partial(mock_server_post, test_data)) + obj = obj_class(raw_data, workspace.name) + mapper_manager.save(obj, command.id) + + def test_update_without_command(self, monkeypatch, session): + workspace = WorkspaceFactory.create(name='test') + mapper_manager = MapperManager() + mapper_manager.createMappers(workspace.name) + for obj_class, test_data in OBJ_DATA.items(): + relational_model = test_data['factory'].create() + session.commit() + raw_data = test_data['data'] + def mock_server_put(test_data, put_url, update=False, expected_response=201, **params): + assert put_url == 'http://localhost:5985/_api/v2/ws/test/{0}/{1}/'.format(test_data['api_end_point'], test_data['id']) + assert expected_response == 200 + assert update == False + metadata = params.pop('metadata') + assert metadata['owner'] == 'leo' + params.pop('command_id') + test_data['expected_payload'].pop('command_id', None) + assert params == test_data['expected_payload'] + + return { + 'id': 1, + 'ok': True, + 'rev': '' + } + + raw_data['id'] = relational_model.id + test_data['id'] = relational_model.id + monkeypatch.setattr(persistence.server.server, '_put', partial(mock_server_put, test_data)) + + obj = obj_class(raw_data, workspace.name) + mapper_manager.update(obj) + + def test_update_with_command(self, monkeypatch, session): + session.commit() + workspace = WorkspaceFactory.create(name='test') + command = CommandFactory.create(workspace=workspace) + session.commit() + mapper_manager = MapperManager() + mapper_manager.createMappers(workspace.name) + for obj_class, test_data in OBJ_DATA.items(): + raw_data = test_data['data'] + relational_model = test_data['factory'].create() + session.commit() + def mock_server_put(put_url, update=False, expected_response=201, **params): + assert put_url == 'http://localhost:5985/_api/v2/ws/test/{0}/{1}/?command_id={2}'.format(test_data['api_end_point'], test_data['id'], params['command_id']) + assert expected_response == 200 + assert update == False + + return { + 'id': 1, + 'ok': True, + 'rev': '' + } + + raw_data['id'] = relational_model.id + test_data['id'] = relational_model.id + monkeypatch.setattr(persistence.server.server, '_put', mock_server_put) + obj = obj_class(raw_data, workspace.name) + mapper_manager.update(obj, command.id) \ No newline at end of file diff --git a/test_cases/test_persistence_server_models.py b/test_cases/test_persistence_server_models.py new file mode 100644 index 00000000000..e69de29bb2d From cc21374832e2f2f2d4db1e588edb8c0bae18e531 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 13 Dec 2017 18:43:37 -0300 Subject: [PATCH 0650/1506] Add new feature: Get exploits in WEBUI --- bin/getExploits.py | 143 ------------------ server/api/modules/get_exploits.py | 64 ++++++++ .../www/scripts/commons/controllers/modal.js | 19 ++- .../commons/partials/exploitsModal.html | 38 +++++ .../www/scripts/commons/providers/server.js | 20 ++- .../statusReport/controllers/statusReport.js | 38 ++++- .../statusReport/partials/statusReport.html | 1 + 7 files changed, 165 insertions(+), 158 deletions(-) delete mode 100755 bin/getExploits.py create mode 100644 server/api/modules/get_exploits.py create mode 100644 server/www/scripts/commons/partials/exploitsModal.html diff --git a/bin/getExploits.py b/bin/getExploits.py deleted file mode 100755 index a7072ca20cc..00000000000 --- a/bin/getExploits.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python2.7 -# -*- coding: utf-8 -*- - -""" -Faraday Penetration Test IDE -Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) -See the file 'doc/LICENSE' for the license information - -Autor: Ezequiel Tavella - -This script get all CVEs of vulns in the active workspace and search -for exploits in the vFeed database. -Support : Exploit-db, Metasploit, Milworm, D2, Saint - -Thanks ToolsWatch!!! -www.toolswatch.org -""" - -import os -import sqlite3 - -from persistence.server import models - -__description__ = 'Get possible exploits from open services' -__prettyname__ = 'Get Exploits' - -faraday_directory = os.path.dirname(os.path.realpath(os.path.join(__file__, "../"))) - -DB_PATH = os.path.join(faraday_directory, 'data/vfeed.db') -URL_DB = "http://www.toolswatch.org/vfeed/vfeed.db.tgz" - - -def getExploits(cve_id, cursor): - result = { - 'exploit-db': [], - 'metasploit': [], - 'milworm': [], - 'd2': [], - 'saint': [] - } - - value = (cve_id.upper(),) - - # D2 exploits - consult = cursor.execute( - "SELECT d2_script_file FROM map_cve_d2 WHERE cveid = ?", - value - ) - - for row in consult: - for i in row: - result['d2'].append(i) - - # Exploit-db exploits - consult = cursor.execute( - "SELECT exploitdbscript FROM map_cve_exploitdb WHERE cveid = ?", - value - ) - - for row in consult: - for i in row: - result['exploit-db'].append(i) - - # Metasploit exploits - consult = cursor.execute( - "SELECT msf_script_file FROM map_cve_msf WHERE cveid = ?", - value - ) - - for row in consult: - for i in row: - result['metasploit'].append(i) - - # Milworm exploits - consult = cursor.execute( - "SELECT milw0rmid FROM map_cve_milw0rm WHERE cveid = ?", - value - ) - - for row in consult: - for i in row: - result['milworm'].append(i) - - # Saint exploits - consult = cursor.execute( - "SELECT saintexploitlink FROM map_cve_saint WHERE cveid = ?", - value - ) - - for row in consult: - for i in row: - result['saint'].append(i) - - return result - - -def printExploits(vuln, references, cursor): - global getExploits - - for ref in references: - if ref.startswith('CVE') or ref.startswith('cve'): - - ret = getExploits(ref, cursor) - if ret: - print '[Exploits ' + vuln + ' ' + ref + ']\n' - - for tool, info in ret.iteritems(): - - if not info: - continue - print '[Tool] ' + tool - - for path in info: - print path - print '\n' - - -def main(workspace='', args=None, parser=None): - print '[*]Checking DB...' - - if not os.path.isfile(DB_PATH): - print '[!]DB not found: please download the DB from: ' + URL_DB - print '[!]Extract this to $FARADAY/data/ and try again!' - raise Exception('DB not found', 'Check if DB exists') - - print '[*]DB Found!' - print '[*]Searching exploits...\n' - - connection = sqlite3.connect(DB_PATH) - cursor = connection.cursor() - - for host in models.get_hosts(workspace): - for v in host.getVulns(): - print '[' + host.name + '] ' + v.name - printExploits(v.name, v.getRefs(), cursor) - - for i in host.getAllInterfaces(): - for s in i.getAllServices(): - for v in s.getVulns(): - print '[' + host.name + '] ' + v.name - printExploits(v.name, v.getRefs(), cursor) - - return 0, None diff --git a/server/api/modules/get_exploits.py b/server/api/modules/get_exploits.py new file mode 100644 index 00000000000..d044874141b --- /dev/null +++ b/server/api/modules/get_exploits.py @@ -0,0 +1,64 @@ +# Faraday Penetration Test IDE +# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) +# See the file 'doc/LICENSE' for the license information + +import flask +import requests + +from server.app import app +from server.utils.logger import get_logger +from server.utils.web import gzipped + +@gzipped +@app.route('/vulners/exploits/', methods=['GET']) +def get_exploits(cveid): + """ + Use Vulns API to get all exploits available for a specific CVE-ID + """ + + get_logger(__name__).debug( + "Request parameters: {!r}".format(flask.request.args)) + + headers = { + "Content-Type": "application/json", + "Accept": "application/json"} + + response = requests.get( + "https://vulners.com/api/v3/search/id/?references=True&id=" + cveid, + headers=headers + ) + + try: + + json_response = { + "metasploit": [], + "exploitdb": [], + "cveid": cveid + } + + if response.status_code == 200: + + obj_response = response.json() + if obj_response["data"]["references"] is not {}: + + metasploit_modules = obj_response["data"]["references"][cveid]["metasploit"] + exploitdb_modules = obj_response["data"]["references"][cveid]["exploitdb"] + + for module in metasploit_modules: + obj_module = {} + obj_module.update({"title": module["title"]}) + obj_module.update({"id": module["id"]}) + obj_module.update({"href": module["href"]}) + json_response["metasploit"].append(obj_module) + + for module in exploitdb_modules: + obj_module = {} + obj_module.update({"title": module["title"]}) + obj_module.update({"id": module["id"]}) + obj_module.update({"href": module["href"]}) + json_response["exploitdb"].append(obj_module) + + except: + return flask.abort(500) + + return flask.jsonify(json_response) diff --git a/server/www/scripts/commons/controllers/modal.js b/server/www/scripts/commons/controllers/modal.js index 345ffad4f87..b337c9c1e52 100644 --- a/server/www/scripts/commons/controllers/modal.js +++ b/server/www/scripts/commons/controllers/modal.js @@ -182,7 +182,7 @@ angular.module('faradayApp') $scope.data.refs = refs; $modalInstance.close($scope.data); - } + } }; $scope.newReference = function() { @@ -198,4 +198,19 @@ angular.module('faradayApp') $scope.cancel = function() { $modalInstance.dismiss(); } - }]); \ No newline at end of file + }]); + +angular.module('faradayApp') + .controller('commonsModalExploitsCtrl', + ['$scope', '$modalInstance', 'msg', + function($scope, $modalInstance, msg) { + + $scope.metasploit = msg['metasploit']; + $scope.exploitdb = msg['exploitdb']; + $scope.cveid = msg['cveid']; + + $scope.ok = function() { + $modalInstance.close(); + }; + +}]); \ No newline at end of file diff --git a/server/www/scripts/commons/partials/exploitsModal.html b/server/www/scripts/commons/partials/exploitsModal.html new file mode 100644 index 00000000000..682ac57bd4c --- /dev/null +++ b/server/www/scripts/commons/partials/exploitsModal.html @@ -0,0 +1,38 @@ + + + + +
    +

    + + Exploit(s) found for vulnerability! +

    +
    + +
    +
    + Based on the {{cveid}} the following has been found: +

    Metasploit

    +
    + +
    + {{n.title | lowercase}} +
    {{ n.id.replace('MSF:','use ') | lowercase }}
    + https://www.rapid7.com/db/modules/{{ n.id.replace('MSF:','') | lowercase }} +
    + +
    +

    ExploitDB

    +
    + +
    + {{n.id}} | {{n.title}} +
    + +
    +
    + +
    +
    + +
    diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index 145a3661aaa..623d544b088 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -34,7 +34,7 @@ angular.module("faradayApp") return APIURL + "ws/" + wsName; } - var createDeleteUrl = createPostUrl; + var createDeleteUrl = createPostUrl; var serverComm = function(method, url, data) { var success = function (response) { @@ -47,7 +47,7 @@ angular.module("faradayApp") // return a promise :) if (method === 'GET' || method === 'DELETE') { return $http({method: method, url: url, params: data}).then(success).catch(error); - } else { + } else { return $http({method: method, url: url, data: data}).then(success).catch(error); } }; @@ -79,7 +79,7 @@ angular.module("faradayApp") if (typeof rev_provided === "undefined") {var rev_provided = false;} var deferred = $q.defer(); var data = {}; - + if (rev_provided === false) { get(url).then( function s(r) { @@ -162,7 +162,7 @@ angular.module("faradayApp") var modCredential = function(createOrUpdate, wsName, credential) { if (typeof credential.owner === "undefined") {credential.owner = ""}; if (typeof credential.description === "undefined") {credential.description = ""}; - return createOrUpdate(wsName, credential._id, credential); + return createOrUpdate(wsName, credential._id, credential); } var modCommand = function(createOrUpdate, wsName, command) { @@ -193,7 +193,7 @@ angular.module("faradayApp") var url = createGetUrl(wsName, 'hosts'); return get(url, data); } - + ServerAPI.getVulns = function(wsName, data) { var getUrl = createGetUrl(wsName, 'vulns'); return get(getUrl, data); @@ -241,7 +241,7 @@ angular.module("faradayApp") if (confirmed !== undefined) { payload.confirmed = confirmed; } - + return get(getUrl, payload); } @@ -287,11 +287,11 @@ angular.module("faradayApp") var url = createGetUrl(wsName, 'vulns') + '/count'; var payload = {'group_by': 'severity'} - + if (confirmed !== undefined) { payload.confirmed = confirmed; } - + return get(url, payload) } @@ -449,5 +449,9 @@ angular.module("faradayApp") return _delete(dbUrl, false); } + ServerAPI.getExploits = function(cveId) { + return get(APIURL + 'vulners/exploits/' + cveId); + } + return ServerAPI; }]); diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index a2cb6c1f083..80b978c72cd 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -7,11 +7,11 @@ angular.module('faradayApp') ['$scope', '$filter', '$routeParams', '$location', '$uibModal', '$cookies', '$q', '$window', 'BASEURL', 'SEVERITIES', 'EASEOFRESOLUTION', 'STATUSES', 'hostsManager', 'commonsFact', - 'vulnsManager', 'workspacesFact', 'csvService', 'uiGridConstants', 'vulnModelsManager', + 'vulnsManager', 'workspacesFact', 'csvService', 'uiGridConstants', 'vulnModelsManager','ServerAPI', function($scope, $filter, $routeParams, $location, $uibModal, $cookies, $q, $window, BASEURL, SEVERITIES, EASEOFRESOLUTION, STATUSES, hostsManager, commonsFact, - vulnsManager, workspacesFact, csvService, uiGridConstants, vulnModelsManager) { + vulnsManager, workspacesFact, csvService, uiGridConstants, vulnModelsManager, ServerAPI) { $scope.baseurl; $scope.columns; $scope.easeofresolution; @@ -441,6 +441,34 @@ angular.module('faradayApp') return res; }; + $scope.searchExploits = function(){ + var selected = $scope.getCurrentSelection(); + + selected.forEach(function(vuln){ + + vuln.refs.forEach(function(ref){ + + if(ref.toLowerCase().startsWith('cve')){ + + var response = ServerAPI.getExploits(ref); + response.then(function(res) { + + var modal = $uibModal.open({ + templateUrl: 'scripts/commons/partials/exploitsModal.html', + controller: 'commonsModalExploitsCtrl', + resolve: { + msg: function() { + return res.data; + } + } + }); + }, function(error){ + showMessage("Exploit for this vulnerability not found", false); + }); + } + }); + }); + }; $scope.saveAsModel = function() { var self = this; @@ -930,13 +958,13 @@ angular.module('faradayApp') // Add the total amount of vulnerabilities as an option for pagination // if it is larger than our biggest page size if ($scope.gridOptions.totalItems > paginationOptions.defaultPageSizes[paginationOptions.defaultPageSizes.length - 1]) { - + $scope.gridOptions.paginationPageSizes = paginationOptions.defaultPageSizes.concat([$scope.gridOptions.totalItems]); - + // sadly, this will load the vuln list again because it fires a paginationChanged event if ($scope.gridOptions.paginationPageSize > $scope.gridOptions.totalItems) $scope.gridOptions.paginationPageSize = $scope.gridOptions.totalItems; - + // New vuln and MAX items per page setted => reload page size. if ($scope.gridOptions.paginationPageSize === $scope.gridOptions.totalItems - 1) $scope.gridOptions.paginationPageSize = $scope.gridOptions.totalItems; diff --git a/server/www/scripts/statusReport/partials/statusReport.html b/server/www/scripts/statusReport/partials/statusReport.html index cbeaff33672..4b9e8b5336e 100644 --- a/server/www/scripts/statusReport/partials/statusReport.html +++ b/server/www/scripts/statusReport/partials/statusReport.html @@ -45,6 +45,7 @@

    Status report for {{ workspace

    From 2a8ac71219c29ed6b802b6d98e6a6bbc931990f9 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 15 Dec 2017 14:47:58 -0300 Subject: [PATCH 0663/1506] Fix/update and improve unit tests --- test_cases/plugins/test_acunetix.py | 57 +++++++++++++-------- test_cases/plugins/test_burp.py | 48 ++++++++++-------- test_cases/plugins/test_nessus.py | 46 ++++++++++------- test_cases/plugins/test_nexpose_full.py | 41 ++++++++------- test_cases/plugins/test_nmap.py | 58 ++++++++++++++-------- test_cases/plugins/test_ping.py | 29 ++++++++--- test_cases/plugins/test_telnet.py | 32 ++++++++---- test_cases/plugins/test_whois.py | 28 +++++++---- test_cases/test_managers_mapper_manager.py | 8 +-- test_cases/test_models.py | 15 ------ test_cases/test_plugins_controller.py | 34 ++++++++----- test_cases/test_server_io.py | 30 +++-------- 12 files changed, 245 insertions(+), 181 deletions(-) diff --git a/test_cases/plugins/test_acunetix.py b/test_cases/plugins/test_acunetix.py index 0b151906788..6e80b31510a 100644 --- a/test_cases/plugins/test_acunetix.py +++ b/test_cases/plugins/test_acunetix.py @@ -10,7 +10,12 @@ import unittest import sys +from Queue import Queue +from collections import defaultdict + import os +import pytest + sys.path.append(os.path.abspath(os.getcwd())) from plugins.repo.acunetix.plugin import AcunetixPlugin from model.common import factory @@ -21,15 +26,18 @@ Note, Host, Service, -) + ModelBase) from plugins.modelactions import modelactions -class AcunetixParserTest(unittest.TestCase): + +class TestAcunetixParser: cd = os.path.dirname(os.path.realpath(__file__)) - def setUp(self): + + + def test_Plugin_creates_apropiate_objects(self, monkeypatch): self.plugin = AcunetixPlugin() factory.register(Host) factory.register(Service) @@ -38,25 +46,32 @@ def setUp(self): factory.register(Note) factory.register(Credential) - def test_Plugin_creates_apropiate_objects(self): + pending_actions = Queue() + # getID will wait for faraday-server api response. + # Since the thread model controller is not running + # no object will be persisted. + # The mock is to simulated the api response + monkeypatch.setattr(ModelBase, 'getID', lambda _: 1) + self.plugin.set_actions_queue(pending_actions) self.plugin.processReport(self.cd + '/acunetix_xml') - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDHOST) - self.assertEqual(action[1].name, "5.175.17.140") - # action = self.plugin._pending_actions.get(block=True) - # self.assertEqual(action[0], modelactions.ADDINTERFACE) - # self.assertEqual(action[2].name, "5.175.17.140") - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDSERVICEINT) - self.assertEqual(action[3].ports, [80]) - self.assertEqual(action[3].name, 'http') - self.assertEqual(action[3].protocol, 'tcp') - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDNOTESRV) - action = self.plugin._pending_actions.get(block=True) - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDVULNWEBSRV) - self.assertEqual(action[3].name, "ASP.NET error message") + actions = defaultdict(list) + while not pending_actions.empty(): + action = self.plugin._pending_actions.get(block=True) + actions[action[0]].append(action[1]) + + assert actions.keys() == [2000, 20008, 2027, 2037, 2039] + assert len(actions[2000]) == 1 + assert actions[2000][0].name == "5.175.17.140" + assert len(actions[20008]) == 1 + assert len(actions[2027]) == 1 + assert len(actions[2037]) == 52 + assert len(actions[2039]) == 1 + + assert actions[20008][0].ports == [80] + assert actions[20008][0].name == 'http' + assert actions[20008][0].protocol == 'tcp' + + assert "ASP.NET error message" in map(lambda vuln_web: vuln_web.name, actions[2037]) if __name__ == '__main__': diff --git a/test_cases/plugins/test_burp.py b/test_cases/plugins/test_burp.py index 62d14ce854d..a38a350c4e4 100644 --- a/test_cases/plugins/test_burp.py +++ b/test_cases/plugins/test_burp.py @@ -10,6 +10,9 @@ import unittest import sys +from Queue import Queue +from collections import defaultdict + import os sys.path.append(os.path.abspath(os.getcwd())) from plugins.repo.burp.plugin import BurpPlugin @@ -21,16 +24,16 @@ Note, Host, Service, -) + ModelBase) from plugins.modelactions import modelactions -import test_common -class BurpTest(unittest.TestCase): +class TestBurp: cd = os.path.dirname(os.path.realpath(__file__)) - def setUp(self): + + def test_Plugin_creates_adecuate_objects(self, monkeypatch): self.plugin = BurpPlugin() factory.register(Host) factory.register(Service) @@ -38,24 +41,27 @@ def setUp(self): factory.register(VulnWeb) factory.register(Note) factory.register(Credential) - - def test_Plugin_creates_adecuate_objects(self): + pending_actions = Queue() + self.plugin.set_actions_queue(pending_actions) + monkeypatch.setattr(ModelBase, 'getID', lambda _: 1) self.plugin.processReport(self.cd + '/burp_xml') - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDHOST) - self.assertEqual(action[1].name, "200.20.20.201") - # action = self.plugin._pending_actions.get(block=True) - # self.assertEqual(action[0], modelactions.ADDINTERFACE) - # self.assertEqual(action[2].name, "200.20.20.201") - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDSERVICEINT) - self.assertEqual(action[3].name, 'http') - self.assertEqual(action[3].protocol, 'tcp') - self.assertEqual(action[3].ports, [80]) - self.assertEqual(action[3].status, 'open') - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDNOTESRV) - # TODO: Fix broken test + actions = defaultdict(list) + while not pending_actions.empty(): + action = self.plugin._pending_actions.get(block=True) + actions[action[0]].append(action[1]) + + assert actions[2000][0].name == "200.20.20.201" + assert actions.keys() == [2000, 20008, 2027, 2037, 2039] + assert len(actions[20008]) == 14 + assert len(actions[2027]) == 14 + assert len(actions[2037]) == 14 + assert len(actions[2039]) == 14 + + assert all('http' == name for name in map(lambda service: service.name, actions[20008])) + assert all([80] == ports for ports in map(lambda service: service.ports, actions[20008])) + assert all('tcp' == protocol for protocol in map(lambda service: service.protocol, actions[20008])) + assert all('open' for status in map(lambda service: service.status, actions[20008])) + # self.assertEqual(action[3], 'Cleartext submission of password') if __name__ == '__main__': diff --git a/test_cases/plugins/test_nessus.py b/test_cases/plugins/test_nessus.py index ae3daad50fa..dc389cf5685 100644 --- a/test_cases/plugins/test_nessus.py +++ b/test_cases/plugins/test_nessus.py @@ -10,6 +10,9 @@ import unittest import sys +from Queue import Queue +from collections import defaultdict + import os sys.path.append(os.path.abspath(os.getcwd())) from plugins.repo.nessus.plugin import NessusPlugin @@ -21,15 +24,15 @@ Note, Host, Service, -) + ModelBase) from plugins.modelactions import modelactions import test_common -class NessusParserTest(unittest.TestCase): +class TestNessusParser: cd = os.path.dirname(os.path.realpath(__file__)) - def setUp(self): + def test_Plugin_Calls_createAndAddHost(self, monkeypatch): self.plugin = NessusPlugin() factory.register(Host) factory.register(Service) @@ -38,23 +41,28 @@ def setUp(self): factory.register(Note) factory.register(Credential) - def test_Plugin_Calls_createAndAddHost(self): + pending_actions = Queue() + self.plugin.set_actions_queue(pending_actions) + monkeypatch.setattr(ModelBase, 'getID', lambda _: 1) self.plugin.processReport(self.cd + '/nessus_xml') - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDHOST) - self.assertEqual(action[1].name, "12.233.108.201") - # action = self.plugin._pending_actions.get(block=True) - # self.assertEqual(action[0], modelactions.ADDINTERFACE) - # self.assertEqual(action[2].name, "12.233.108.201") - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDVULNHOST) - self.assertEqual(action[2].name, "Nessus Scan Information") - test_common.skip(self, 4) - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDSERVICEINT) - self.assertEqual(action[3].ports, [443]) - self.assertEqual(action[3].name, 'https?') - self.assertEqual(action[3].protocol, 'tcp') + actions = defaultdict(list) + while not pending_actions.empty(): + action = self.plugin._pending_actions.get(block=True) + actions[action[0]].append(action[1]) + + assert actions[2000][0].name == "12.233.108.201" + assert actions.keys() == [2017, 20008, 2027, 2000, 2037, 2039] + assert len(actions[20008]) == 1 + assert len(actions[2027]) == 1 + assert len(actions[2037]) == 1 + assert len(actions[2039]) == 1 + + assert actions[2037][0].name == "Nessus SYN scanner" + + assert actions[20008][0].ports == [443] + assert actions[20008][0].name == 'https?' + assert actions[20008][0].protocol == 'tcp' + if __name__ == '__main__': diff --git a/test_cases/plugins/test_nexpose_full.py b/test_cases/plugins/test_nexpose_full.py index 0c5435e0387..7b4c5aeade8 100644 --- a/test_cases/plugins/test_nexpose_full.py +++ b/test_cases/plugins/test_nexpose_full.py @@ -10,6 +10,9 @@ import unittest import sys +from Queue import Queue +from collections import defaultdict + import os sys.path.append(os.path.abspath(os.getcwd())) # module's path has a dash (-) in it, so we need to do this... @@ -24,14 +27,14 @@ Note, Host, Service, -) + ModelBase) from plugins.modelactions import modelactions -class NexposeTest(unittest.TestCase): +class TestNexpose: cd = os.path.dirname(os.path.realpath(__file__)) - def setUp(self): + def test_Plugin_creates_apropiate_objects(self, monkeypatch): self.plugin = NexposeFullPlugin() factory.register(Host) factory.register(Service) @@ -39,23 +42,25 @@ def setUp(self): factory.register(VulnWeb) factory.register(Note) factory.register(Credential) - - def test_Plugin_creates_apropiate_objects(self): + pending_actions = Queue() + self.plugin.set_actions_queue(pending_actions) + monkeypatch.setattr(ModelBase, 'getID', lambda _: 1) self.plugin.processReport(self.cd + '/nexpose_full_xml') - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDHOST) - self.assertEqual(action[1].name, "192.168.1.1") - # action = self.plugin._pending_actions.get(block=True) - # self.assertEqual(action[0], modelactions.ADDINTERFACE) - # self.assertEqual(action[2].name, "192.168.1.1") - for i in range(131): + + actions = defaultdict(list) + while not pending_actions.empty(): action = self.plugin._pending_actions.get(block=True) - if type(action[2]) in [Vuln, VulnWeb]: - assert action[0] == modelactions.ADDVULNHOST - elif type(action[-1]) == Service: - assert action[0] == modelactions.ADDSERVICEINT - action = self.plugin._pending_actions.get(block=True) - assert action[0] == modelactions.ADDVULNSRV + actions[action[0]].append(action[1]) + + assert actions[2000][0].name == "192.168.1.1" + assert actions.keys() == [2000, 2017, 2019, 2037, 20008] + + assert len(actions[2000]) == 8 + assert len(actions[20008]) == 20 + assert len(actions[2027]) == 0 + assert len(actions[2037]) == 403 + assert len(actions[2039]) == 0 + if __name__ == '__main__': diff --git a/test_cases/plugins/test_nmap.py b/test_cases/plugins/test_nmap.py index 4cc0dbcbe6f..e1d91d403f6 100644 --- a/test_cases/plugins/test_nmap.py +++ b/test_cases/plugins/test_nmap.py @@ -7,10 +7,11 @@ See the file 'doc/LICENSE' for the license information ''' - -import unittest -import sys import os +import sys +from Queue import Queue +from collections import defaultdict + sys.path.append(os.path.abspath(os.getcwd())) from plugins.repo.nmap.plugin import NmapPlugin from model.common import factory @@ -21,11 +22,10 @@ Note, Host, Service, -) -from plugins.modelactions import modelactions + ModelBase) -class NmapXMLParserTest(unittest.TestCase): +class TestNmapXMLParserTest: plugin = NmapPlugin() outputNmapBlog = ("Starting Nmap 7.12 ( https://nmap.org ) at 2016-05-16 14:56 ART\n" "Nmap scan report for joaquinlp.me (198.38.82.159)\n" @@ -53,29 +53,45 @@ class NmapXMLParserTest(unittest.TestCase): with open(cd + '/nmap_output_xml', 'r') as output: xml_output = output.read() - def setUp(self): + def register_factorties(self, monkeypatch): factory.register(Host) factory.register(Service) factory.register(Vuln) factory.register(VulnWeb) factory.register(Note) factory.register(Credential) + self.pending_actions = Queue() + self.plugin.set_actions_queue(self.pending_actions) + monkeypatch.setattr(ModelBase, 'getID', lambda _: 1) - def test_Plugin_Calls_createAndAddHost(self): - self.plugin.parseOutputString(self.xml_output) - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDHOST) - self.assertEqual(action[1].name, "198.38.82.159") + def test_Plugin_Calls_createAndAddHost(self, monkeypatch): + self.register_factorties(monkeypatch) - def test_Plugin_Calls_createAndAddService(self): self.plugin.parseOutputString(self.xml_output) - action = self.plugin._pending_actions.get(block=True) - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDSERVICEINT) - self.assertEqual(action[3].ports, [25]) - self.assertEqual(action[3].name, 'smtp') - self.assertEqual(action[3].protocol, 'tcp') + actions = defaultdict(list) + while not self.pending_actions.empty(): + action = self.plugin._pending_actions.get(block=True) + actions[action[0]].append(action[1]) + + assert actions[2000][0].name == "198.38.82.159" + assert actions.keys() == [2000, 20008] + + assert len(actions[2000]) == 1 + assert len(actions[20008]) == 13 + assert map(lambda service: service.name, actions[20008]) == [ + 'ftp', + 'smtp', + 'domain', + 'http', + 'pop3', + 'imap', + 'https', + 'smtps', + 'submission', + 'imaps', + 'pop3s', + 'ms-v-worlds', + 'mysql' + ] -if __name__ == '__main__': - unittest.main() diff --git a/test_cases/plugins/test_ping.py b/test_cases/plugins/test_ping.py index 7729884eeb3..bcd98458ff0 100644 --- a/test_cases/plugins/test_ping.py +++ b/test_cases/plugins/test_ping.py @@ -10,6 +10,9 @@ import unittest import sys +from Queue import Queue +from collections import defaultdict + import os sys.path.append(os.path.abspath(os.getcwd())) from plugins.repo.ping.plugin import CmdPingPlugin @@ -21,28 +24,38 @@ Note, Host, Service, -) + ModelBase) from plugins.modelactions import modelactions -class CmdPingPluginTest(unittest.TestCase): +class TestCmdPingPlugin: plugin = CmdPingPlugin() outputPingGoogle = ("PING google.com (216.58.222.142) 56(84) bytes of" "data.\n64 bytes from scl03s11-in-f14.1e100.net" "(216.58.222.142): icmp_seq=1 ttl=53 time=28.9 ms") - def setUp(self): + + def test_Plugin_Calls_createAndAddHost(self, monkeypatch): factory.register(Host) factory.register(Service) factory.register(Vuln) factory.register(VulnWeb) factory.register(Note) factory.register(Credential) - - def test_Plugin_Calls_createAndAddHost(self): + pending_actions = Queue() + self.plugin.set_actions_queue(pending_actions) + monkeypatch.setattr(ModelBase, 'getID', lambda _: 1) self.plugin.parseOutputString(self.outputPingGoogle) - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDHOST) - self.assertEqual(action[1].name, "216.58.222.142") + + actions = defaultdict(list) + while not pending_actions.empty(): + action = self.plugin._pending_actions.get(block=True) + actions[action[0]].append(action[1]) + + assert actions[2000][0].name == "216.58.222.142" + assert actions.keys() == [2000] + + assert len(actions[2000]) == 1 + if __name__ == '__main__': diff --git a/test_cases/plugins/test_telnet.py b/test_cases/plugins/test_telnet.py index 340c48b011e..0962517faad 100644 --- a/test_cases/plugins/test_telnet.py +++ b/test_cases/plugins/test_telnet.py @@ -7,10 +7,13 @@ See the file 'doc/LICENSE' for the license information ''' +from Queue import Queue +from collections import defaultdict -import unittest -import sys import os +import sys +import unittest + sys.path.append(os.path.abspath(os.getcwd())) from plugins.repo.telnet.plugin import TelnetRouterPlugin from model.common import factory @@ -21,11 +24,11 @@ Note, Host, Service, -) + ModelBase) from plugins.modelactions import modelactions -class CmdPingPluginTest(unittest.TestCase): +class TestCmdPingPlugin: plugin = TelnetRouterPlugin() outputTelnetLocalhost = ("Connection failed: Connection refused\n" "Trying ::1%1...\n" @@ -38,19 +41,30 @@ class CmdPingPluginTest(unittest.TestCase): "Date: Mon, 16 May 2016 17:42:18 GMT\n" "Content-Length: 0\n\n" "Connection closed by foreign host.\n") - def setUp(self): + + def test_Plugin_Calls_createAndAddHost(self, monkeypatch): factory.register(Host) factory.register(Service) factory.register(Vuln) factory.register(VulnWeb) factory.register(Note) factory.register(Credential) + pending_actions = Queue() + self.plugin.set_actions_queue(pending_actions) + monkeypatch.setattr(ModelBase, 'getID', lambda _: 1) - def test_Plugin_Calls_createAndAddHost(self): self.plugin.parseOutputString(self.outputTelnetLocalhost) - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDHOST) - self.assertEqual(action[1].name, "127.0.0.1") + + actions = defaultdict(list) + while not pending_actions.empty(): + action = self.plugin._pending_actions.get(block=True) + actions[action[0]].append(action[1]) + + assert actions[2000][0].name == "127.0.0.1" + assert actions.keys() == [2000, 20008] + + assert len(actions[2000]) == 1 + assert len(actions[20008]) == 1 if __name__ == '__main__': diff --git a/test_cases/plugins/test_whois.py b/test_cases/plugins/test_whois.py index f73eee3ba85..5759993ffd0 100644 --- a/test_cases/plugins/test_whois.py +++ b/test_cases/plugins/test_whois.py @@ -10,6 +10,9 @@ import unittest import sys +from Queue import Queue +from collections import defaultdict + import os sys.path.append(os.path.abspath(os.getcwd())) from plugins.repo.whois.plugin import CmdWhoisPlugin @@ -21,30 +24,35 @@ Note, Host, Service, -) + ModelBase) from plugins.modelactions import modelactions -class CmdPingPluginTest(unittest.TestCase): +class TestCmdPingPlugin: plugin = CmdWhoisPlugin() cd = os.path.dirname(os.path.realpath(__file__)) with open(cd + '/whois_output', 'r') as output: outputWhoisInfobyte = output.read() - def setUp(self): + + def test_Plugin_Calls_createAndAddHost(self, monkeypatch): factory.register(Host) factory.register(Service) factory.register(Vuln) factory.register(VulnWeb) factory.register(Note) factory.register(Credential) - - def test_Plugin_Calls_createAndAddHost(self): + pending_actions = Queue() + self.plugin.set_actions_queue(pending_actions) + monkeypatch.setattr(ModelBase, 'getID', lambda _: 1) self.plugin.parseOutputString(self.outputWhoisInfobyte) - action = self.plugin._pending_actions.get(block=True) - self.assertEqual(action[0], modelactions.ADDHOST) - self.assertEqual(action[1].name, "205.251.196.172") + actions = defaultdict(list) + while not pending_actions.empty(): + action = self.plugin._pending_actions.get(block=True) + actions[action[0]].append(action[1]) + + assert actions[2000][0].name == "205.251.196.172" + assert actions.keys() == [2000] -if __name__ == '__main__': - unittest.main() + assert len(actions[2000]) == 8 diff --git a/test_cases/test_managers_mapper_manager.py b/test_cases/test_managers_mapper_manager.py index a936b156614..9b419fc6c00 100644 --- a/test_cases/test_managers_mapper_manager.py +++ b/test_cases/test_managers_mapper_manager.py @@ -121,7 +121,7 @@ def test_save_without_command(self, monkeypatch, session): test_data['data']['parent_type'] = test_data['parent']['parent_type'] test_data['expected_payload']['parent'] = parent.id def mock_server_post(test_data, post_url, update=False, expected_response=201, **params): - assert post_url == 'http://localhost:5985/_api/v2/ws/test/{0}/'.format(test_data['api_end_point']) + assert post_url == 'http://localhost:5984/_api/v2/ws/test/{0}/'.format(test_data['api_end_point']) assert expected_response == 201 assert update == False metadata = params.pop('metadata') @@ -153,7 +153,7 @@ def test_save_with_command(self, monkeypatch, session): test_data['data']['parent_type'] = test_data['parent']['parent_type'] test_data['expected_payload']['parent'] = parent.id def mock_server_post(test_data, post_url, update=False, expected_response=201, **params): - assert post_url == 'http://localhost:5985/_api/v2/ws/test/{0}/?command_id={1}'.format(test_data['api_end_point'], params['command_id']) + assert post_url == 'http://localhost:5984/_api/v2/ws/test/{0}/?command_id={1}'.format(test_data['api_end_point'], params['command_id']) assert expected_response == 201 assert update == False metadata = params.pop('metadata') @@ -187,7 +187,7 @@ def test_update_without_command(self, monkeypatch, session): test_data['data']['parent_type'] = test_data['parent']['parent_type'] test_data['expected_payload']['parent'] = parent.id def mock_server_put(test_data, put_url, update=False, expected_response=201, **params): - assert put_url == 'http://localhost:5985/_api/v2/ws/test/{0}/{1}/'.format(test_data['api_end_point'], test_data['id']) + assert put_url == 'http://localhost:5984/_api/v2/ws/test/{0}/{1}/'.format(test_data['api_end_point'], test_data['id']) assert expected_response == 200 assert update == False metadata = params.pop('metadata') @@ -228,7 +228,7 @@ def test_update_with_command(self, monkeypatch, session): relational_model = test_data['factory'].create() session.commit() def mock_server_put(put_url, update=False, expected_response=201, **params): - assert put_url == 'http://localhost:5985/_api/v2/ws/test/{0}/{1}/?command_id={2}'.format(test_data['api_end_point'], test_data['id'], params['command_id']) + assert put_url == 'http://localhost:5984/_api/v2/ws/test/{0}/{1}/?command_id={2}'.format(test_data['api_end_point'], test_data['id'], params['command_id']) assert expected_response == 200 assert update == False diff --git a/test_cases/test_models.py b/test_cases/test_models.py index 8253c1720c4..7595bb1f5e4 100644 --- a/test_cases/test_models.py +++ b/test_cases/test_models.py @@ -73,18 +73,3 @@ def test_faraday_ready_objects_getter(self): self.assertTrue(all([isinstance(s, models.Service) for s in services])) self.assertTrue(all([isinstance(v, models.Vuln) for v in vulns])) self.assertTrue(all([isinstance(v, models.VulnWeb) for v in vulns_web])) - - def test_id_creation(self): - # ideally, the flatten dictionary should be provided and shouldnt depend upon - # our implementation. - # ideally. - classes = [models.Host, models.Service, models.Vuln, models.VulnWeb, models.Note] - dicts = [self.a_host_dictionary, self.a_service_dictionary, - self.a_vuln_dictionary, self.a_vuln_web_dictionary, self.a_note_dictionary] - dicts = map(models._flatten_dictionary, dicts) - for class_, dictionary in zip(classes, dicts): - expected_id = dictionary['id'] - parent_id = '.'.join(expected_id.split('.')[:-1]) - obj = class_(dictionary, self.ws) - obj.setID(parent_id) - self.assertEqual(expected_id, unicode(obj.id)) diff --git a/test_cases/test_plugins_controller.py b/test_cases/test_plugins_controller.py index cc9a8a3766f..01bca562cb2 100644 --- a/test_cases/test_plugins_controller.py +++ b/test_cases/test_plugins_controller.py @@ -1,9 +1,10 @@ import sys sys.path.append('.') import unittest -import plugins.controller +from Queue import Queue from mock import MagicMock as mock -from mock import patch + +import plugins.controller class PluginControllerUnitTest(unittest.TestCase): @@ -12,10 +13,10 @@ def setUp(self): def create_not_plugin(name, can_parse_command_string): plugin = mock() - plugin.canParseCommandString = mock(return_value = can_parse_command_string) + plugin.canParseCommandString = mock(return_value=can_parse_command_string) plugin.updateSettings = mock() plugin.name = name - plugin.processCommandString = mock(return_value = 'modified cmd string') + plugin.processCommandString = mock(return_value='modified cmd string') return plugin def create_not_plugin_manager(): @@ -24,20 +25,22 @@ def create_not_plugin_manager(): self.plugin2 = create_not_plugin('plugin2', False) self.plugin3 = create_not_plugin('plugin3', False) self.plugin4 = create_not_plugin('plugin4', False) - not_plugin_manager.getPlugins = mock(return_value = {'plugin1' : self.plugin1, - 'plugin2' : self.plugin2, - 'plugin3' : self.plugin3, - 'plugin4' : self.plugin4}) + not_plugin_manager.getPlugins = mock(return_value={'plugin1': self.plugin1, + 'plugin2': self.plugin2, + 'plugin3': self.plugin3, + 'plugin4': self.plugin4}) return not_plugin_manager def create_not_mappers_manager(): not_mappers_manager = mock() + self.pending_actions = Queue() self.not_plugin_manager = create_not_plugin_manager() self.not_mappers_manager = create_not_mappers_manager() self.controller = plugins.controller.PluginController('PluginController', self.not_plugin_manager, - self.not_mappers_manager) + self.not_mappers_manager, + self.pending_actions) def test_find_plugin_that_exists(self): plugin = self.controller._find_plugin('plugin1') @@ -60,18 +63,25 @@ def test_command_not_malformed(self): self.assertEqual(nice_command_blocked, False) def test_getting_plugins_by_input_that_can_parse_cmd(self): - plugin_set = {'1': self.plugin1, '2' : self.plugin2, '3' : self.plugin3, '4' : self.plugin4} + plugin_set = { + '1': self.plugin1, + '2': self.plugin2, + '3': self.plugin3, + '4': self.plugin4} should_be_plugin_1 = self.controller._get_plugins_by_input('ping', plugin_set) self.assertIs(should_be_plugin_1, self.plugin1) def test_return_none_when_cant_find_plugin_that_can_parse_cmd(self): - plugin_set = {'2' : self.plugin2, '3' : self.plugin3, '4' : self.plugin4} + plugin_set = { + '2': self.plugin2, + '3': self.plugin3, + '4': self.plugin4} should_be_none = self.controller._get_plugins_by_input('ping', plugin_set) self.assertIs(should_be_none, None) def test_update_plugin_settings(self): plugin_id = 'plugin1' - new_settings = {'setting1' : 'value1', 'setting2' : 'value2'} + new_settings = {'setting1': 'value1', 'setting2': 'value2'} self.controller.updatePluginSettings(plugin_id, new_settings) self.plugin1.updateSettings.assert_called_once_with(new_settings) diff --git a/test_cases/test_server_io.py b/test_cases/test_server_io.py index 6fedde59d9f..ee5aeb67a35 100644 --- a/test_cases/test_server_io.py +++ b/test_cases/test_server_io.py @@ -10,7 +10,7 @@ from mock import MagicMock, patch server.FARADAY_UP = False -server.SERVER_URL = "http://s:p" +server.SERVER_URL = "http://localhost:5984" example_url = "http://just_some_url" @@ -18,7 +18,7 @@ class ClientServerAPITests(unittest.TestCase): def setUp(self): self.ws_name = "a_ws" - self.server_api_url = "http://s:p/_api" + self.server_api_url = "http://localhost:5984/_api" def test_get_base_server_url(self): s = server._get_base_server_url() @@ -26,21 +26,20 @@ def test_get_base_server_url(self): def test_create_server_api_url(self): s = server._create_server_api_url() - self.assertEqual("{0}/_api".format(server.SERVER_URL), s) + self.assertEqual("{0}/_api/v2".format(server.SERVER_URL), s) def test_create_server_get_url(self): obj_name = "hosts" s = server._create_server_get_url(self.ws_name, obj_name) - self.assertEqual("{0}/_api/ws/{1}/{2}".format(server.SERVER_URL, self.ws_name, obj_name), s) + self.assertEqual("{0}/_api/v2/ws/{1}/{2}".format(server.SERVER_URL, self.ws_name, obj_name), s) def test_create_serve_post_url(self): - objid = "123456" - server_post_url = server._create_server_post_url(self.ws_name, objid) - self.assertEqual(self.server_api_url + '/ws/' + self.ws_name + '/doc/' + objid, server_post_url) + server_post_url = server._create_server_post_url(self.ws_name, 'Service', 1) + self.assertEqual(self.server_api_url + '/v2/ws/a_ws/services/?command_id=1', server_post_url) def test_create_server_get_ws_names_url(self): s = server._create_server_get_url(self.ws_name) - self.assertEqual("{0}/_api/ws/{1}".format(server.SERVER_URL, self.ws_name), s) + self.assertEqual("{0}/_api/v2/ws/{1}".format(server.SERVER_URL, self.ws_name), s) @responses.activate def test_raise_conflict_in_database(self): @@ -108,21 +107,6 @@ def test_put_with_no_update(self): responses.add(responses.PUT, example_url, body='{"ok": "true"}', status=200) self.assertEqual(server._put(example_url, expected_response=200), {"ok": "true"}) - @responses.activate - def test_put_with_update(self): - responses.add(responses.GET, example_url, body='{"_rev": "1-asf"}') - responses.add(responses.PUT, example_url, body='{"ok": "true"}', status=200) - server._put(example_url, update=True, expected_response=200) - self.assertIn("_rev", responses.calls[0].response.text) - - @responses.activate - def test_delete_object(self): - responses.add(responses.GET, example_url, body='{"_rev": "1-asf"}') - responses.add(responses.DELETE, example_url, body='{"ok": "true"}', status=200) - server._delete(example_url) - self.assertIn("_rev", responses.calls[0].response.text) - self.assertEqual(responses.calls[1].request.method, 'DELETE') - def test_faraday_dictionary_dispatcher_result(self): mock_raw_hosts = MagicMock() mock_raw_hosts.return_value = {'rows': [{'a': 'host', 'value': {'stuff': 'other_stuff'}}], 'total_rows': 4} From 244504cd9cc5fe099cf8b2237796908975834d30 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 15 Dec 2017 18:33:38 -0300 Subject: [PATCH 0664/1506] Add server check --- manage.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/manage.py b/manage.py index 4313250ddd3..5e2a6c3fbd0 100755 --- a/manage.py +++ b/manage.py @@ -1,8 +1,11 @@ #!/usr/bin/env python import click +import requests +from requests import ConnectionError from sqlalchemy.exc import OperationalError +from persistence.server.server import _conf, FARADAY_UP, SERVER_URL from server.importer import ImportCouchDB from server.commands.initdb import InitDB from server.commands.faraday_schema_display import DatabaseSchema @@ -18,17 +21,28 @@ def cli(): pass + +def check_faraday_server(url): + return requests.get(url) + + @click.command() @click.option('--debug/--no-debug', default=False) @click.option('--workspace', default=None) def process_reports(debug, workspace): setUpLogger(debug) + configuration = _conf() + url = '{0}/_api/v2/info'.format(configuration.getServerURI() if FARADAY_UP else SERVER_URL) with app.app_context(): try: + check_faraday_server(url) import_external_reports(workspace) except OperationalError as ex: print('{0}'.format(ex)) print('Please verify database is running or configuration on server.ini!') + except ConnectionError: + print('Can\'t connect to {0}. Please check if the server is running.'.format(url)) + @click.command() def reset_db(): From 04d1e940c9b9f8d1be2caa90f2a8e91fba286988 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 15 Dec 2017 18:35:07 -0300 Subject: [PATCH 0665/1506] Delete some old code. Fix some bugs when command id is passed in args --- model/controller.py | 184 +---------------------------------- persistence/server/models.py | 2 +- 2 files changed, 5 insertions(+), 181 deletions(-) diff --git a/model/controller.py b/model/controller.py index b8776acc79c..685f15a50a8 100644 --- a/model/controller.py +++ b/model/controller.py @@ -459,44 +459,6 @@ def __del(self, objId, *args): return True return False - def delHostASYNC(self, hostId): - """ - ASYNC API - Adds an action to the ModelController actions queue indicating a - particular host must be removed from the model - """ - self.__addPendingAction(modelactions.DELHOST, hostId) - - def delHostSYNC(self, hostId): - """ - SYNC API - Deletes a host from model - """ - self._processAction(modelactions.DELHOST, [hostId], sync=True) - - def editHostSYNC(self, host, name, description, os, owned): - """ - SYNC API - Modifies a host from model - """ - self._processAction(modelactions.EDITHOST, [ - host, name, description, os, owned], sync=True) - - def editServiceSYNC(self, service, name, description, protocol, ports, status, version, owned): - """ - SYNC API - Modifies a host from model - """ - self._processAction(modelactions.EDITSERVICE, [ - service, name, description, protocol, ports, status, version, owned], sync=True) - - def editServiceASYNC(self, service, name, description, protocol, ports, status, version, owned): - """ - ASYNC API - Modifies a service from model - """ - self.__addPendingAction(modelactions.EDITSERVICE, service, - name, description, protocol, ports, status, version, owned) def __editService(self, service, name=None, description=None, protocol=None, ports=None, status=None, @@ -515,158 +477,20 @@ def addPluginStart(self, name): def addPluginEnd(self, name): self.__addPendingAction(modelactions.PLUGINEND, name) - def _pluginStart(self, name): + def _pluginStart(self, name, command_id): self.active_plugins_count_lock.acquire() - getLogger(self).info("Plugin Started: " + name) + getLogger(self).info("Plugin Started: {0}. ".format(name, command_id)) self.active_plugins_count += 1 self.active_plugins_count_lock.release() return True - def _pluginEnd(self, name): + def _pluginEnd(self, name, command_id): self.active_plugins_count_lock.acquire() - getLogger(self).info("Plugin Ended: " + name) + getLogger(self).info("Plugin Ended: {0}".format(name)) self.active_plugins_count -= 1 self.active_plugins_count_lock.release() return True - def addVulnToHostASYNC(self, hostId, newVuln): - self.__addPendingAction(modelactions.ADDVULNHOST, newVuln, hostId) - - def addVulnToHostSYNC(self, hostId, newVuln): - self._processAction(modelactions.ADDVULNHOST, [ - newVuln, hostId], sync=True) - - def addVulnToServiceASYNC(self, host, srvId, newVuln): - self.__addPendingAction(modelactions.ADDVULNSRV, newVuln, srvId) - - def addVulnToServiceSYNC(self, host, srvId, newVuln): - self._processAction(modelactions.ADDVULNSRV, [ - newVuln, srvId], sync=True) - - def addVulnSYNC(self, modelObjectId, newVuln): - self._processAction(modelactions.ADDVULN, [ - newVuln, modelObjectId], sync=True) - - def addVulnWebToServiceASYNC(self, host, srvId, newVuln): - self.__addPendingAction(modelactions.ADDVULNWEBSRV, newVuln, srvId) - - def addVulnWebToServiceSYNC(self, host, srvId, newVuln): - self._processAction(modelactions.ADDVULNWEBSRV, - [newVuln, srvId], sync=True) - - def delVulnFromHostASYNC(self, hostId, vulnId): - self.__addPendingAction(modelactions.DELVULNHOST, vulnId) - - def delVulnFromHostSYNC(self, hostname, vulnId): - self._processAction(modelactions.DELVULNHOST, [vulnId], sync=True) - - def delVulnFromServiceASYNC(self, hostname, srvname, vulnId): - self.__addPendingAction(modelactions.DELVULNSRV, vulnId) - - def delVulnFromServiceSYNC(self, hostname, srvname, vulnId): - self._processAction(modelactions.DELVULNSRV, [vulnId], sync=True) - - def delVulnSYNC(self, model_object, vuln_id): - self._processAction(modelactions.DELVULN, [vuln_id], sync=True) - - def editVulnSYNC(self, vuln, name, desc, severity, resolution, refs): - self._processAction(modelactions.EDITVULN, [ - vuln, name, desc, severity, resolution, refs], sync=True) - - def editVulnASYNC(self, vuln, name, desc, severity, resolution, refs): - self.__addPendingAction(modelactions.EDITVULN, - vuln, name, desc, severity, resolution, refs) - - def editVulnWebSYNC(self, vuln, name, desc, website, path, refs, severity, resolution, - request, response, method, pname, params, query, - category): - self._processAction(modelactions.EDITVULN, - [vuln, name, desc, website, path, refs, severity, resolution, - request, response, method, pname, params, query, category], sync=True) - - def editVulnWebASYNC(self, vuln, name, desc, website, path, refs, - severity, resolution, request, response, method, pname, - params, query, category): - self.__addPendingAction(modelactions.EDITVULN, - vuln, name, desc, website, path, refs, - severity, resolution, request, response, method, - pname, params, query, category) - - def addNoteToHostASYNC(self, hostId, newNote): - self.__addPendingAction(modelactions.ADDNOTEHOST, newNote, hostId) - - def addNoteToHostSYNC(self, hostId, newNote): - self._processAction(modelactions.ADDNOTEHOST, [ - newNote, hostId], sync=True) - - def addNoteToServiceASYNC(self, host, srvId, newNote): - self.__addPendingAction(modelactions.ADDNOTESRV, newNote, srvId) - - def addNoteToNoteASYNC(self, host, srvname, note_id, newNote): - self.__addPendingAction(modelactions.ADDNOTENOTE, newNote, note_id) - - def addNoteToNoteSYNC(self, noteId, newNote): - self._processAction(modelactions.ADDNOTENOTE, [ - newNote, noteId], sync=True) - - def addNoteToServiceSYNC(self, host, srvId, newNote): - self._processAction(modelactions.ADDNOTESRV, [ - newNote, srvId], sync=True) - - def addNoteSYNC(self, model_object, newNote): - self._processAction(modelactions.ADDNOTE, [ - newNote, model_object], sync=True) - - def delNoteFromHostASYNC(self, hostId, noteId): - self.__addPendingAction(modelactions.DELNOTEHOST, noteId) - - def delNoteFromHostSYNC(self, hostname, noteId): - self._processAction(modelactions.DELNOTEHOST, [noteId], sync=True) - - def delNoteFromServiceASYNC(self, hostId, srvId, noteId): - self.__addPendingAction(modelactions.DELNOTESRV, noteId) - - def delNoteFromServiceSYNC(self, hostname, srvname, noteId): - self._processAction(modelactions.DELNOTESRV, [noteId], sync=True) - - def delNoteSYNC(self, model_object, note_id): - self._processAction(modelactions.DELNOTE, [note_id], sync=True) - - def addCredToServiceASYNC(self, host, srvId, newCred): - self.__addPendingAction(modelactions.ADDCREDSRV, newCred, srvId) - - def addCredToServiceSYNC(self, host, srvId, newCred): - self._processAction(modelactions.ADDCREDSRV, [ - newCred, srvId], sync=True) - - def delCredFromServiceASYNC(self, hostname, srvname, credId): - self.__addPendingAction(modelactions.DELCREDSRV, credId) - - def delCredFromServiceSYNC(self, hostname, srvname, credId): - self._processAction(modelactions.DELCREDSRV, [credId], sync=True) - - def editNoteSYNC(self, note, name, text): - self._processAction(modelactions.EDITNOTE, [ - note, name, text], sync=True) - - def editNoteASYNC(self, note, name, text): - self.__addPendingAction(modelactions.EDITNOTE, note, name, text) - - def editCredSYNC(self, cred, username, password): - self._processAction(modelactions.EDITCRED, [ - cred, username, password], sync=True) - - def editCredASYNC(self, cred, username, password): - self.__addPendingAction(modelactions.EDITCRED, - cred, username, password) - - def addCredSYNC(self, model_object_id, newCred): - self._processAction(modelactions.ADDCRED, [ - newCred, model_object_id], sync=True) - - def delCredSYNC(self, model_object, cred_id): - self._processAction(modelactions.DELCRED, [cred_id], sync=True) - def newHost(self, ip, os="Unknown"): return model.common.factory.createModelObject( models.Host.class_signature, ip, diff --git a/persistence/server/models.py b/persistence/server/models.py index 320e360c3c0..49a9d87b7e4 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -465,7 +465,7 @@ def create_command(workspace_name, command, command_id): @_ignore_in_changes -def update_command(workspace_name, command): +def update_command(workspace_name, command, command_id): """Take a workspace_name and a Command object and update it in the sever. Return the server's json response as a dictionary. """ From 08e31402788e1d8f9db46177529b651558ed9809 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 18 Dec 2017 14:41:33 -0300 Subject: [PATCH 0666/1506] Remove interface functions --- model/controller.py | 1 + model/guiapi.py | 12 ------------ persistence/server/server.py | 3 +-- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/model/controller.py b/model/controller.py index 685f15a50a8..44638e18994 100644 --- a/model/controller.py +++ b/model/controller.py @@ -491,6 +491,7 @@ def _pluginEnd(self, name, command_id): self.active_plugins_count_lock.release() return True + def newHost(self, ip, os="Unknown"): return model.common.factory.createModelObject( models.Host.class_signature, ip, diff --git a/model/guiapi.py b/model/guiapi.py index a7cb1875c71..93e29cf49b6 100644 --- a/model/guiapi.py +++ b/model/guiapi.py @@ -229,13 +229,6 @@ def addServiceToApplication(host_id, application_id, service): return True return False -def addServiceToInterface(host_id, interface_id, service): - if service is not None: - __model_controller.addServiceToInterfaceSYNC(host_id, interface_id, service) - return True - return False - - def addVulnToHost(host_id, vuln): if vuln is not None: @@ -243,11 +236,6 @@ def addVulnToHost(host_id, vuln): return True return False -def addVulnToInterface(host_id, interface_id, vuln): - if vuln is not None: - __model_controller.addVulnToInterfaceSYNC(host_id, interface_id, vuln) - return True - return False def addVulnToApplication(host_id, application_id, vuln): if vuln is not None: diff --git a/persistence/server/server.py b/persistence/server/server.py index cb863afc3a2..ee3815991f0 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -41,8 +41,7 @@ CantCommunicateWithServerError, ConflictInDatabase, ResourceDoesNotExist, - Unauthorized, - MoreThanOneObjectFoundByID) + Unauthorized) from persistence.server.changes_stream import CouchChangesStream From dd31599f0ef5a43445ab656500891b0830ccb194 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 18 Dec 2017 14:42:21 -0300 Subject: [PATCH 0667/1506] Fix credential create and update --- persistence/server/models.py | 14 +++---- persistence/server/server.py | 11 ++++- persistence/server/utils.py | 8 +++- test_cases/test_managers_mapper_manager.py | 47 +++++++++++++++++++--- 4 files changed, 63 insertions(+), 17 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index 49a9d87b7e4..5df20daf830 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -437,16 +437,16 @@ def create_credential(workspace_name, credential, command_id): Return the server's json response as a dictionary. """ credential_properties = get_credential_properties(credential) - return server.create_credential(workspace_name, **credential_properties) + return server.create_credential(workspace_name, command_id, **credential_properties) @_ignore_in_changes -def update_credential(workspace_name, credential): +def update_credential(workspace_name, credential, command_id): """Take a workspace_name and a Credential object and update it in the sever. Return the server's json response as a dictionary. """ credential_properties = get_credential_properties(credential) - return server.update_credential(workspace_name, credential.getID(), **credential_properties) + return server.update_credential(workspace_name, command_id, credential.getID(), **credential_properties) @_ignore_in_changes @@ -713,6 +713,10 @@ def __init__(self, obj, workspace_name): self.id_available = Event() if self.id is not None: self.id_available.set() + self.parent_type = obj.get('parent_type', None) + + def getParentType(self): + return self.parent_type def getParent(self): return self.parent_id @@ -942,10 +946,6 @@ def __init__(self, vuln, workspace_name): self.resolution = vuln.get('resolution') self.status = vuln.get('status', "opened") self.policyviolations = vuln.get('policyviolations', list()) - self.parent_type = vuln.get('parent_type') - - def getParentType(self): - return self.parent_type @staticmethod def publicattrsrefs(): diff --git a/persistence/server/server.py b/persistence/server/server.py index ee3815991f0..6c0b131bd63 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -58,6 +58,7 @@ 'VulnerabilityWeb': 'vulns', 'Service': 'services', 'Note': 'comment', + 'Cred': 'credential', } @@ -1271,7 +1272,8 @@ def update_note(workspace_name, command_id, id, name, text, owned=None, owner="" def create_credential(workspace_name, command_id, name, username, password, - owned=None, owner="", description="", metadata=None): + parent, parent_type, owned=None, owner="", + description="", metadata=None): """Creates a credential. Args: @@ -1291,6 +1293,8 @@ def create_credential(workspace_name, command_id, name, username, password, """ return _save_to_server(workspace_name, command_id=command_id, + parent=parent, + parent_type=parent_type, name=name, description=description, owned=owned, @@ -1301,7 +1305,8 @@ def create_credential(workspace_name, command_id, name, username, password, type="Cred") def update_credential(workspace_name, command_id, id, name, username, password, - owned=None, owner="", description="", metadata=None): + parent, parent_type, owned=None, owner="", + description="", metadata=None): """Updates a credential. Args: @@ -1321,6 +1326,8 @@ def update_credential(workspace_name, command_id, id, name, username, password, """ return _update_in_server(workspace_name, id, + parent=parent, + parent_type=parent_type, command_id=command_id, name=name, description=description, diff --git a/persistence/server/utils.py b/persistence/server/utils.py index 9cc3220a142..2a54a8e0748 100644 --- a/persistence/server/utils.py +++ b/persistence/server/utils.py @@ -110,8 +110,12 @@ def get_note_properties(note): def get_credential_properties(credential): - cred_dict = {'username': credential.getUsername(), - 'password': credential.getPassword()} + cred_dict = { + 'username': credential.getUsername(), + 'password': credential.getPassword(), + 'parent': credential.getParent(), + 'parent_type': credential.getParentType(), + } cred_dict.update(get_object_properties(credential)) return cred_dict diff --git a/test_cases/test_managers_mapper_manager.py b/test_cases/test_managers_mapper_manager.py index 9b419fc6c00..caddabc3899 100644 --- a/test_cases/test_managers_mapper_manager.py +++ b/test_cases/test_managers_mapper_manager.py @@ -1,9 +1,9 @@ from functools import partial from managers.mapper_manager import MapperManager -from persistence.server.models import Host, Service, Vuln +from persistence.server.models import Host, Service, Vuln, Credential import persistence.server.server from test_cases.factories import WorkspaceFactory, CommandFactory, HostFactory, \ - ServiceFactory, VulnerabilityFactory + ServiceFactory, VulnerabilityFactory, CredentialFactory # OBJ_DATA is like a fixture. # We use it to test all model classes. @@ -100,6 +100,33 @@ 'status': 'opened', 'resolution': None, }, + }], + Credential: [{ + 'factory': CredentialFactory, + 'api_end_point': 'credential', + 'parent': { + 'parent_type': 'Host', + 'parent_factory': HostFactory + }, + 'data': { + '_id': 1, + 'name': 'New credential', + 'description': 'Test credential', + 'owned': False, + 'owner': 'leo', + 'password': 'testpass', + 'username': 'username1' + }, + 'expected_payload': { + 'command_id': None, + 'name': 'New credential', + 'description': 'Test credential', + 'owner': 'leo', + 'owned': False, + 'password': 'testpass', + 'username': 'username1', + 'type': 'Cred', + }, }] } @@ -120,8 +147,10 @@ def test_save_without_command(self, monkeypatch, session): test_data['data']['parent'] = parent.id test_data['data']['parent_type'] = test_data['parent']['parent_type'] test_data['expected_payload']['parent'] = parent.id + if obj_class in [Vuln, Credential]: + test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] def mock_server_post(test_data, post_url, update=False, expected_response=201, **params): - assert post_url == 'http://localhost:5984/_api/v2/ws/test/{0}/'.format(test_data['api_end_point']) + assert post_url == 'http://localhost:5985/_api/v2/ws/test/{0}/'.format(test_data['api_end_point']) assert expected_response == 201 assert update == False metadata = params.pop('metadata') @@ -152,8 +181,10 @@ def test_save_with_command(self, monkeypatch, session): test_data['data']['parent'] = parent.id test_data['data']['parent_type'] = test_data['parent']['parent_type'] test_data['expected_payload']['parent'] = parent.id + if obj_class in [Vuln, Credential]: + test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] def mock_server_post(test_data, post_url, update=False, expected_response=201, **params): - assert post_url == 'http://localhost:5984/_api/v2/ws/test/{0}/?command_id={1}'.format(test_data['api_end_point'], params['command_id']) + assert post_url == 'http://localhost:5985/_api/v2/ws/test/{0}/?command_id={1}'.format(test_data['api_end_point'], params['command_id']) assert expected_response == 201 assert update == False metadata = params.pop('metadata') @@ -186,8 +217,10 @@ def test_update_without_command(self, monkeypatch, session): test_data['data']['parent'] = parent.id test_data['data']['parent_type'] = test_data['parent']['parent_type'] test_data['expected_payload']['parent'] = parent.id + if obj_class in [Vuln, Credential]: + test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] def mock_server_put(test_data, put_url, update=False, expected_response=201, **params): - assert put_url == 'http://localhost:5984/_api/v2/ws/test/{0}/{1}/'.format(test_data['api_end_point'], test_data['id']) + assert put_url == 'http://localhost:5985/_api/v2/ws/test/{0}/{1}/'.format(test_data['api_end_point'], test_data['id']) assert expected_response == 200 assert update == False metadata = params.pop('metadata') @@ -225,10 +258,12 @@ def test_update_with_command(self, monkeypatch, session): test_data['data']['parent'] = parent.id test_data['data']['parent_type'] = test_data['parent']['parent_type'] test_data['expected_payload']['parent'] = parent.id + if obj_class in [Vuln, Credential]: + test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] relational_model = test_data['factory'].create() session.commit() def mock_server_put(put_url, update=False, expected_response=201, **params): - assert put_url == 'http://localhost:5984/_api/v2/ws/test/{0}/{1}/?command_id={2}'.format(test_data['api_end_point'], test_data['id'], params['command_id']) + assert put_url == 'http://localhost:5985/_api/v2/ws/test/{0}/{1}/?command_id={2}'.format(test_data['api_end_point'], test_data['id'], params['command_id']) assert expected_response == 200 assert update == False From 6b6d8d4f3fa61d33f11bb2d1b148c720bd94846a Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 18 Dec 2017 19:04:00 -0300 Subject: [PATCH 0668/1506] Add more tests and use parametrize. Fix find on mapper for hosts --- managers/mapper_manager.py | 5 + persistence/server/models.py | 8 +- persistence/server/server.py | 9 +- persistence/server/utils.py | 13 +- plugins/manager.py | 1 + plugins/plugin.py | 1 + test_cases/test_managers_mapper_manager.py | 306 +++++++++++++-------- test_cases/test_model_controller.py | 0 8 files changed, 216 insertions(+), 127 deletions(-) create mode 100644 test_cases/test_model_controller.py diff --git a/managers/mapper_manager.py b/managers/mapper_manager.py index b97e2c85302..274ca24379b 100644 --- a/managers/mapper_manager.py +++ b/managers/mapper_manager.py @@ -3,6 +3,8 @@ Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) See the file 'doc/LICENSE' for the license information ''' +import logging + from persistence.server.models import create_object, get_object, update_object, delete_object # NOTE: This class is intended to be instantiated by the @@ -10,6 +12,7 @@ # IMPORTANT: There should be only one instance of this # class, since it creates the datamappers and those should # be unique too (they have identity maps for every model object) +logger = logging.getLogger(__name__) class MapperManager(object): @@ -33,6 +36,8 @@ def update(self, obj, command_id=None): return False def find(self, class_signature, obj_id): + if self.workspace_name is None: + logger.warn('No workspace detected. please call createMappers first.') return get_object(self.workspace_name, class_signature, obj_id) def remove(self, obj_id, class_signature): diff --git a/persistence/server/models.py b/persistence/server/models.py index 5df20daf830..c218476f637 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -85,8 +85,8 @@ def _flatten_dictionary(dictionary): flattened_dict[u'_id'] = dictionary['_id'] if dictionary.get('id'): flattened_dict[u'id'] = dictionary['id'] - for k, v in dictionary.get('value', {}).items(): - if k != '_id': # this is the couch id, which we have saved on 'id' + for k, v in dictionary.get('value', dictionary).items(): + if k != '_id': flattened_dict[k] = v return flattened_dict @@ -194,7 +194,7 @@ def get_hosts(workspace_name, **params): def get_host(workspace_name, host_id): """Return the host by host_id. None if it can't be found.""" - return force_unique(get_hosts(workspace_name, couchid=host_id)) + return get_hosts(workspace_name, id=host_id).pop() def get_all_vulns(workspace_name, **params): @@ -766,7 +766,7 @@ def tieBreak(self, key, prop1, prop2): """ return None - def addUpdate(self, newModelObject): + def addUpdate(self, newModelObject, command_id): conflict = False diff = ModelObjectDiff(self, newModelObject) for k, v in diff.getPropertiesDiff().items(): diff --git a/persistence/server/server.py b/persistence/server/server.py index 6c0b131bd63..d3b48f5c5f2 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -25,7 +25,7 @@ import os import json -import logging + try: import urlparse from urllib import urlencode @@ -254,7 +254,7 @@ def _delete(delete_url, database=False): def _get_raw_hosts(workspace_name, **params): """Take a workspace_name and an arbitrary number of params and return a dictionary with the hosts table.""" - request_url = _create_server_get_url(workspace_name, 'hosts') + request_url = _create_server_get_url(workspace_name, 'hosts', params.get('id', None)) return _get(request_url, **params) @@ -362,8 +362,9 @@ def _get_faraday_ready_dictionaries(workspace_name, faraday_object_name, appropiate_function = object_to_func[faraday_object_name] appropiate_dictionary = appropiate_function(workspace_name, **params) - faraday_ready_dictionaries = [] - if appropiate_dictionary: + faraday_ready_dictionaries = [appropiate_dictionary] + if faraday_object_row_name in appropiate_dictionary: + faraday_ready_dictionaries = [] for raw_dictionary in appropiate_dictionary[faraday_object_row_name]: if not full_table: faraday_ready_dictionaries.append(raw_dictionary['value']) diff --git a/persistence/server/utils.py b/persistence/server/utils.py index 2a54a8e0748..7957d221542 100644 --- a/persistence/server/utils.py +++ b/persistence/server/utils.py @@ -6,7 +6,12 @@ See the file 'doc/LICENSE' for the license information ''' -import hashlib +import re +import logging + +logger = logging.getLogger(__name__) + + from persistence.server.server_io_exceptions import MoreThanOneObjectFoundByID @@ -48,8 +53,10 @@ def get_host_properties(host): host_dict.update(get_object_properties(host)) # name was removed from host and changed to ip ip = host_dict.pop('name') - # regex ip y lanzar warning si no es ip. - host_dict['ip'] = ip + if 'ip' not in host_dict and ip: + if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip) is None: + logger.warn('Host with invalid IP address detected.') + host_dict['ip'] = ip return host_dict diff --git a/plugins/manager.py b/plugins/manager.py index 65b64d3d818..612560f985e 100644 --- a/plugins/manager.py +++ b/plugins/manager.py @@ -100,6 +100,7 @@ def _loadPlugins(self, plugin_repo_path): module_path = os.path.join(plugin_repo_path, name) sys.path.append(module_path) module_filename = os.path.join(module_path, "plugin.py") + getLogger(self).debug('Loading plugin {0}'.format(name)) self._plugin_modules[name] = imp.load_source( name, module_filename) except Exception as e: diff --git a/plugins/plugin.py b/plugins/plugin.py index e03b3a23045..c8016a89b97 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -33,6 +33,7 @@ from config.configuration import getInstanceConfiguration CONF = getInstanceConfiguration() VERSION = server.config.__get_version() +logger = logging.getLogger(__name__) class PluginBase(object): diff --git a/test_cases/test_managers_mapper_manager.py b/test_cases/test_managers_mapper_manager.py index caddabc3899..f0324274e10 100644 --- a/test_cases/test_managers_mapper_manager.py +++ b/test_cases/test_managers_mapper_manager.py @@ -1,19 +1,28 @@ from functools import partial + +import mock +import pytest + from managers.mapper_manager import MapperManager from persistence.server.models import Host, Service, Vuln, Credential import persistence.server.server +from persistence.server.utils import get_host_properties from test_cases.factories import WorkspaceFactory, CommandFactory, HostFactory, \ ServiceFactory, VulnerabilityFactory, CredentialFactory -# OBJ_DATA is like a fixture. +# OBJ_DATA is used to parametrize tests (https://docs.pytest.org/en/latest/parametrize.html) # We use it to test all model classes. # to add more tests you need to add items in the list or more objects in the dict. OBJ_DATA = { + # the key is the object being tested Host: [{ 'factory': HostFactory, + # api_end_point is used to assert the generated url. 'api_end_point': 'hosts', + # parent is used to assert parent information is correcly generated. 'parent': {}, + # data is used to instanciate a persistence.server.models class. 'data': { '_id': 1, 'name': '192.168.0.20', @@ -23,6 +32,7 @@ 'owned': False, 'owner': 'leo' }, + # expected_payload is asserted with the generated payload that will be sent to the API of faraday-server 'expected_payload': { 'command_id': None, 'default_gateway': '192.168.0.1', @@ -131,150 +141,214 @@ } +# the following dict is used to parametrize find (GET) tests +GET_OBJ_DATA = { + Host: [ + { + 'factory': HostFactory, + 'api_end_point': 'hosts', + 'get_properties_function': get_host_properties, + 'mocked_response': { + 'name': "192.168.1.1", 'default_gateway': None, + 'ip': "192.168.1.1", '_rev': "", + 'description': "Test description", 'owned': False, + 'services': 7, 'hostnames': [], + 'vulns': 45, 'owner': "leonardo", + 'credentials': 1, '_id': 16, + 'os': "Linux 2.6.9", 'id': 16, + 'metadata': { + 'update_time': 1513381792000, 'update_user': "", + 'update_action': 0, 'creator': "", + 'create_time': 1513381792000, 'update_controller_action': "", + 'owner': "leonardo", 'command_id': None + } + }, + 'serialized_expected_results': { + 'description': 'Test description', + 'ip': '192.168.1.1', + 'os': 'Linux 2.6.9', + 'owned': False, + 'owner': 'leonardo'} + + } + ] +} + +class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + + def json(self): + return self.json_data + + +@pytest.mark.usefixtures('logged_user') class TestMapperManager(): - def test_save_without_command(self, monkeypatch, session): + @pytest.mark.parametrize("obj_class, many_test_data", OBJ_DATA.items()) + def test_save_without_command(self, obj_class, many_test_data, monkeypatch, session): workspace = WorkspaceFactory.create(name='test') session.commit() mapper_manager = MapperManager() mapper_manager.createMappers(workspace.name) - for obj_class, many_test_data in OBJ_DATA.items(): - for test_data in many_test_data: - raw_data = test_data['data'] - if test_data['parent']: - parent = test_data['parent']['parent_factory'].create() - session.commit() - test_data['data']['parent'] = parent.id - test_data['data']['parent_type'] = test_data['parent']['parent_type'] - test_data['expected_payload']['parent'] = parent.id - if obj_class in [Vuln, Credential]: - test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] - def mock_server_post(test_data, post_url, update=False, expected_response=201, **params): - assert post_url == 'http://localhost:5985/_api/v2/ws/test/{0}/'.format(test_data['api_end_point']) - assert expected_response == 201 - assert update == False - metadata = params.pop('metadata') - assert metadata['owner'] == test_data['expected_payload']['owner'] - assert params == test_data['expected_payload'] - return { - 'id': 1, - 'ok': True, - 'rev': '' - } - monkeypatch.setattr(persistence.server.server, '_post', partial(mock_server_post, test_data)) - obj = obj_class(raw_data, workspace.name) - mapper_manager.save(obj) + for test_data in many_test_data: + raw_data = test_data['data'] + if test_data['parent']: + parent = test_data['parent']['parent_factory'].create() + session.commit() + test_data['data']['parent'] = parent.id + test_data['data']['parent_type'] = test_data['parent']['parent_type'] + test_data['expected_payload']['parent'] = parent.id + if obj_class in [Vuln, Credential]: + test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] + def mock_server_post(test_data, post_url, update=False, expected_response=201, **params): + assert post_url == 'http://localhost:5985/_api/v2/ws/test/{0}/'.format(test_data['api_end_point']) + assert expected_response == 201 + assert update == False + metadata = params.pop('metadata') + assert metadata['owner'] == test_data['expected_payload']['owner'] + assert params == test_data['expected_payload'] + return { + 'id': 1, + 'ok': True, + 'rev': '' + } + + monkeypatch.setattr(persistence.server.server, '_post', partial(mock_server_post, test_data)) + obj = obj_class(raw_data, workspace.name) + mapper_manager.save(obj) - def test_save_with_command(self, monkeypatch, session): + @pytest.mark.parametrize("obj_class, many_test_data", OBJ_DATA.items()) + def test_save_with_command(self, obj_class, many_test_data, monkeypatch, session): workspace = WorkspaceFactory.create(name='test') command = CommandFactory.create(workspace=workspace) session.commit() mapper_manager = MapperManager() mapper_manager.createMappers(workspace.name) - for obj_class, many_test_data in OBJ_DATA.items(): - for test_data in many_test_data: - raw_data = test_data['data'] - if test_data['parent']: - parent = test_data['parent']['parent_factory'].create() - session.commit() - test_data['data']['parent'] = parent.id - test_data['data']['parent_type'] = test_data['parent']['parent_type'] - test_data['expected_payload']['parent'] = parent.id - if obj_class in [Vuln, Credential]: - test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] - def mock_server_post(test_data, post_url, update=False, expected_response=201, **params): - assert post_url == 'http://localhost:5985/_api/v2/ws/test/{0}/?command_id={1}'.format(test_data['api_end_point'], params['command_id']) - assert expected_response == 201 - assert update == False - metadata = params.pop('metadata') - assert metadata['owner'] == test_data['expected_payload']['owner'] - params.pop('command_id') - test_data['expected_payload'].pop('command_id') - assert params == test_data['expected_payload'] - return { - 'id': 1, - 'ok': True, - 'rev': '' - } + for test_data in many_test_data: + raw_data = test_data['data'] + if test_data['parent']: + parent = test_data['parent']['parent_factory'].create() + session.commit() + test_data['data']['parent'] = parent.id + test_data['data']['parent_type'] = test_data['parent']['parent_type'] + test_data['expected_payload']['parent'] = parent.id + if obj_class in [Vuln, Credential]: + test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] + def mock_server_post(test_data, post_url, update=False, expected_response=201, **params): + assert post_url == 'http://localhost:5985/_api/v2/ws/test/{0}/?command_id={1}'.format(test_data['api_end_point'], params['command_id']) + assert expected_response == 201 + assert update == False + metadata = params.pop('metadata') + assert metadata['owner'] == test_data['expected_payload']['owner'] + params.pop('command_id') + test_data['expected_payload'].pop('command_id') + assert params == test_data['expected_payload'] + return { + 'id': 1, + 'ok': True, + 'rev': '' + } - monkeypatch.setattr(persistence.server.server, '_post', partial(mock_server_post, test_data)) - obj = obj_class(raw_data, workspace.name) - mapper_manager.save(obj, command.id) + monkeypatch.setattr(persistence.server.server, '_post', partial(mock_server_post, test_data)) + obj = obj_class(raw_data, workspace.name) + mapper_manager.save(obj, command.id) - def test_update_without_command(self, monkeypatch, session): + @pytest.mark.parametrize("obj_class, many_test_data", OBJ_DATA.items()) + def test_update_without_command(self, obj_class, many_test_data, monkeypatch, session): workspace = WorkspaceFactory.create(name='test') mapper_manager = MapperManager() mapper_manager.createMappers(workspace.name) - for obj_class, many_test_data in OBJ_DATA.items(): - for test_data in many_test_data: - relational_model = test_data['factory'].create() + + for test_data in many_test_data: + relational_model = test_data['factory'].create() + session.commit() + raw_data = test_data['data'] + if test_data['parent']: + parent = test_data['parent']['parent_factory'].create() session.commit() - raw_data = test_data['data'] - if test_data['parent']: - parent = test_data['parent']['parent_factory'].create() - session.commit() - test_data['data']['parent'] = parent.id - test_data['data']['parent_type'] = test_data['parent']['parent_type'] - test_data['expected_payload']['parent'] = parent.id - if obj_class in [Vuln, Credential]: - test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] - def mock_server_put(test_data, put_url, update=False, expected_response=201, **params): - assert put_url == 'http://localhost:5985/_api/v2/ws/test/{0}/{1}/'.format(test_data['api_end_point'], test_data['id']) - assert expected_response == 200 - assert update == False - metadata = params.pop('metadata') - assert metadata['owner'] == test_data['expected_payload']['owner'] - params.pop('command_id') - test_data['expected_payload'].pop('command_id', None) - assert params == test_data['expected_payload'] + test_data['data']['parent'] = parent.id + test_data['data']['parent_type'] = test_data['parent']['parent_type'] + test_data['expected_payload']['parent'] = parent.id + if obj_class in [Vuln, Credential]: + test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] + def mock_server_put(test_data, put_url, update=False, expected_response=201, **params): + assert put_url == 'http://localhost:5985/_api/v2/ws/test/{0}/{1}/'.format(test_data['api_end_point'], test_data['id']) + assert expected_response == 200 + assert update == False + metadata = params.pop('metadata') + assert metadata['owner'] == test_data['expected_payload']['owner'] + params.pop('command_id') + test_data['expected_payload'].pop('command_id', None) + assert params == test_data['expected_payload'] - return { - 'id': 1, - 'ok': True, - 'rev': '' - } + return { + 'id': 1, + 'ok': True, + 'rev': '' + } - raw_data['id'] = relational_model.id - test_data['id'] = relational_model.id - monkeypatch.setattr(persistence.server.server, '_put', partial(mock_server_put, test_data)) + raw_data['id'] = relational_model.id + test_data['id'] = relational_model.id + monkeypatch.setattr(persistence.server.server, '_put', partial(mock_server_put, test_data)) - obj = obj_class(raw_data, workspace.name) - mapper_manager.update(obj) + obj = obj_class(raw_data, workspace.name) + mapper_manager.update(obj) - def test_update_with_command(self, monkeypatch, session): + @pytest.mark.parametrize("obj_class, many_test_data", OBJ_DATA.items()) + def test_update_with_command(self, obj_class, many_test_data, monkeypatch, session): session.commit() workspace = WorkspaceFactory.create(name='test') command = CommandFactory.create(workspace=workspace) session.commit() mapper_manager = MapperManager() mapper_manager.createMappers(workspace.name) - for obj_class, many_test_data in OBJ_DATA.items(): - for test_data in many_test_data: - raw_data = test_data['data'] - if test_data['parent']: - parent = test_data['parent']['parent_factory'].create() - session.commit() - test_data['data']['parent'] = parent.id - test_data['data']['parent_type'] = test_data['parent']['parent_type'] - test_data['expected_payload']['parent'] = parent.id - if obj_class in [Vuln, Credential]: - test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] - relational_model = test_data['factory'].create() + + for test_data in many_test_data: + raw_data = test_data['data'] + if test_data['parent']: + parent = test_data['parent']['parent_factory'].create() session.commit() - def mock_server_put(put_url, update=False, expected_response=201, **params): - assert put_url == 'http://localhost:5985/_api/v2/ws/test/{0}/{1}/?command_id={2}'.format(test_data['api_end_point'], test_data['id'], params['command_id']) - assert expected_response == 200 - assert update == False + test_data['data']['parent'] = parent.id + test_data['data']['parent_type'] = test_data['parent']['parent_type'] + test_data['expected_payload']['parent'] = parent.id + if obj_class in [Vuln, Credential]: + test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] + relational_model = test_data['factory'].create() + session.commit() + def mock_server_put(put_url, update=False, expected_response=201, **params): + assert put_url == 'http://localhost:5985/_api/v2/ws/test/{0}/{1}/?command_id={2}'.format(test_data['api_end_point'], test_data['id'], params['command_id']) + assert expected_response == 200 + assert update == False + return { + 'id': 1, + 'ok': True, + 'rev': '' + } - return { - 'id': 1, - 'ok': True, - 'rev': '' - } + raw_data['id'] = relational_model.id + test_data['id'] = relational_model.id + monkeypatch.setattr(persistence.server.server, '_put', mock_server_put) + obj = obj_class(raw_data, workspace.name) + mapper_manager.update(obj, command.id) + + @pytest.mark.parametrize("obj_class, many_test_data", GET_OBJ_DATA.items()) + def test_find_obj_by_id(self, obj_class, many_test_data, session, monkeypatch): + for test_data in many_test_data: + persisted_obj = test_data['factory'].create() + session.commit() + mapper_manager = MapperManager() + mapper_manager.createMappers(persisted_obj.workspace.name) + + def mock_unsafe_io_with_server(host, test_data, server_io_function, server_expected_response, server_url, **payload): + mocked_response = test_data['mocked_response'] + assert 'http://localhost:5985/_api/v2/ws/{0}/{1}/{2}/'.format(persisted_obj.workspace.name, test_data['api_end_point'], persisted_obj.id) == server_url + return MockResponse(mocked_response, 200) - raw_data['id'] = relational_model.id - test_data['id'] = relational_model.id - monkeypatch.setattr(persistence.server.server, '_put', mock_server_put) - obj = obj_class(raw_data, workspace.name) - mapper_manager.update(obj, command.id) \ No newline at end of file + monkeypatch.setattr(persistence.server.server, '_unsafe_io_with_server', partial(mock_unsafe_io_with_server, persisted_obj, test_data)) + found_obj = mapper_manager.find(obj_class.class_signature, persisted_obj.id) + serialized_obj = test_data['get_properties_function'](found_obj) + metadata = serialized_obj.pop('metadata') + assert serialized_obj == test_data['serialized_expected_results'] diff --git a/test_cases/test_model_controller.py b/test_cases/test_model_controller.py new file mode 100644 index 00000000000..e69de29bb2d From 51f3b1178aa745de1afc7e268595aa35a96189f6 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 19 Dec 2017 15:02:02 -0300 Subject: [PATCH 0669/1506] Update initdb command to use already existing user and hostname configuration --- server/commands/initdb.py | 65 +++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 16 deletions(-) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index db23a535d80..89fcf9e8261 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -61,24 +61,59 @@ def run(self): # we use psql_log_filename for historical saving. we will ask faraday users this file. # current_psql_output is for checking psql command already known errors for each execution. psql_log_filename = os.path.join(faraday_path_conf, 'logs', 'psql_log.log') + configure_existing_user = None + psql_output = '' with open(psql_log_filename, 'a+') as psql_log_file: - username, password, process_status = self._configure_postgres(current_psql_output) - current_psql_output.seek(0) - psql_output = current_psql_output.read() - psql_log_file.write(psql_output) - current_psql_output.seek(0) - psql_output = current_psql_output.read() - self._check_psql_output(psql_output, process_status) - database_name, process_status = self._create_database(username, current_psql_output) - self._check_psql_output(psql_output, process_status) + while configure_existing_user is None: + configure_existing_user = raw_input('Do you {blue} already have {white} a postgresql username and password? (yes/no): '.format(blue=Fore.BLUE, white=Fore.WHITE)) + if configure_existing_user.lower() == 'yes': + configure_existing_user = True + elif configure_existing_user.lower() == 'no': + configure_existing_user = False + else: + print('Invalid option. Please type "yes" or "no" (ctrl-c to cancel): ') + configure_existing = None + hostname = raw_input( + 'Please enter the postgresql hostname or ip address (enter for "localhost"): ') or 'localhost' + + if not configure_existing_user: + if hostname not in ['localhost', '127.0.0.1']: + print('ERROR: can only create postgresql user on localhost') + sys.exit(1) + username, password, process_status = self._configure_new_postgres_user(current_psql_output) + if hostname.lower() in ['localhost', '127.0.0.1']: + database_name = raw_input( + 'Please enter the {blue} database name {white} (press enter to use "faraday"): '.format( + blue=Fore.BLUE, white=Fore.WHITE)) or 'faraday' + + process_status = self._create_database(username, database_name, current_psql_output) + self._check_psql_output(psql_output, process_status) + current_psql_output.seek(0) + psql_output = current_psql_output.read() + psql_log_file.write(psql_output) + current_psql_output.seek(0) + psql_output = current_psql_output.read() + self._check_psql_output(psql_output, process_status) + else: + username, password = self._configure_existing_postgres_user(current_psql_output) + database_name = raw_input( + 'Please enter the {blue} database name {white} (press enter to use "faraday"): '.format( + blue=Fore.BLUE, white=Fore.WHITE)) or 'faraday' + current_psql_output.close() - conn_string = self._save_config(config, username, password, database_name) + conn_string = self._save_config(config, username, password, database_name, hostname) self._create_tables(conn_string) except KeyboardInterrupt: current_psql_output.close() print('User cancelled.') sys.exit(1) + def _configure_existing_postgres_user(self, psql_log_file): + username = raw_input('Please enter the postgresql username: ') + password = getpass.getpass('Please enter the postgresql password: ') + + return username, password + def _check_psql_output(self, psql_log_output, process_status): if 'unknown user: postgres' in psql_log_output: print('ERROR: Postgres user not found. Did you install package {blue}postgresql{white}?'.format(blue=Fore.BLUE, white=Fore.WHITE)) @@ -94,7 +129,7 @@ def generate_random_pw(self, pwlen): rng = SystemRandom() return "".join([rng.choice(string.ascii_letters + string.digits) for _ in xrange(pwlen)]) - def _configure_postgres(self, psql_log_file): + def _configure_new_postgres_user(self, psql_log_file): """ This step will create the role on the database. we return username and password and those values will be saved in the config file. @@ -109,29 +144,27 @@ def _configure_postgres(self, psql_log_file): p.wait() return username, password, p.returncode - def _create_database(self, username, psql_log_file): + def _create_database(self, database_name, username, psql_log_file): """ This step uses the createdb command to add a new database. """ postgres_command = ['sudo', '-u', 'postgres'] - database_name = raw_input('Please enter the {blue} database name {white} (press enter to use "faraday"): '.format(blue=Fore.BLUE, white=Fore.WHITE)) or 'faraday' print('Creating database {0}'.format(database_name)) command = postgres_command + ['createdb', '-O', username, database_name] p = Popen(command, stderr=psql_log_file, stdout=psql_log_file) p.wait() return database_name, p.returncode - def _save_config(self, config, username, password, database_name): + def _save_config(self, config, username, password, database_name, hostname): """ This step saves database configuration to server.ini """ - db_server = 'localhost' print('Saving database credentials file in {0}'.format(LOCAL_CONFIG_FILE)) conn_string = 'postgresql+psycopg2://{username}:{password}@{server}/{database_name}'.format( username=username, password=password, - server=db_server, + server=hostname, database_name=database_name ) config.set('database', 'connection_string', conn_string) From af322b38a3a0d3e9474592117c26f219b92c98f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 19 Dec 2017 16:11:50 -0300 Subject: [PATCH 0670/1506] Fix status field of the service Don't allow blank values, use a select box instead of allowing arbitrary inputs when editing (it was already done when creating) --- server/www/scripts/services/controllers/serviceModalEdit.js | 5 +++-- server/www/scripts/services/partials/modalEdit.html | 3 ++- server/www/scripts/services/partials/modalNew.html | 1 - 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/server/www/scripts/services/controllers/serviceModalEdit.js b/server/www/scripts/services/controllers/serviceModalEdit.js index 906bcc11c38..381d53d2a2f 100644 --- a/server/www/scripts/services/controllers/serviceModalEdit.js +++ b/server/www/scripts/services/controllers/serviceModalEdit.js @@ -4,8 +4,8 @@ angular.module('faradayApp') .controller('serviceModalEdit', - ['$scope', '$modalInstance', '$routeParams','service', 'servicesManager', 'commonsFact', - function($scope, $modalInstance, $routeParams, service, servicesManager, commons) { + ['$scope', '$modalInstance', '$routeParams', 'SERVICE_STATUSES', 'service', 'servicesManager', 'commonsFact', + function($scope, $modalInstance, $routeParams, SERVICE_STATUSES, service, servicesManager, commons) { init = function() { // current Workspace @@ -26,6 +26,7 @@ angular.module('faradayApp') } else { $scope.services_selected = service; } + $scope.statuses = SERVICE_STATUSES; }; $scope.ok = function() { diff --git a/server/www/scripts/services/partials/modalEdit.html b/server/www/scripts/services/partials/modalEdit.html index 3a59a658895..680ed229c9f 100644 --- a/server/www/scripts/services/partials/modalEdit.html +++ b/server/www/scripts/services/partials/modalEdit.html @@ -55,7 +55,8 @@
    Version
    Status
    - +
    diff --git a/server/www/scripts/services/partials/modalNew.html b/server/www/scripts/services/partials/modalNew.html index 8141a8f8f40..db60e26a68a 100644 --- a/server/www/scripts/services/partials/modalNew.html +++ b/server/www/scripts/services/partials/modalNew.html @@ -56,7 +56,6 @@
    Version
    Status
    From c76fdaf95a54b56f40cc843d4967692257871066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 19 Dec 2017 16:23:33 -0300 Subject: [PATCH 0671/1506] Use same service schema in normal and host-nested API --- server/api/modules/hosts.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 9101c5aeff7..006a02da871 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -24,6 +24,7 @@ SelfNestedField ) from server.models import Host, Service, db, Hostname +from server.api.modules.services import ServiceSchema host_api = Blueprint('host_api', __name__) @@ -75,18 +76,6 @@ class Meta(FilterSetMeta): service = ServiceFilter(fields.Str()) -class ServiceSchema(AutoSchema): - # TODO migration: use the schema in ./services.py - vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) - credentials = fields.Integer(attribute='credentials_count', dump_only=True) - ports = fields.Integer(attribute='port') - - class Meta: - model = Service - fields = ('id', 'name', 'description', 'port', 'ports', 'protocol', - 'status', 'vulns', 'credentials', 'version') - - class HostsView(PaginatedMixin, FilterAlchemyMixin, ReadWriteWorkspacedView): From 47cfbbff706eb0b221c722f4a154730ded5dca3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 19 Dec 2017 16:24:08 -0300 Subject: [PATCH 0672/1506] Replace "ports" for "port" in the Web UI Our new schema only allows one port per service. If a service is run on two or more ports, the user will have to create different services. --- server/www/scripts/services/partials/list.html | 2 +- server/www/scripts/services/partials/modalEdit.html | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/www/scripts/services/partials/list.html b/server/www/scripts/services/partials/list.html index 19e6020e85b..9e80894b06c 100644 --- a/server/www/scripts/services/partials/list.html +++ b/server/www/scripts/services/partials/list.html @@ -64,7 +64,7 @@

    No services found for {{host.ip}}

    Version - Ports + Port Protocol diff --git a/server/www/scripts/services/partials/modalEdit.html b/server/www/scripts/services/partials/modalEdit.html index 680ed229c9f..0bfd952155a 100644 --- a/server/www/scripts/services/partials/modalEdit.html +++ b/server/www/scripts/services/partials/modalEdit.html @@ -39,7 +39,7 @@

    Edit service

    Port
    - +
    @@ -74,7 +74,7 @@
    Description - Ports + Port Protocol From 8783c286fb2f7516d7c58574db9603a27f0e2cde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 19 Dec 2017 18:01:48 -0300 Subject: [PATCH 0673/1506] Fix service update in the web ui Now it works for updating one or many objects. --- server/api/modules/services.py | 2 +- server/www/scripts/hosts/controllers/host.js | 47 ++++++++++--------- .../scripts/services/providers/services.js | 13 ++--- 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/server/api/modules/services.py b/server/api/modules/services.py index fdb34461a13..55250035cd3 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -29,7 +29,7 @@ class ServiceSchema(AutoSchema): owner = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator') port = fields.Integer(dump_only=True) # Port is loaded via ports - ports = MutableField(fields.String(), + ports = MutableField(fields.Integer(), fields.Method(deserialize='load_port'), required=True, attribute='port') diff --git a/server/www/scripts/hosts/controllers/host.js b/server/www/scripts/hosts/controllers/host.js index 7a9b02a4848..ee5f4ad968a 100644 --- a/server/www/scripts/hosts/controllers/host.js +++ b/server/www/scripts/hosts/controllers/host.js @@ -21,6 +21,29 @@ angular.module('faradayApp') }); }; + loadServices = function(){ + // services by host + var hostId = $routeParams.hidId; + dashboardSrv.getServicesByHost($scope.workspace, hostId) + .then(function(services) { + return $q.all(services); + }) + .then(function(services) { + $scope.services = services; + + $scope.services.forEach(function(service) { + service.uri = encodeURIComponent(encodeURIComponent("(" + service.ports + "/" + service.protocol + ") " + service.name)); + }); + + $scope.loadedServices = true; + + return services; + }) + .catch(function(e) { + console.log(e); + }); + }; + init = function() { $scope.selectall_service = false; // current workspace @@ -43,28 +66,9 @@ angular.module('faradayApp') $scope.workspaces = wss; }); - // current host + // current host and its services loadHosts(); - - // services by host - dashboardSrv.getServicesByHost($scope.workspace, hostId) - .then(function(services) { - return $q.all(services); - }) - .then(function(services) { - $scope.services = services; - - $scope.services.forEach(function(service) { - service.uri = encodeURIComponent(encodeURIComponent("(" + service.ports + "/" + service.protocol + ") " + service.name)); - }); - - $scope.loadedServices = true; - - return services; - }) - .catch(function(e) { - console.log(e); - }); + loadServices(hostId); $scope.pageSize = 10; $scope.currentPage = 1; @@ -268,6 +272,7 @@ angular.module('faradayApp') $scope.update = function(services, data) { services.forEach(function(service) { servicesManager.updateService(service, data, $scope.workspace).then(function(s) { + loadServices(); }, function(message) { console.log(message); }); diff --git a/server/www/scripts/services/providers/services.js b/server/www/scripts/services/providers/services.js index f3f68d0582d..3a98ae97587 100644 --- a/server/www/scripts/services/providers/services.js +++ b/server/www/scripts/services/providers/services.js @@ -106,16 +106,9 @@ angular.module('faradayApp') servicesManager.updateService = function(service, data, ws) { var deferred = $q.defer(); var self = this; - this.getService(service._id, ws).then(function(resp) { - resp.update(data, ws).then(function() { - // we need to reload the service in order - // to update _rev - service = self._load(service._id, ws, deferred); - deferred.resolve(service); - }) - }, function(){ - // service doesn't exist - deferred.reject("Service doesn't exist"); + self._get(service._id, service).update(data, ws).then(function(){ + service = self._load(service._id, ws, deferred); + deferred.resolve(service); }); return deferred.promise; } From 6ab0acdfea969ce931162ef45895fa2672843963 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 21 Dec 2017 13:52:55 -0300 Subject: [PATCH 0674/1506] Fix initdb bugs after adding new workflow --- server/commands/initdb.py | 40 ++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index 89fcf9e8261..cb96dde02c2 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -51,7 +51,6 @@ def run(self): * save new configuration on server.ini. * creates tables. """ - current_psql_output = TemporaryFile() try: config = ConfigParser() config.read(LOCAL_CONFIG_FILE) @@ -62,7 +61,6 @@ def run(self): # current_psql_output is for checking psql command already known errors for each execution. psql_log_filename = os.path.join(faraday_path_conf, 'logs', 'psql_log.log') configure_existing_user = None - psql_output = '' with open(psql_log_filename, 'a+') as psql_log_file: while configure_existing_user is None: configure_existing_user = raw_input('Do you {blue} already have {white} a postgresql username and password? (yes/no): '.format(blue=Fore.BLUE, white=Fore.WHITE)) @@ -72,7 +70,7 @@ def run(self): configure_existing_user = False else: print('Invalid option. Please type "yes" or "no" (ctrl-c to cancel): ') - configure_existing = None + configure_existing_user = None hostname = raw_input( 'Please enter the postgresql hostname or ip address (enter for "localhost"): ') or 'localhost' @@ -80,22 +78,24 @@ def run(self): if hostname not in ['localhost', '127.0.0.1']: print('ERROR: can only create postgresql user on localhost') sys.exit(1) + current_psql_output = TemporaryFile() username, password, process_status = self._configure_new_postgres_user(current_psql_output) + current_psql_output.seek(0) + psql_output = current_psql_output.read() + # persist log in the faraday log psql_log.log + psql_log_file.write(psql_output) + self._check_psql_output(current_psql_output, process_status) + if hostname.lower() in ['localhost', '127.0.0.1']: database_name = raw_input( 'Please enter the {blue} database name {white} (press enter to use "faraday"): '.format( blue=Fore.BLUE, white=Fore.WHITE)) or 'faraday' - - process_status = self._create_database(username, database_name, current_psql_output) - self._check_psql_output(psql_output, process_status) - current_psql_output.seek(0) - psql_output = current_psql_output.read() - psql_log_file.write(psql_output) - current_psql_output.seek(0) - psql_output = current_psql_output.read() - self._check_psql_output(psql_output, process_status) + current_psql_output = TemporaryFile() + process_status = self._create_database(database_name, username, current_psql_output) + current_psql_output.seek(0) + self._check_psql_output(current_psql_output, process_status) else: - username, password = self._configure_existing_postgres_user(current_psql_output) + username, password = self._configure_existing_postgres_user() database_name = raw_input( 'Please enter the {blue} database name {white} (press enter to use "faraday"): '.format( blue=Fore.BLUE, white=Fore.WHITE)) or 'faraday' @@ -108,21 +108,23 @@ def run(self): print('User cancelled.') sys.exit(1) - def _configure_existing_postgres_user(self, psql_log_file): + def _configure_existing_postgres_user(self): username = raw_input('Please enter the postgresql username: ') password = getpass.getpass('Please enter the postgresql password: ') return username, password - def _check_psql_output(self, psql_log_output, process_status): - if 'unknown user: postgres' in psql_log_output: + def _check_psql_output(self, current_psql_output_file, process_status): + psql_output = current_psql_output_file.read() + if 'unknown user: postgres' in psql_output: print('ERROR: Postgres user not found. Did you install package {blue}postgresql{white}?'.format(blue=Fore.BLUE, white=Fore.WHITE)) - elif 'could not connect to server' in psql_log_output: + elif 'could not connect to server' in psql_output: print('ERROR: {red}PostgreSQL service{white} is not running. Please verify that it is running in port 5432 before executing setup script.'.format(red=Fore.RED, white=Fore.WHITE)) elif process_status > 0: - print('ERROR: ' + psql_log_output) + print('ERROR: ' + psql_output) if process_status is not 0: + current_psql_output_file.close() # delete temp file sys.exit(process_status) def generate_random_pw(self, pwlen): @@ -151,7 +153,7 @@ def _create_database(self, database_name, username, psql_log_file): postgres_command = ['sudo', '-u', 'postgres'] print('Creating database {0}'.format(database_name)) command = postgres_command + ['createdb', '-O', username, database_name] - p = Popen(command, stderr=psql_log_file, stdout=psql_log_file) + p = Popen(command, stderr=psql_log_file, stdout=psql_log_file, cwd='/tmp') p.wait() return database_name, p.returncode From b02225b5196a2fe2152565146f1675a926d5132b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 21 Dec 2017 15:26:37 -0300 Subject: [PATCH 0675/1506] Return conflict when vuln already exists * Added support for conflict for vuln and vuln web. * Added unit tests that raise conflicts for both cases --- server/api/base.py | 2 +- server/api/modules/vulns.py | 10 +++++- test_cases/test_api_vulnerability.py | 53 ++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 98c0f2805f6..2200c09d54a 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -252,7 +252,7 @@ def _validate_uniqueness(self, obj, object_id=None): db.session.rollback() abort(409, ValidationError( { - 'message': 'Existing value for %s field: %s' % (field_name, value), + 'message': 'Existing value for unique columns: %s' % (field_names, ), 'object': conflict_data, } )) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 48b3081c593..7010bbc8dfd 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -301,17 +301,25 @@ class VulnerabilityView(PaginatedMixin, ReadWriteWorkspacedView): route_base = 'vulns' filterset_class = VulnerabilityFilterSet + unique_fields_by_class = { + 'Vulnerability': [('name', 'description', 'host_id', 'service_id')], + 'VulnerabilityWeb': [('name', 'description', 'service_id', 'method', 'parameter_name', 'path', 'website')], + } model_class_dict = { 'Vulnerability': Vulnerability, 'VulnerabilityWeb': VulnerabilityWeb, - 'VulnerabilityGeneric': VulnerabilityGeneric, + 'VulnerabilityGeneric': VulnerabilityGeneric, # For listing objects } schema_class_dict = { 'Vulnerability': VulnerabilitySchema, 'VulnerabilityWeb': VulnerabilityWebSchema } + def _validate_uniqueness(self, obj, object_id=None): + self.unique_fields = self.unique_fields_by_class[obj.__class__.__name__] + super(VulnerabilityView, self)._validate_uniqueness(obj, object_id) + def _perform_create(self, data, **kwargs): data = self._parse_data(self._get_schema_instance(kwargs), request) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 96c972ef777..bb55b4e1ef9 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -913,3 +913,56 @@ def test_update_with_parent_of_other_workspace( res = test_client.put(self.url(credential), data=data) assert res.status_code == 400 assert 'Parent id not found' in res.data + + def test_create_vuln_multiple_times_returns_conflict(self, host_with_hostnames, test_client, session): + """ + This one should only check basic vuln properties + :param host_with_hostnames: + :param test_client: + :param session: + :return: + """ + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='Vulnerability', + parent_id=host_with_hostnames.id, + parent_type='Host', + refs=[], + policyviolations=[], + description='helloworld', + severity='low', + ) + ws_name = host_with_hostnames.workspace.name + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) + assert res.status_code == 201 + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), + data=raw_data) + assert res.status_code == 409 + + def test_create_webvuln_multiple_times_returns_conflict(self, host_with_hostnames, test_client, session): + """ + This one should only check basic vuln properties + :param host_with_hostnames: + :param test_client: + :param session: + :return: + """ + service = ServiceFactory.create(workspace=self.workspace) + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='New vulnsweb', + vuln_type='VulnerabilityWeb', + parent_id=service.id, + parent_type='Service', + refs=[], + policyviolations=[], + description='helloworld', + severity='low', + ) + ws_name = host_with_hostnames.workspace.name + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) + assert res.status_code == 201 + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), + data=raw_data) + assert res.status_code == 409 \ No newline at end of file From 02e4f0143903b59e467b64cc5e24f7cdfb5340f9 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 21 Dec 2017 15:49:24 -0300 Subject: [PATCH 0676/1506] Use _create_server_api_url to generate test urls --- test_cases/test_managers_mapper_manager.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/test_cases/test_managers_mapper_manager.py b/test_cases/test_managers_mapper_manager.py index f0324274e10..a706340294c 100644 --- a/test_cases/test_managers_mapper_manager.py +++ b/test_cases/test_managers_mapper_manager.py @@ -1,9 +1,9 @@ from functools import partial -import mock import pytest from managers.mapper_manager import MapperManager +from persistence.server.server import _create_server_api_url from persistence.server.models import Host, Service, Vuln, Credential import persistence.server.server from persistence.server.utils import get_host_properties @@ -204,7 +204,8 @@ def test_save_without_command(self, obj_class, many_test_data, monkeypatch, sess if obj_class in [Vuln, Credential]: test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] def mock_server_post(test_data, post_url, update=False, expected_response=201, **params): - assert post_url == 'http://localhost:5985/_api/v2/ws/test/{0}/'.format(test_data['api_end_point']) + assert post_url == '{0}/ws/test/{1}/'.format( + _create_server_api_url(), test_data['api_end_point']) assert expected_response == 201 assert update == False metadata = params.pop('metadata') @@ -238,7 +239,7 @@ def test_save_with_command(self, obj_class, many_test_data, monkeypatch, session if obj_class in [Vuln, Credential]: test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] def mock_server_post(test_data, post_url, update=False, expected_response=201, **params): - assert post_url == 'http://localhost:5985/_api/v2/ws/test/{0}/?command_id={1}'.format(test_data['api_end_point'], params['command_id']) + assert post_url == '{0}/ws/test/{1}/?command_id={2}'.format(_create_server_api_url(), test_data['api_end_point'], params['command_id']) assert expected_response == 201 assert update == False metadata = params.pop('metadata') @@ -275,7 +276,7 @@ def test_update_without_command(self, obj_class, many_test_data, monkeypatch, se if obj_class in [Vuln, Credential]: test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] def mock_server_put(test_data, put_url, update=False, expected_response=201, **params): - assert put_url == 'http://localhost:5985/_api/v2/ws/test/{0}/{1}/'.format(test_data['api_end_point'], test_data['id']) + assert put_url == '{0}/ws/test/{1}/{2}/'.format(_create_server_api_url(), test_data['api_end_point'], test_data['id']) assert expected_response == 200 assert update == False metadata = params.pop('metadata') @@ -319,7 +320,11 @@ def test_update_with_command(self, obj_class, many_test_data, monkeypatch, sessi relational_model = test_data['factory'].create() session.commit() def mock_server_put(put_url, update=False, expected_response=201, **params): - assert put_url == 'http://localhost:5985/_api/v2/ws/test/{0}/{1}/?command_id={2}'.format(test_data['api_end_point'], test_data['id'], params['command_id']) + assert put_url == '{0}/ws/test/{1}/{2}/?command_id={3}'.format( + _create_server_api_url(), + test_data['api_end_point'], + test_data['id'], + params['command_id']) assert expected_response == 200 assert update == False return { @@ -344,7 +349,11 @@ def test_find_obj_by_id(self, obj_class, many_test_data, session, monkeypatch): def mock_unsafe_io_with_server(host, test_data, server_io_function, server_expected_response, server_url, **payload): mocked_response = test_data['mocked_response'] - assert 'http://localhost:5985/_api/v2/ws/{0}/{1}/{2}/'.format(persisted_obj.workspace.name, test_data['api_end_point'], persisted_obj.id) == server_url + assert '{0}/ws/{1}/{2}/{3}/'.format( + _create_server_api_url(), + persisted_obj.workspace.name, + test_data['api_end_point'], + persisted_obj.id) == server_url return MockResponse(mocked_response, 200) monkeypatch.setattr(persistence.server.server, '_unsafe_io_with_server', partial(mock_unsafe_io_with_server, persisted_obj, test_data)) From dfaf70802a6254eadf426b93f252fcaf3a4a8050 Mon Sep 17 00:00:00 2001 From: Winna_Z Date: Thu, 21 Dec 2017 17:22:30 -0300 Subject: [PATCH 0677/1506] Screenshot saver --- bin/scheenshot_server.py | 72 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 bin/scheenshot_server.py diff --git a/bin/scheenshot_server.py b/bin/scheenshot_server.py new file mode 100644 index 00000000000..f191fbdf019 --- /dev/null +++ b/bin/scheenshot_server.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python2.7 +# -*- coding: utf-8 -*- +""" +Faraday Penetration Test IDE +Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information +""" +from __future__ import print_function +from persistence.server.server_io_exceptions import ResourceDoesNotExist +from persistence.server import models +from utils.user_input import query_yes_no +import os +try: + from selenium import webdriver +except Exception: + print ("Missing dependencies: (selenium)") + + + +__description__ = 'Takes a Schreenshot of the ip:ports of a given protocol' +__prettyname__ = 'Scheenshot_server' + +def scheenshot(path, protocol, ip, port): + driver = webdriver.PhantomJS() + driver.set_window_size(1024, 768) # set the window size that you need + driver.set_page_load_timeout(5) + try: + driver.get(protocol + "://" + ip + ":" + port + "/") + driver.get_screenshot_as_file (os.path.join( path , ip + "_" + port + ".png")) + except Exception: + print("Coudn't connect") + finally: + driver.quit + + return 0 + +def main(workspace='', args=None, parser=None): + parser.add_argument( 'protocol', help="Desired protocol" , default="") + parser.add_argument( '--path', help="Saves the Image in a given path", default="." ) + parsed_args = parser.parse_args(args) + + protocols = parsed_args.protocol.split(",") + print (protocols) + path = parsed_args.path + + for protocol in protocols: + + if not os.path.exists(path): + print ("Invalid Path") + exit() + + + try: + services = models.get_services(workspace) + except ResourceDoesNotExist: + print ("Invalid workspace name: ", workspace) + return 1, None + + + for service in services: + service_protocol = service.protocol.lower() + + if service_protocol == protocol: + port = str(service.ports[0]) + + interface_id = ".".join(service.id.split(".")[:2]) + interface = models.get_interface(workspace, interface_id) + ip = interface.ipv4["address"] + + print (protocol + "://" + ip + ":" + port) + scheenshot(path, protocol , ip, port ) + return 0, None \ No newline at end of file From 9b71a55b9964bb64f2068eebd9d40ef4ae9b0563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 21 Dec 2017 18:26:45 -0300 Subject: [PATCH 0678/1506] Add generic write tests for credentials API (tests success story) Testing success story: I found a bug when updating a host credential and changing its parent to a service. It was trying to save a credential with both a host and a service, and the check constraint failed. This happen because host_id wasn't set to None. The downside of this is that the bug only happened some times because the credential factory randomly choices if its parent is a host or a service. --- server/api/modules/credentials.py | 6 +++- server/api/modules/vulns.py | 2 ++ test_cases/factories.py | 48 +++++++++++++++++++++++--- test_cases/test_api_credentials.py | 18 ++++++---- test_cases/test_api_vulnerability.py | 2 +- test_cases/test_api_workspaced_base.py | 14 +++++--- 6 files changed, 72 insertions(+), 18 deletions(-) diff --git a/server/api/modules/credentials.py b/server/api/modules/credentials.py index c9fba8a5c74..b15ccda3354 100644 --- a/server/api/modules/credentials.py +++ b/server/api/modules/credentials.py @@ -21,7 +21,8 @@ class CredentialSchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') _rev = fields.String(default='', dump_only=True) owned = fields.Boolean(default=False) - owner = fields.String(dump_only=True, attribute='creator.username', default='') + owner = fields.String(dump_only=True, attribute='creator.username', + default='') username = fields.String(default='') password = fields.String(default='') description = fields.String(default='') @@ -61,9 +62,11 @@ def set_parent(self, data): if parent_type == 'Host': parent_class = Host parent_field = 'host_id' + not_parent_field = 'service_id' elif parent_type == 'Service': parent_class = Service parent_field = 'service_id' + not_parent_field = 'host_id' else: raise ValidationError( 'Unknown parent type: {}'.format(parent_type)) @@ -74,6 +77,7 @@ def set_parent(self, data): except NoResultFound: raise InvalidUsage('Parent id not found: {}'.format(parent_id)) data[parent_field] = parent.id + data[not_parent_field] = None return data diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 0e2e3b2b571..fc74dd81f1f 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -222,6 +222,8 @@ def post_load_parent(self, data): except NoResultFound: raise ValidationError('Parent id not found: {}'.format(parent_id)) data[parent_field] = parent.id + # TODO migration: check what happens when updating the parent from + # service to host or viceverse return data diff --git a/test_cases/factories.py b/test_cases/factories.py index 59f575a2f51..823bb65834b 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -194,8 +194,12 @@ def _after_postgeneration(cls, obj, create, results=None): obj, create, results) if isinstance(obj, dict): # This happens when built with build_dict - return - if obj.host and obj.service: + if obj['host'] and obj['service']: + if random.choice([True, False]): + obj['host'] = None + else: + obj['service'] = None + elif obj.host and obj.service: # Setting both service and host to a vuln is not allowed. # This will pick one of them randomly. # TODO: Check is this is recommended @@ -204,6 +208,38 @@ def _after_postgeneration(cls, obj, create, results=None): else: obj.service = None + @classmethod + def build_dict(cls, **kwargs): + ret = super(HasParentHostOrService, cls).build_dict(**kwargs) + service = ret.pop('service') + host = ret.pop('host') + if host is not None: + assert service is None + + # This should be set by the SelfAttribute of the SubFactory, but I + # don't kwown why it doesn't work here + host.workspace = kwargs.get('workspace', host.workspace) + + db.session.add(host) + db.session.commit() # Needed to get the object IDs + ret['parent_type'] = 'Host' + ret['parent'] = host.id + elif service is not None: + assert host is None + + # This should be set by the SelfAttribute of the SubFactory, but I + # don't kwown why it doesn't work here + service.workspace = service.host.workspace = kwargs.get( + 'workspace', service.workspace) + + db.session.add(service) + db.session.commit() # Needed to get the object IDs + ret['parent_type'] = 'Service' + ret['parent'] = service.id + else: + raise ValueError("Either host or service must be set") + return ret + class VulnerabilityFactory(HasParentHostOrService, VulnerabilityGenericFactory): @@ -249,8 +285,12 @@ class Meta: class CredentialFactory(HasParentHostOrService, WorkspaceObjectFactory): - host = factory.SubFactory(HostFactory, workspace=factory.SelfAttribute('..workspace')) - service = None # factory.SubFactory(ServiceFactory) + host = factory.SubFactory( + HostFactory, workspace=factory.SelfAttribute('..workspace') + ) + service = factory.SubFactory( + ServiceFactory, workspace=factory.SelfAttribute('..workspace') + ) username = FuzzyText() password = FuzzyText() diff --git a/test_cases/test_api_credentials.py b/test_cases/test_api_credentials.py index 7452cd80289..fa80e1b5bf8 100644 --- a/test_cases/test_api_credentials.py +++ b/test_cases/test_api_credentials.py @@ -2,14 +2,14 @@ from test_cases import factories from test_api_workspaced_base import ( - ReadOnlyAPITests, + ReadWriteAPITests, ) from server.api.modules.credentials import CredentialView from server.models import Credential from test_cases.factories import HostFactory, ServiceFactory -class TestCredentialsAPIGeneric(ReadOnlyAPITests): +class TestCredentialsAPIGeneric(ReadWriteAPITests): model = Credential factory = factories.CredentialFactory view_class = CredentialView @@ -61,8 +61,10 @@ def test_create_from_raw_data_host_as_parent(self, session, test_client, res = test_client.post(self.url(), data=raw_data) assert res.status_code == 201 - def test_get_credentials_for_a_host_backwards_compatibility(self, session, test_client): - credential = self.factory.create() + def test_get_credentials_for_a_host_backwards_compatibility( + self, session, test_client, host): + credential = self.factory.create(host=host, service=None, + workspace=self.workspace) session.commit() res = test_client.get(self.url(workspace=credential.workspace) + '?host_id={0}'.format(credential.host.id)) assert res.status_code == 200 @@ -104,11 +106,13 @@ def test_update_credentials_with_invalid_parent(self, test_client, session): res = test_client.put(self.url(workspace=credential.workspace) + str(credential.id) + '/', data=raw_data) assert res.status_code == 400 - def test_update_credentials(self, test_client, session): - credential = self.factory.create() + def test_update_credentials(self, test_client, session, host): + credential = self.factory.create(host=host, service=None, + workspace=self.workspace) session.commit() - raw_data = self._generate_raw_update_data('Name1', 'Username2', 'Password3', parent_id=credential.host.id) + raw_data = self._generate_raw_update_data( + 'Name1', 'Username2', 'Password3', parent_id=credential.host.id) res = test_client.put(self.url(workspace=credential.workspace) + str(credential.id) + '/', data=raw_data) assert res.status_code == 200 diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 9c2401a5587..6bd749e9566 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -21,7 +21,7 @@ @pytest.mark.usefixtures('logged_user') -class TestListVulnerabilityView(ReadOnlyAPITests): +class TestListVulnerabilityView(ReadOnlyAPITests): # TODO migration: use read write api tests model = Vulnerability factory = factories.VulnerabilityFactory api_endpoint = 'vulns' diff --git a/test_cases/test_api_workspaced_base.py b/test_cases/test_api_workspaced_base.py index c95a2e72409..eddb34bd337 100644 --- a/test_cases/test_api_workspaced_base.py +++ b/test_cases/test_api_workspaced_base.py @@ -97,9 +97,10 @@ def test_404_when_retrieving_unexistent_object(self, test_client, class CreateTestsMixin: def test_create_succeeds(self, test_client): + data = self.factory.build_dict(workspace=self.workspace) res = test_client.post(self.url(), - data=self.factory.build_dict()) - assert res.status_code == 201 + data=data) + assert res.status_code == 201, (res.status_code, res.data) assert self.model.query.count() == OBJECT_COUNT + 1 object_id = res.json['id'] obj = self.model.query.get(object_id) @@ -120,6 +121,8 @@ def test_create_fails_with_existing(self, session, test_client): def test_create_with_existing_in_other_workspace(self, test_client, session, second_workspace): + if not self.unique_fields: + return unique_field = self.unique_fields[0] other_object = self.factory.create(workspace=second_workspace) session.commit() @@ -135,13 +138,14 @@ def test_create_with_existing_in_other_workspace(self, test_client, class UpdateTestsMixin: def test_update_an_object(self, test_client): + data = self.factory.build_dict(workspace=self.workspace) res = test_client.put(self.url(self.first_object), - data=self.factory.build_dict()) + data=data) assert res.status_code == 200 assert self.model.query.count() == OBJECT_COUNT for updated_field in self.update_fields: assert res.json[updated_field] == getattr(self.first_object, - updated_field) + updated_field) def test_update_fails_with_existing(self, test_client, session): for unique_field in self.unique_fields: @@ -157,7 +161,7 @@ def test_update_an_object_fails_with_empty_dict(self, test_client): assert res.status_code == 400 def test_update_cant_change_id(self, test_client): - raw_json = self.factory.build_dict() + raw_json = self.factory.build_dict(workspace=self.workspace) expected_id = self.first_object.id raw_json['id'] = 100000 res = test_client.put(self.url(self.first_object), From 59d669017d1018c57bae18c58af26b856898c18e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 21 Dec 2017 18:37:12 -0300 Subject: [PATCH 0679/1506] Update code to support find obj for vulns and services --- persistence/server/models.py | 12 +- persistence/server/server.py | 12 +- server/api/modules/services.py | 9 +- test_cases/test_managers_mapper_manager.py | 139 ++++++++++++++++++++- test_cases/test_model_events.py | 1 - 5 files changed, 152 insertions(+), 21 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index c218476f637..b079a6bba2d 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -219,7 +219,7 @@ def get_vulns(workspace_name, **params): def get_vuln(workspace_name, vuln_id): """Return the Vuln of id vuln_id. None if not found.""" - return force_unique(get_vulns(workspace_name, couchid=vuln_id)) + return force_unique(get_vulns(workspace_name, id=vuln_id)) def get_web_vulns(workspace_name, **params): @@ -234,7 +234,7 @@ def get_web_vulns(workspace_name, **params): def get_web_vuln(workspace_name, vuln_id): """Return the WebVuln of id vuln_id. None if not found.""" - return force_unique(get_web_vulns(workspace_name, couchid=vuln_id)) + return force_unique(get_web_vulns(workspace_name, id=vuln_id)) def get_services(workspace_name, **params): @@ -249,7 +249,7 @@ def get_services(workspace_name, **params): def get_service(workspace_name, service_id): """Return the Service of id service_id. None if not found.""" - return force_unique(get_services(workspace_name, service_id=service_id)) + return force_unique(get_services(workspace_name, id=service_id)) def get_credentials(workspace_name, **params): @@ -264,7 +264,7 @@ def get_credentials(workspace_name, **params): def get_credential(workspace_name, credential_id): """Return the Credential of id credential_id. None if not found.""" - return force_unique(get_credentials(workspace_name, couchid=credential_id)) + return force_unique(get_credentials(workspace_name, id=credential_id)) def get_notes(workspace_name, **params): @@ -279,7 +279,7 @@ def get_notes(workspace_name, **params): def get_note(workspace_name, note_id): """Return the Note of id note_id. None if not found.""" - return force_unique(get_notes(workspace_name, couchid=note_id)) + return force_unique(get_notes(workspace_name, id=note_id)) def get_workspace(workspace_name): @@ -300,7 +300,7 @@ def get_commands(workspace_name, **params): def get_command(workspace_name, command_id): """Return the Command of id command_id. None if not found.""" - return force_unique(get_commands(workspace_name, couchid=command_id)) + return force_unique(get_commands(workspace_name, id=command_id)) def get_object(workspace_name, object_signature, object_id): diff --git a/persistence/server/server.py b/persistence/server/server.py index d3b48f5c5f2..bd6b48d4aa5 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -261,21 +261,14 @@ def _get_raw_hosts(workspace_name, **params): def _get_raw_vulns(workspace_name, **params): """Take a workspace_name and an arbitrary number of params and return a dictionary with the vulns table.""" - request_url = _create_server_get_url(workspace_name, 'vulns') - return _get(request_url, **params) - - -def _get_raw_interfaces(workspace_name, **params): - """Take a workspace_name and an arbitrary number of params and return - a dictionary with the interfaces table.""" - request_url = _create_server_get_url(workspace_name, 'interfaces') + request_url = _create_server_get_url(workspace_name, 'vulns', params.get('id', None)) return _get(request_url, **params) def _get_raw_services(workspace_name, **params): """Take a workspace_name and an arbitrary number of params and return a dictionary with the services table.""" - request_url = _create_server_get_url(workspace_name, 'services') + request_url = _create_server_get_url(workspace_name, 'services', params.get('id', None)) return _get(request_url, **params) @@ -354,7 +347,6 @@ def _get_faraday_ready_dictionaries(workspace_name, faraday_object_name, """ object_to_func = {'hosts': _get_raw_hosts, 'vulns': _get_raw_vulns, - 'interfaces': _get_raw_interfaces, 'services': _get_raw_services, 'notes': _get_raw_notes, 'credentials': _get_raw_credentials, diff --git a/server/api/modules/services.py b/server/api/modules/services.py index 750c5df72ee..af3269cf0a0 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -30,21 +30,24 @@ class ServiceSchema(AutoSchema): attribute='creator') port = fields.Integer(dump_only=True) # Port is loaded via ports ports = MutableField(fields.String(), - fields.Method(deserialize='load_port'), + fields.Method(deserialize='load_ports', serialize='get_ports'), required=True, attribute='port') status = fields.String(default='open') - parent = fields.Integer(attribute='host_id', load_only=True) # parent is not required for updates + parent = fields.Integer(attribute='host_id') # parent is not required for updates host_id = fields.Integer(attribute='host_id', dump_only=True) summary = fields.Method('get_summary') vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) credentials = fields.Integer(attribute='credentials_count', dump_only=True) metadata = SelfNestedField(MetadataSchema()) - def load_port(self, value): + def load_ports(self, value): # TODO migration: handle empty list and not numeric value return str(value.pop()) + def get_ports(self, obj): + return [obj.port] + def get_summary(self, obj): return "(%s/%s) %s" % (obj.port, obj.protocol, obj.name) diff --git a/test_cases/test_managers_mapper_manager.py b/test_cases/test_managers_mapper_manager.py index a706340294c..48f5960de0d 100644 --- a/test_cases/test_managers_mapper_manager.py +++ b/test_cases/test_managers_mapper_manager.py @@ -6,7 +6,8 @@ from persistence.server.server import _create_server_api_url from persistence.server.models import Host, Service, Vuln, Credential import persistence.server.server -from persistence.server.utils import get_host_properties +from persistence.server.utils import get_host_properties, \ + get_service_properties, get_vuln_properties from test_cases.factories import WorkspaceFactory, CommandFactory, HostFactory, \ ServiceFactory, VulnerabilityFactory, CredentialFactory @@ -143,6 +144,90 @@ # the following dict is used to parametrize find (GET) tests GET_OBJ_DATA = { + Vuln: [ + { + 'factory': VulnerabilityFactory, + 'api_end_point': 'vulns', + 'get_properties_function': get_vuln_properties, + 'mocked_response': { + "website": "", + "_rev": "", + "parent_type": "Service", + "owned": False, + "owner": "leonardo", + "query": "", + "refs": [], + "impact": { + "accountability": False, + "integrity": False, + "confidentiality": False, + "availability": False + }, + "confirmed": True, + "severity": "med", + "service": { + "status": "open", + "protocol": "tcp", + "name": "ssh", + "summary": "(21/tcp) ssh", + "version": "", + "_id": 1, + "ports": "21" + }, + "policyviolations": [], + "params": "", + "type": "Vulnerability", + "method": "", + "metadata": { + "update_time": 1513290499000, + "update_user": "", + "update_action": 0, + "creator": "", + "create_time": 1513290499000, + "update_controller_action": "", + "owner": "leonardo", + "command_id": None + }, + "status": "opened", + "issuetracker": {}, + "description": "description", + "parent": 1, + "tags": [], + "easeofresolution": "trivial", + "hostnames": [], + "pname": "", + "date": "2017-12-14T19:28:19.427274+00:00", + "path": "", + "data": "data", + "response": "", + "desc": "description", + "name": "Vuln test", + "obj_id": "1", + "request": "", + "_attachments": [], + "target": "192.168.0.1", + "_id": 1, + "resolution": "" + }, + 'serialized_expected_results': { + 'confirmed': True, + 'data': 'data', + 'desc': 'description', + 'description': 'description', + 'name': 'Vuln test', + 'owned': False, + 'owner': 'leonardo', + 'parent': 1, + 'parent_type': 'Service', + 'policyviolations': [], + 'refs': [], + 'resolution': '', + 'severity': 'med', + 'status': 'opened' + } + + } + ], Host: [ { 'factory': HostFactory, @@ -170,6 +255,58 @@ 'owned': False, 'owner': 'leonardo'} + } + ], + Service: [ + { + 'factory': ServiceFactory, + 'api_end_point': 'services', + 'parent': { + 'parent_type': 'Host', + 'parent_factory': HostFactory + }, + 'get_properties_function': get_service_properties, + 'mocked_response': { + "status": "open", + "protocol": "tcp", + "description": "Test description", + "vulns": 2, + "_rev": "", + "metadata": { + "update_time": 1513290473000, + "update_user": "", + "update_action": 0, + "creator": "", + "create_time": 1513290473000, + "update_controller_action": "", + "owner": "leonardo", + "command_id": None + }, + "owned": False, + "summary": "(21/tcp) ssh", + "port": 21, + "owner": "leonardo", + "version": "", + "host_id": 1, + "parent": 1, + "id": 1, + "credentials": 0, + "_id": 1, + "ports": [21], + "name": "ssh" + }, + 'serialized_expected_results': { + 'name': 'ssh', + 'description': 'Test description', + 'ports': [21], + 'protocol': 'tcp', + 'status': 'open', + 'parent': 1, + 'version': '', + 'owned': False, + 'owner': 'leonardo' + } + } ] } diff --git a/test_cases/test_model_events.py b/test_cases/test_model_events.py index 0074437d5a7..ece1c9d78c9 100644 --- a/test_cases/test_model_events.py +++ b/test_cases/test_model_events.py @@ -1,5 +1,4 @@ import pytest -from server import events from test_cases.factories import HostFactory, ServiceFactory From ce4235df2a018f4c507c2bdff3a453cf8ad1aa30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 22 Dec 2017 12:39:07 -0300 Subject: [PATCH 0680/1506] Add target to credentials API This way it is shown on the Web UI without doing any change to the frontend --- server/api/modules/credentials.py | 22 ++++++++++++++------ test_cases/test_api_credentials.py | 33 +++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/server/api/modules/credentials.py b/server/api/modules/credentials.py index b15ccda3354..7707c52d7db 100644 --- a/server/api/modules/credentials.py +++ b/server/api/modules/credentials.py @@ -33,6 +33,11 @@ class CredentialSchema(AutoSchema): parent = MutableField(fields.Method('get_parent'), fields.Integer(), required=True) + host_ip = fields.String(dump_only=True, attribute="host.ip", + default=None) + service_name = fields.String(dump_only=True, attribute="service.name", + default=None) + target = fields.Method('get_target', dump_only=True) # for filtering host_id = fields.Integer(load_only=True) @@ -46,14 +51,18 @@ def get_parent_type(self, obj): assert obj.host_id is not None or obj.service_id is not None return 'Service' if obj.service_id is not None else 'Host' + def get_target(self, obj): + if obj.host is not None: + return obj.host.ip + else: + return obj.service.host.ip + '/' + obj.service.name + class Meta: model = Credential - fields = ('id', '_id', "_rev", 'parent', - 'username', 'description', - 'name', 'password', - 'owner', 'owned', 'couchdbid', - 'parent', 'parent_type', - 'metadata') + fields = ('id', '_id', "_rev", 'parent', 'username', 'description', + 'name', 'password', 'owner', 'owned', 'couchdbid', 'parent', + 'parent_type', 'metadata', 'host_ip', 'service_name', + 'target') @post_load def set_parent(self, data): @@ -98,6 +107,7 @@ class CredentialView(FilterAlchemyMixin, ReadWriteWorkspacedView): model_class = Credential schema_class = CredentialSchema filterset_class = CredentialFilterSet + get_joinedloads = [Credential.host, Credential.service] def _envelope_list(self, objects, pagination_metadata=None): credentials = [] diff --git a/test_cases/test_api_credentials.py b/test_cases/test_api_credentials.py index fa80e1b5bf8..88b9b976bc5 100644 --- a/test_cases/test_api_credentials.py +++ b/test_cases/test_api_credentials.py @@ -34,13 +34,16 @@ def test_get_list_backwards_compatibility(self, test_client, session, second_wor u'owner', u'password', u'username', + u'host_ip', + u'service_name', + u'target' ] expected = set(object_properties) result = set(vuln['value'].keys()) assert expected - result == set() def test_create_from_raw_data_host_as_parent(self, session, test_client, - workspace, host_factory): + workspace, host_factory): host = host_factory.create(workspace=workspace) session.commit() raw_data = { @@ -60,6 +63,34 @@ def test_create_from_raw_data_host_as_parent(self, session, test_client, } res = test_client.post(self.url(), data=raw_data) assert res.status_code == 201 + assert res.json['host_ip'] == host.ip + assert res.json['service_name'] is None + assert res.json['target'] == host.ip + + def test_create_from_raw_data_service_as_parent( + self, session, test_client, workspace, service_factory): + service = service_factory.create(workspace=workspace) + session.commit() + raw_data = { + "_id":"1.e5069bb0718aa519852e6449448eedd717f1b90d", + "name":"name", + "username":"username", + "metadata":{"update_time":1508794240799,"update_user":"", + "update_action":0,"creator":"UI Web", + "create_time":1508794240799,"update_controller_action":"", + "owner":""}, + "password":"pass", + "type":"Cred", + "owner":"", + "description":"", + "parent": service.id, + "parent_type": "Service" + } + res = test_client.post(self.url(), data=raw_data) + assert res.status_code == 201 + assert res.json['host_ip'] is None + assert res.json['service_name'] == service.name + assert res.json['target'] == service.host.ip + '/' + service.name def test_get_credentials_for_a_host_backwards_compatibility( self, session, test_client, host): From 53e4e3b33abcf9dd3c937d88ce7d5958d36e8e7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 22 Dec 2017 12:40:36 -0300 Subject: [PATCH 0681/1506] Add global --ignore-nplusone option to pytest To avoid having to use the ignore_nplusone in many tests, the developer now can run pytest with the --ignore-nplusone option and it will be disabled globally. This makes a better TDD process of write tests -> code something that works -> fix performance issues --- test_cases/conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 330283571bf..09209d281fc 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -59,6 +59,8 @@ def pytest_addoption(parser): parser.addoption('--connection-string', default='sqlite://', help="Database connection string. Defaults to in-memory " "sqlite if not specified:") + parser.addoption('--ignore-nplusone', action='store_true', + help="Globally ignore nplusone errors") @pytest.fixture(scope='session') @@ -75,6 +77,8 @@ def teardown(): ctx.pop() request.addfinalizer(teardown) + app.config['NPLUSONE_RAISE'] = not request.config.getoption( + '--ignore-nplusone') return app From aa089d77bc4f5b307b5311827fa23af6989717ef Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 22 Dec 2017 14:33:10 -0300 Subject: [PATCH 0682/1506] Add create and update for vuln web --- persistence/server/models.py | 4 +- persistence/server/server.py | 18 +++++-- test_cases/test_managers_mapper_manager.py | 56 +++++++++++++++++++++- 3 files changed, 69 insertions(+), 9 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index b079a6bba2d..af150154e39 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -404,13 +404,13 @@ def create_vuln_web(workspace_name, vuln_web, command_id): @_ignore_in_changes -def update_vuln_web(workspace_name, vuln_web): +def update_vuln_web(workspace_name, vuln_web, command_id): """Take a workspace_name and a VulnWeb object and update it in the sever. Return the server's json response as a dictionary. """ vuln_web_properties = get_vuln_web_properties(vuln_web) - return server.update_vuln_web(workspace_name, vuln_web.getID(), **vuln_web_properties) + return server.update_vuln_web(workspace_name, command_id, vuln_web.getID(), **vuln_web_properties) @_ignore_in_changes diff --git a/persistence/server/server.py b/persistence/server/server.py index bd6b48d4aa5..83ea994e4e3 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -1079,10 +1079,13 @@ def update_vuln(workspace_name, command_id, id, name, description, parent, metadata=metadata, policyviolations=policyviolations) -def create_vuln_web(workspace_name, command_id, name, description, owned=None, owner="", - confirmed=False, data="", refs=None, severity="info", resolution="", - desc="", metadata=None, method=None, params="", path=None, pname=None, - query=None, request=None, response=None, category="", website=None, + +def create_vuln_web(workspace_name, command_id, name, description, parent, + parent_type, owned=None, owner="", confirmed=False, + data="", refs=None, severity="info", resolution="", + desc="", metadata=None, method=None, params="", + path=None, pname=None, query=None, request=None, + response=None, category="", website=None, status=None, policyviolations=[]): """Creates a vuln web. @@ -1116,6 +1119,8 @@ def create_vuln_web(workspace_name, command_id, name, description, owned=None, o A dictionary with the server's response. """ return _save_to_server(workspace_name, + parent=parent, + parent_type=parent_type, command_id=command_id, name=name, description=description, @@ -1141,7 +1146,8 @@ def create_vuln_web(workspace_name, command_id, name, description, owned=None, o type='VulnerabilityWeb', policyviolations=policyviolations) -def update_vuln_web(workspace_name, command_id, id, name, description, owned=None, owner="", +def update_vuln_web(workspace_name, command_id, id, name, description, + parent, parent_type, owned=None, owner="", confirmed=False, data="", refs=None, severity="info", resolution="", desc="", metadata=None, method=None, params="", path=None, pname=None, query=None, request=None, response=None, category="", website=None, @@ -1179,6 +1185,8 @@ def update_vuln_web(workspace_name, command_id, id, name, description, owned=Non """ return _update_in_server(workspace_name, id, + parent=parent, + parent_type=parent_type, command_id=command_id, name=name, description=description, diff --git a/test_cases/test_managers_mapper_manager.py b/test_cases/test_managers_mapper_manager.py index 48f5960de0d..af6daf49856 100644 --- a/test_cases/test_managers_mapper_manager.py +++ b/test_cases/test_managers_mapper_manager.py @@ -4,12 +4,13 @@ from managers.mapper_manager import MapperManager from persistence.server.server import _create_server_api_url -from persistence.server.models import Host, Service, Vuln, Credential +from persistence.server.models import Host, Service, Vuln, Credential, VulnWeb import persistence.server.server from persistence.server.utils import get_host_properties, \ get_service_properties, get_vuln_properties from test_cases.factories import WorkspaceFactory, CommandFactory, HostFactory, \ - ServiceFactory, VulnerabilityFactory, CredentialFactory + ServiceFactory, VulnerabilityFactory, CredentialFactory, \ + VulnerabilityWebFactory # OBJ_DATA is used to parametrize tests (https://docs.pytest.org/en/latest/parametrize.html) # We use it to test all model classes. @@ -112,6 +113,57 @@ 'resolution': None, }, }], + VulnWeb: [{ + 'factory': VulnerabilityWebFactory, + 'api_end_point': 'vulns', + 'parent': { + 'parent_type': 'Service', + 'parent_factory': ServiceFactory + }, + 'data': { + '_id': 1, + 'name': 'Service vulnerable', + 'desc': 'My vuln', + 'owned': False, + 'owner': 'leo', + 'severity': 'critical', + 'data': '', + 'website': 'www.faradaysec.com', + 'method': 'GET', + 'pname': 'param_name', + 'params': 'params', + 'path': 'path', + 'request': 'test', + 'query': 'query test', + 'response': 'repsonse data', + }, + 'expected_payload': { + 'category': '', + 'command_id': None, + 'name': 'Service vulnerable', + 'desc': 'My vuln', + 'description': 'My vuln', + 'owner': 'leo', + 'owned': False, + 'confirmed': False, + 'severity': 'critical', + 'data': '', + 'type': 'VulnerabilityWeb', + 'parent_type': 'Service', + 'policyviolations': [], + 'refs': [], + 'status': 'opened', + 'resolution': None, + 'website': 'www.faradaysec.com', + 'method': 'GET', + 'pname': 'param_name', + 'params': 'params', + 'path': 'path', + 'request': 'test', + 'query': 'query test', + 'response': 'repsonse data', + }, + }], Credential: [{ 'factory': CredentialFactory, 'api_end_point': 'credential', From aa6e6974ddcb1883cedc88ad3b20cbf3605d766f Mon Sep 17 00:00:00 2001 From: micabot Date: Fri, 22 Dec 2017 16:01:01 -0300 Subject: [PATCH 0683/1506] Fix License list and insert --- server/api/modules/licenses.py | 12 ++++-- .../www/scripts/licenses/providers/license.js | 42 +++++++------------ .../scripts/licenses/providers/licenses.js | 42 ++++++++----------- 3 files changed, 42 insertions(+), 54 deletions(-) diff --git a/server/api/modules/licenses.py b/server/api/modules/licenses.py index fbc59954b23..d3bd4f7499c 100644 --- a/server/api/modules/licenses.py +++ b/server/api/modules/licenses.py @@ -7,16 +7,22 @@ from marshmallow import fields from server.models import License -from server.api.base import ReadWriteView, AutoSchema +from server.api.base import ( + ReadWriteView, + AutoSchema, +) license_api = Blueprint('license_api', __name__) class LicenseSchema(AutoSchema): - + _id = fields.Integer(dump_only=True, attribute='id') + end = fields.DateTime(attribute='end_date') + lictype = fields.String(attribute='type') + start = fields.DateTime(attribute='start_date') class Meta: model = License - fields = ('id', 'product', 'start_date', 'end_date') + fields = ('_id', 'id', 'product', 'start', 'end', 'lictype') class LicenseView(ReadWriteView): diff --git a/server/www/scripts/licenses/providers/license.js b/server/www/scripts/licenses/providers/license.js index 83236901a22..bfbb7432efc 100644 --- a/server/www/scripts/licenses/providers/license.js +++ b/server/www/scripts/licenses/providers/license.js @@ -6,7 +6,7 @@ angular.module('faradayApp') .factory('License', ['BASEURL', 'configSrv', '$http', '$q', function(BASEURL, configSrv, $http, $q) { function License(data) { - var APIURL = BASEURL + "_api/v2/"; + this.LICAPI = BASEURL + "_api/v2/licenses/"; var now = new Date(), date = now.getTime() / 1000.0; @@ -58,18 +58,13 @@ angular.module('faradayApp') var deferred = $q.defer(), self = this; - configSrv.promise - .then(function() { - var url = APIURL + "licenses/" + self._id; + var url = self.LICAPI + self._id; - $http.delete(url) - .then(function(resp) { - deferred.resolve(resp); - }, function(data, status, headers, config) { - deferred.reject("Unable to delete License from database. " + status); - }); - }, function(reason) { - deferred.reject(reason); + $http.delete(url) + .then(function(resp) { + deferred.resolve(resp); + }, function(data, status, headers, config) { + deferred.reject("Unable to delete License from database. " + status); }); return deferred.promise; @@ -80,7 +75,7 @@ angular.module('faradayApp') configSrv.promise .then(function() { - var url = APIURL + "licenses/" + self._id; + var url = self.LICAPI + self._id; $http.put(url, data) .then(function(res) { @@ -104,20 +99,13 @@ angular.module('faradayApp') delete this._id; delete this._rev; - configSrv.promise - .then(function() { - var url = APIURL + "licenses/"; - - $http.post(url, self) - .then(function(data) { - self._id = data._id; - self._rev = data.rev; - deferred.resolve(self); - }, function(res) { - deferred.reject("Unable to save the License. " + res.data.reason); - }); - }, function(reason) { - deferred.reject(reason); + $http.post(self.LICAPI, self) + .then(function(data) { + self._id = data._id; + self._rev = data.rev; + deferred.resolve(self); + }, function(res) { + deferred.reject("Unable to save the License. " + res.data.reason); }); return deferred.promise; diff --git a/server/www/scripts/licenses/providers/licenses.js b/server/www/scripts/licenses/providers/licenses.js index d9559094ca6..b42b537068d 100644 --- a/server/www/scripts/licenses/providers/licenses.js +++ b/server/www/scripts/licenses/providers/licenses.js @@ -78,30 +78,24 @@ angular.module('faradayApp') var deferred = $q.defer(), self = this; - configSrv.promise - .then(function() { - var url = APIURL + "licenses/"; - - $http.get(url) - .then(function(res) { - var data = res.data; - var licenses = []; - - if(data.hasOwnProperty("rows")) { - data.rows.forEach(function(row) { - try { - licenses.push(new License(row.doc)); - } catch(e) { - console.log(e.stack); - } - }); - } - - angular.copy(licenses, self.licenses); - deferred.resolve(licenses); - }, function(data, status, headers, config) { - deferred.reject("Unable to retrieve Licenses. " + status); - }); + var url = APIURL + "licenses/"; + + $http.get(url) + .then(function(res) { + var licenses = []; + res.data.forEach(function(row) { + try { + var new_lic = new License(row); + licenses.push(new_lic); + } catch(e) { + console.log(e.stack); + } + }); + + angular.copy(licenses, self.licenses); + deferred.resolve(licenses); + }, function(data, status, headers, config) { + deferred.reject("Unable to retrieve Licenses. " + status); }); return deferred.promise; From 081c3372014f61f4c23141c297ad6fd5b00d88f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 22 Dec 2017 16:02:56 -0300 Subject: [PATCH 0684/1506] Fix credentials view with many ugly hacks It's javscript after all.... :( --- server/api/modules/credentials.py | 1 + server/www/scripts/credentials/controllers/credentials.js | 2 +- server/www/scripts/credentials/partials/list.html | 8 ++++---- server/www/scripts/credentials/providers/credential.js | 3 ++- server/www/scripts/services/partials/list.html | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/server/api/modules/credentials.py b/server/api/modules/credentials.py index 7707c52d7db..f32693f1e83 100644 --- a/server/api/modules/credentials.py +++ b/server/api/modules/credentials.py @@ -114,6 +114,7 @@ def _envelope_list(self, objects, pagination_metadata=None): for credential in objects: credentials.append({ 'id': credential['_id'], + '_id': credential['_id'], 'key': credential['_id'], 'value': credential }) diff --git a/server/www/scripts/credentials/controllers/credentials.js b/server/www/scripts/credentials/controllers/credentials.js index 25585a0f5f7..b4bdb571176 100644 --- a/server/www/scripts/credentials/controllers/credentials.js +++ b/server/www/scripts/credentials/controllers/credentials.js @@ -49,7 +49,7 @@ angular.module('faradayApp') $scope.parentObject.nameService = response['data']['name']; // and also, load all host information needed. - var hostId = response['data']['_id'].split('.')[0]; + var hostId = response['data']['host_id']; ServerAPI.getObj($scope.workspace, hostId, 'hosts').then(function (response) { $scope.parentObject.nameHost = response['data']['name']; diff --git a/server/www/scripts/credentials/partials/list.html b/server/www/scripts/credentials/partials/list.html index f265b0bd350..1491cf190af 100644 --- a/server/www/scripts/credentials/partials/list.html +++ b/server/www/scripts/credentials/partials/list.html @@ -6,8 +6,8 @@

    - {{credentials.length > 0? credentials.length: 'Not found'}} credentials for {{parentObject.type}} {{parentObject.nameHost}}{{parentObject.nameService? '/' + parentObject.nameService: ''}} - {{credentials.length > 0? credentials.length: 'Not found'}} credentials for workspace {{workspace}} + {{credentials.length > 0? credentials.length: 'Not found'}} credentials for {{parentObject.parent_type}} {{parentObject.nameHost}}{{parentObject.nameService? '/' + parentObject.nameService: ''}} + {{credentials.length > 0? credentials.length: 'Not found'}} credentials for workspace {{workspace}} Select a workspace
    @@ -54,7 +54,7 @@

    - + Target @@ -75,7 +75,7 @@

    selection-model-selected-class="multi-selected" selection-model-on-change="selectedCredentials()"> - {{credential.target}} + {{credential.target}} {{credential.name}} {{credential.username}} {{credential.password}} diff --git a/server/www/scripts/credentials/providers/credential.js b/server/www/scripts/credentials/providers/credential.js index 7765cf99c35..5d9e8f65ab3 100644 --- a/server/www/scripts/credentials/providers/credential.js +++ b/server/www/scripts/credentials/providers/credential.js @@ -72,6 +72,7 @@ angular.module('faradayApp') update: function(ws) { var self = this; self.metadata = updateMetadata(self.metadata); + if(!self.id) self.id = self._id; return ServerAPI.updateCredential(ws, buildObjectServer(self)); }, @@ -83,7 +84,7 @@ angular.module('faradayApp') return ServerAPI.createCredential(ws, buildObjectServer(self)). then(function(credential_data) { - self._rev = credential_data.rev; + self._id = credential_data.data.id }); }, diff --git a/server/www/scripts/services/partials/list.html b/server/www/scripts/services/partials/list.html index 6a53f19d105..1a1ce72cc83 100644 --- a/server/www/scripts/services/partials/list.html +++ b/server/www/scripts/services/partials/list.html @@ -109,7 +109,7 @@

    No services found for {{host.ip}}

    - {{service.credentials}} + {{service.credentials}} From c9a0bf48570c7100e6c18ed1aafe9f34d64e4901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 22 Dec 2017 16:40:25 -0300 Subject: [PATCH 0685/1506] Fix tests broken after doing ugly hacks --- test_cases/test_api_credentials.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_cases/test_api_credentials.py b/test_cases/test_api_credentials.py index 88b9b976bc5..5dd737c008e 100644 --- a/test_cases/test_api_credentials.py +++ b/test_cases/test_api_credentials.py @@ -23,7 +23,7 @@ def test_get_list_backwards_compatibility(self, test_client, session, second_wor assert res.status_code == 200 assert 'rows' in res.json for vuln in res.json['rows']: - assert set([u'id', u'key', u'value']) == set(vuln.keys()) + assert set([u'_id', u'id', u'key', u'value']) == set(vuln.keys()) object_properties = [ u'_id', u'couchdbid', From b14fe129adaaea643addf54185c487b6937191bf Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 22 Dec 2017 16:50:59 -0300 Subject: [PATCH 0686/1506] Add test for vuln web find by id method --- test_cases/test_managers_mapper_manager.py | 98 +++++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/test_cases/test_managers_mapper_manager.py b/test_cases/test_managers_mapper_manager.py index af6daf49856..84fecb83952 100644 --- a/test_cases/test_managers_mapper_manager.py +++ b/test_cases/test_managers_mapper_manager.py @@ -7,7 +7,7 @@ from persistence.server.models import Host, Service, Vuln, Credential, VulnWeb import persistence.server.server from persistence.server.utils import get_host_properties, \ - get_service_properties, get_vuln_properties + get_service_properties, get_vuln_properties, get_vuln_web_properties from test_cases.factories import WorkspaceFactory, CommandFactory, HostFactory, \ ServiceFactory, VulnerabilityFactory, CredentialFactory, \ VulnerabilityWebFactory @@ -196,6 +196,102 @@ # the following dict is used to parametrize find (GET) tests GET_OBJ_DATA = { + VulnWeb: [ + { + 'factory': VulnerabilityWebFactory, + 'api_end_point': 'vulns', + 'get_properties_function': get_vuln_web_properties, + 'mocked_response': { + "website": "www.faradaysec.com", + "_rev": "", + "parent_type": "Service", + "owned": False, + "owner": "leonardo", + "query": "query", + "refs": [ + "ref" + ], + "impact": { + "accountability": False, + "integrity": False, + "confidentiality": False, + "availability": False + }, + "confirmed": True, + "severity": "high", + "service": { + "status": "open", + "protocol": "fdsf", + "name": "gfdgfd", + "summary": "(32/fdsf) gfdgfd", + "version": "", + "_id": 299, + "ports": "32" + }, + "policyviolations": [], + "params": "parameters", + "type": "VulnerabilityWeb", + "method": "GET", + "metadata": { + "update_time": 1513982385000, + "update_user": "", + "update_action": 0, + "creator": "", + "create_time": 1513982385000, + "update_controller_action": "", + "owner": "leonardo", + "command_id": None + }, + "status": "opened", + "issuetracker": {}, + "description": "Description", + "parent": 299, + "tags": [ ], + "easeofresolution": "simple", + "hostnames": [ + "macbookpro-c9a7" + ], + "pname": "pname", + "date": "2017-12-22T19:39:45.014203+00:00", + "path": "path", + "data": "data", + "response": "response", + "desc": "Description", + "name": "Vuln web", + "obj_id": "348", + "request": "request", + "_attachments": [], + "target": "172.16.138.1", + "_id": 348, + "resolution": "resolution" + }, + 'serialized_expected_results': { + 'confirmed': True, + 'data': 'data', + 'desc': 'Description', + 'description': 'Description', + 'name': 'Vuln web', + 'owned': False, + 'owner': 'leonardo', + 'parent': 299, + 'parent_type': 'Service', + 'params': 'parameters', + 'path': 'path', + 'policyviolations': [], + 'response': 'response', + 'method': 'GET', + 'refs': ['ref'], + 'request': 'request', + 'resolution': 'resolution', + 'severity': 'high', + 'status': 'opened', + 'website': 'www.faradaysec.com', + "query": "query", + "pname": "pname" + } + + } + ], Vuln: [ { 'factory': VulnerabilityFactory, From 65b88890c4fedbfc3808a75ddebc58ebf81bf5e2 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 22 Dec 2017 17:33:56 -0300 Subject: [PATCH 0687/1506] Update code for Note (now comments) --- persistence/server/models.py | 4 +- persistence/server/server.py | 7 +++- test_cases/factories.py | 6 +++ test_cases/test_managers_mapper_manager.py | 46 +++++++++++++++++++--- 4 files changed, 54 insertions(+), 9 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index af150154e39..021f86243db 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -423,12 +423,12 @@ def create_note(workspace_name, note, command_id): @_ignore_in_changes -def update_note(workspace_name, note): +def update_note(workspace_name, note, command_id): """Take a workspace_name and a Note object and update it in the sever. Return the server's json response as a dictionary. """ note_properties = get_note_properties(note) - return server.update_note(workspace_name, note.getID(), **note_properties) + return server.update_note(workspace_name, command_id, note.getID(), **note_properties) @_ignore_in_changes diff --git a/persistence/server/server.py b/persistence/server/server.py index 83ea994e4e3..839967b14f1 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -1242,8 +1242,9 @@ def create_note(workspace_name, command_id, object_type, object_id, name, text, type="Note", metadata=metadata) -def update_note(workspace_name, command_id, id, name, text, owned=None, owner="", - description="", metadata=None): +def update_note(workspace_name, command_id, id, name, text, + object_type, object_id, owned=None, + owner="", description="", metadata=None): """Updates a note. Args: @@ -1262,6 +1263,8 @@ def update_note(workspace_name, command_id, id, name, text, owned=None, owner="" """ return _update_in_server(workspace_name, id, + object_id=object_id, + object_type=object_type, command_id=command_id, name=name, description=description, diff --git a/test_cases/factories.py b/test_cases/factories.py index 75d3fecf777..3996091c481 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -358,3 +358,9 @@ class TagFactory(FaradayFactory): class Meta: model = Tag sqlalchemy_session = db.session + + +class NoteFactory(FaradayFactory): + + class Meta: + model = Comment diff --git a/test_cases/test_managers_mapper_manager.py b/test_cases/test_managers_mapper_manager.py index 84fecb83952..467afeb25bd 100644 --- a/test_cases/test_managers_mapper_manager.py +++ b/test_cases/test_managers_mapper_manager.py @@ -4,13 +4,14 @@ from managers.mapper_manager import MapperManager from persistence.server.server import _create_server_api_url -from persistence.server.models import Host, Service, Vuln, Credential, VulnWeb +from persistence.server.models import Host, Service, Vuln, Credential, VulnWeb, \ + Note import persistence.server.server from persistence.server.utils import get_host_properties, \ get_service_properties, get_vuln_properties, get_vuln_web_properties from test_cases.factories import WorkspaceFactory, CommandFactory, HostFactory, \ ServiceFactory, VulnerabilityFactory, CredentialFactory, \ - VulnerabilityWebFactory + VulnerabilityWebFactory, CommentFactory # OBJ_DATA is used to parametrize tests (https://docs.pytest.org/en/latest/parametrize.html) # We use it to test all model classes. @@ -190,11 +191,41 @@ 'username': 'username1', 'type': 'Cred', }, + }], + Note: [{ + 'factory': CommentFactory, + 'api_end_point': 'comment', + 'parent': { + 'parent_type': 'Host', + 'parent_factory': HostFactory + }, + 'data': { + '_id': 1, + 'text': 'Text from Note', + 'object_id': 10, + 'object_type': 'Host', + 'owned': False, + 'owner': 'leo', + 'password': 'testpass', + 'username': 'username1' + }, + 'expected_payload': { + 'command_id': None, + 'description': '', + 'name': None, + 'object_id': 10, + 'object_type': 'Host', + 'owned': False, + 'owner': 'leo', + 'text': 'Text from Note', + 'type': 'Note'}, }] } # the following dict is used to parametrize find (GET) tests +# mocked_response is the json returned by the api +# serialized_expected_results the expected serialized result. GET_OBJ_DATA = { VulnWeb: [ { @@ -485,7 +516,8 @@ def test_save_without_command(self, obj_class, many_test_data, monkeypatch, sess session.commit() test_data['data']['parent'] = parent.id test_data['data']['parent_type'] = test_data['parent']['parent_type'] - test_data['expected_payload']['parent'] = parent.id + if obj_class not in [Note]: + test_data['expected_payload']['parent'] = parent.id if obj_class in [Vuln, Credential]: test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] def mock_server_post(test_data, post_url, update=False, expected_response=201, **params): @@ -518,9 +550,11 @@ def test_save_with_command(self, obj_class, many_test_data, monkeypatch, session if test_data['parent']: parent = test_data['parent']['parent_factory'].create() session.commit() + test_data['data']['parent'] = parent.id test_data['data']['parent_type'] = test_data['parent']['parent_type'] - test_data['expected_payload']['parent'] = parent.id + if obj_class not in [Note]: + test_data['expected_payload']['parent'] = parent.id if obj_class in [Vuln, Credential]: test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] def mock_server_post(test_data, post_url, update=False, expected_response=201, **params): @@ -555,9 +589,11 @@ def test_update_without_command(self, obj_class, many_test_data, monkeypatch, se if test_data['parent']: parent = test_data['parent']['parent_factory'].create() session.commit() + test_data['data']['parent'] = parent.id test_data['data']['parent_type'] = test_data['parent']['parent_type'] - test_data['expected_payload']['parent'] = parent.id + if obj_class not in [Note]: + test_data['expected_payload']['parent'] = parent.id if obj_class in [Vuln, Credential]: test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] def mock_server_put(test_data, put_url, update=False, expected_response=201, **params): From dd8d4e586e43bf94f017e9b178238940feffe01a Mon Sep 17 00:00:00 2001 From: micabot Date: Fri, 22 Dec 2017 16:01:01 -0300 Subject: [PATCH 0688/1506] Fix License list and insert --- server/api/modules/licenses.py | 12 +++- server/www/scripts/app.js | 4 ++ .../www/scripts/licenses/providers/license.js | 68 +++++++------------ .../scripts/licenses/providers/licenses.js | 47 ++++++------- 4 files changed, 57 insertions(+), 74 deletions(-) diff --git a/server/api/modules/licenses.py b/server/api/modules/licenses.py index fbc59954b23..d3bd4f7499c 100644 --- a/server/api/modules/licenses.py +++ b/server/api/modules/licenses.py @@ -7,16 +7,22 @@ from marshmallow import fields from server.models import License -from server.api.base import ReadWriteView, AutoSchema +from server.api.base import ( + ReadWriteView, + AutoSchema, +) license_api = Blueprint('license_api', __name__) class LicenseSchema(AutoSchema): - + _id = fields.Integer(dump_only=True, attribute='id') + end = fields.DateTime(attribute='end_date') + lictype = fields.String(attribute='type') + start = fields.DateTime(attribute='start_date') class Meta: model = License - fields = ('id', 'product', 'start_date', 'end_date') + fields = ('_id', 'id', 'product', 'start', 'end', 'lictype') class LicenseView(ReadWriteView): diff --git a/server/www/scripts/app.js b/server/www/scripts/app.js index 5a64bfdfe17..e8f2824563d 100644 --- a/server/www/scripts/app.js +++ b/server/www/scripts/app.js @@ -16,6 +16,10 @@ var faradayApp = angular.module('faradayApp', ['ngRoute', 'selectionModel', 'ui. var url = window.location.origin + "/"; return url; })()) + .constant("APIURL", (function() { + var url = window.location.origin + "/_api/v2/"; + return url; + })()) .constant("EASEOFRESOLUTION", (function() { var resolutions = [ "trivial", diff --git a/server/www/scripts/licenses/providers/license.js b/server/www/scripts/licenses/providers/license.js index 83236901a22..60f6185af27 100644 --- a/server/www/scripts/licenses/providers/license.js +++ b/server/www/scripts/licenses/providers/license.js @@ -3,10 +3,9 @@ // See the file 'doc/LICENSE' for the license information angular.module('faradayApp') - .factory('License', ['BASEURL', 'configSrv', '$http', '$q', - function(BASEURL, configSrv, $http, $q) { + .factory('License', ['APIURL', '$http', '$q', + function(APIURL, $http, $q) { function License(data) { - var APIURL = BASEURL + "_api/v2/"; var now = new Date(), date = now.getTime() / 1000.0; @@ -58,18 +57,13 @@ angular.module('faradayApp') var deferred = $q.defer(), self = this; - configSrv.promise - .then(function() { - var url = APIURL + "licenses/" + self._id; + var url = APIURL + "licenses/" + self._id; - $http.delete(url) - .then(function(resp) { - deferred.resolve(resp); - }, function(data, status, headers, config) { - deferred.reject("Unable to delete License from database. " + status); - }); - }, function(reason) { - deferred.reject(reason); + $http.delete(url) + .then(function(resp) { + deferred.resolve(resp); + }, function(data, status, headers, config) { + deferred.reject("Unable to delete License from database. " + status); }); return deferred.promise; @@ -77,22 +71,15 @@ angular.module('faradayApp') update: function(data) { var deferred = $q.defer(), self = this; - - configSrv.promise - .then(function() { - var url = APIURL + "licenses/" + self._id; - - $http.put(url, data) - .then(function(res) { - self.set(data); - self._rev = res.rev; - deferred.resolve(self); - }, function(res) { - deferred.reject("Unable to update the License. " + res.data.reason); - }); - - }, function(reason) { - deferred.reject(reason); + var url = APIURL + "licenses/" + self._id; + + $http.put(url, data) + .then(function(res) { + self.set(data); + self._rev = res.rev; + deferred.resolve(self); + }, function(res) { + deferred.reject("Unable to update the License. " + res.data.reason); }); return deferred.promise; @@ -104,20 +91,13 @@ angular.module('faradayApp') delete this._id; delete this._rev; - configSrv.promise - .then(function() { - var url = APIURL + "licenses/"; - - $http.post(url, self) - .then(function(data) { - self._id = data._id; - self._rev = data.rev; - deferred.resolve(self); - }, function(res) { - deferred.reject("Unable to save the License. " + res.data.reason); - }); - }, function(reason) { - deferred.reject(reason); + $http.post(APIURL + "licenses/", self) + .then(function(data) { + self._id = data._id; + self._rev = data.rev; + deferred.resolve(self); + }, function(res) { + deferred.reject("Unable to save the License. " + res.data.reason); }); return deferred.promise; diff --git a/server/www/scripts/licenses/providers/licenses.js b/server/www/scripts/licenses/providers/licenses.js index d9559094ca6..bd970c7edbd 100644 --- a/server/www/scripts/licenses/providers/licenses.js +++ b/server/www/scripts/licenses/providers/licenses.js @@ -4,10 +4,9 @@ angular.module('faradayApp') .factory('licensesManager', - ['License', 'BASEURL', 'configSrv', '$http', '$q', - function(License, BASEURL, configSrv, $http, $q) { + ['License', 'APIURL', '$http', '$q', + function(License, APIURL, $http, $q) { var licensesManager = {}; - var APIURL = BASEURL + "_api/v2/"; licensesManager.licenses = []; @@ -78,30 +77,24 @@ angular.module('faradayApp') var deferred = $q.defer(), self = this; - configSrv.promise - .then(function() { - var url = APIURL + "licenses/"; - - $http.get(url) - .then(function(res) { - var data = res.data; - var licenses = []; - - if(data.hasOwnProperty("rows")) { - data.rows.forEach(function(row) { - try { - licenses.push(new License(row.doc)); - } catch(e) { - console.log(e.stack); - } - }); - } - - angular.copy(licenses, self.licenses); - deferred.resolve(licenses); - }, function(data, status, headers, config) { - deferred.reject("Unable to retrieve Licenses. " + status); - }); + var url = APIURL + "licenses/"; + + $http.get(url) + .then(function(res) { + var licenses = []; + res.data.forEach(function(row) { + try { + var new_lic = new License(row); + licenses.push(new_lic); + } catch(e) { + console.log(e.stack); + } + }); + + angular.copy(licenses, self.licenses); + deferred.resolve(licenses); + }, function(data, status, headers, config) { + deferred.reject("Unable to retrieve Licenses. " + status); }); return deferred.promise; From 7681a363309de14fa6b65189b48980e004fffd24 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 22 Dec 2017 18:03:17 -0300 Subject: [PATCH 0689/1506] Add test and support for commands --- persistence/server/models.py | 5 +- test_cases/test_managers_mapper_manager.py | 84 ++++++++++++++++++---- 2 files changed, 74 insertions(+), 15 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index 021f86243db..3a2cb441487 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -1298,7 +1298,7 @@ class Command: class_signature = 'CommandRunInformation' def __init__(self, command, workspace_name): self._workspace_name = workspace_name - self.id = command['id'] + self.id = command.get('id', None) or command.get('_id', None) self.command = command['command'] self.duration = command['duration'] self.hostname = command['hostname'] @@ -1308,6 +1308,9 @@ def __init__(self, command, workspace_name): self.user = command['user'] self.workspace = command['workspace'] + def getID(self): + return self.id + def getCommand(self): return self.command diff --git a/test_cases/test_managers_mapper_manager.py b/test_cases/test_managers_mapper_manager.py index 467afeb25bd..5f3245e90ca 100644 --- a/test_cases/test_managers_mapper_manager.py +++ b/test_cases/test_managers_mapper_manager.py @@ -4,14 +4,32 @@ from managers.mapper_manager import MapperManager from persistence.server.server import _create_server_api_url -from persistence.server.models import Host, Service, Vuln, Credential, VulnWeb, \ - Note +from persistence.server.models import ( + Host, + Service, + Vuln, + Credential, + VulnWeb, + Note, + Command, +) import persistence.server.server -from persistence.server.utils import get_host_properties, \ - get_service_properties, get_vuln_properties, get_vuln_web_properties -from test_cases.factories import WorkspaceFactory, CommandFactory, HostFactory, \ - ServiceFactory, VulnerabilityFactory, CredentialFactory, \ - VulnerabilityWebFactory, CommentFactory +from persistence.server.utils import ( + get_host_properties, + get_service_properties, + get_vuln_properties, + get_vuln_web_properties +) +from test_cases.factories import ( + WorkspaceFactory, + CommandFactory, + HostFactory, + ServiceFactory, + VulnerabilityFactory, + CredentialFactory, + VulnerabilityWebFactory, + CommentFactory, + EmptyCommandFactory) # OBJ_DATA is used to parametrize tests (https://docs.pytest.org/en/latest/parametrize.html) # We use it to test all model classes. @@ -195,6 +213,7 @@ Note: [{ 'factory': CommentFactory, 'api_end_point': 'comment', + # parent not used 'parent': { 'parent_type': 'Host', 'parent_factory': HostFactory @@ -219,6 +238,37 @@ 'owner': 'leo', 'text': 'Text from Note', 'type': 'Note'}, + }], + Command: [{ + 'factory': EmptyCommandFactory, + 'api_end_point': 'commands', + # parent not used + 'parent': { + 'parent_type': 'Host', + 'parent_factory': HostFactory + }, + 'data': { + 'itime': 1513959206, + 'ip': "192.168.124.1", + 'hostname': "mandarina", + 'command': "Import Nexpose:", + 'user': "leonardo", + 'workspace': "airbnb", + 'duration': 0.164561, + 'params': "/home/lcubo/.faraday/report/airbnb/nexpose_5.7.19_Website_PT.xml", + '_id': 280 + }, + 'expected_payload':{ + 'command': 'Import Nexpose:', + 'duration': 0.164561, + 'hostname': 'mandarina', + 'ip': '192.168.124.1', + 'itime': 1513959206, + 'params': '/home/lcubo/.faraday/report/airbnb/nexpose_5.7.19_Website_PT.xml', + 'type': 'CommandRunInformation', + 'user': 'leonardo', + 'workspace': u'test' + } }] } @@ -516,7 +566,7 @@ def test_save_without_command(self, obj_class, many_test_data, monkeypatch, sess session.commit() test_data['data']['parent'] = parent.id test_data['data']['parent_type'] = test_data['parent']['parent_type'] - if obj_class not in [Note]: + if obj_class not in [Note, Command]: test_data['expected_payload']['parent'] = parent.id if obj_class in [Vuln, Credential]: test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] @@ -525,8 +575,9 @@ def mock_server_post(test_data, post_url, update=False, expected_response=201, * _create_server_api_url(), test_data['api_end_point']) assert expected_response == 201 assert update == False - metadata = params.pop('metadata') - assert metadata['owner'] == test_data['expected_payload']['owner'] + if obj_class not in [Command]: + metadata = params.pop('metadata') + assert metadata['owner'] == test_data['expected_payload']['owner'] assert params == test_data['expected_payload'] return { 'id': 1, @@ -540,6 +591,8 @@ def mock_server_post(test_data, post_url, update=False, expected_response=201, * @pytest.mark.parametrize("obj_class, many_test_data", OBJ_DATA.items()) def test_save_with_command(self, obj_class, many_test_data, monkeypatch, session): + if obj_class == Command: + return workspace = WorkspaceFactory.create(name='test') command = CommandFactory.create(workspace=workspace) session.commit() @@ -592,7 +645,7 @@ def test_update_without_command(self, obj_class, many_test_data, monkeypatch, se test_data['data']['parent'] = parent.id test_data['data']['parent_type'] = test_data['parent']['parent_type'] - if obj_class not in [Note]: + if obj_class not in [Note, Command]: test_data['expected_payload']['parent'] = parent.id if obj_class in [Vuln, Credential]: test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] @@ -600,9 +653,10 @@ def mock_server_put(test_data, put_url, update=False, expected_response=201, **p assert put_url == '{0}/ws/test/{1}/{2}/'.format(_create_server_api_url(), test_data['api_end_point'], test_data['id']) assert expected_response == 200 assert update == False - metadata = params.pop('metadata') - assert metadata['owner'] == test_data['expected_payload']['owner'] - params.pop('command_id') + if obj_class not in [Command]: + metadata = params.pop('metadata') + assert metadata['owner'] == test_data['expected_payload']['owner'] + params.pop('command_id', None) test_data['expected_payload'].pop('command_id', None) assert params == test_data['expected_payload'] @@ -621,6 +675,8 @@ def mock_server_put(test_data, put_url, update=False, expected_response=201, **p @pytest.mark.parametrize("obj_class, many_test_data", OBJ_DATA.items()) def test_update_with_command(self, obj_class, many_test_data, monkeypatch, session): + if obj_class in [Command]: + return session.commit() workspace = WorkspaceFactory.create(name='test') command = CommandFactory.create(workspace=workspace) From a202eb6276f73b96b8188c7b6453ce36353d27b0 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 26 Dec 2017 12:35:20 -0300 Subject: [PATCH 0690/1506] Fix get credential --- persistence/server/server.py | 2 +- test_cases/test_managers_mapper_manager.py | 49 ++++++++++++++++++++-- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/persistence/server/server.py b/persistence/server/server.py index 839967b14f1..785f70b8539 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -282,7 +282,7 @@ def _get_raw_notes(workspace_name, **params): def _get_raw_credentials(workspace_name, **params): """Take a workspace name and an arbitrary number of params and return a dictionary with the credentials table.""" - request_url = _create_server_get_url(workspace_name, 'credentials') + request_url = _create_server_get_url(workspace_name, 'credential', params.get('id', None)) return _get(request_url, **params) diff --git a/test_cases/test_managers_mapper_manager.py b/test_cases/test_managers_mapper_manager.py index 5f3245e90ca..7f37253f31d 100644 --- a/test_cases/test_managers_mapper_manager.py +++ b/test_cases/test_managers_mapper_manager.py @@ -18,8 +18,8 @@ get_host_properties, get_service_properties, get_vuln_properties, - get_vuln_web_properties -) + get_vuln_web_properties, + get_credential_properties) from test_cases.factories import ( WorkspaceFactory, CommandFactory, @@ -537,7 +537,50 @@ } } - ] + ], + Credential: [{ + 'factory': ServiceFactory, + 'api_end_point': 'credential', + 'parent': { + 'parent_type': 'Host', + 'parent_factory': HostFactory + }, + 'get_properties_function': get_credential_properties, + 'mocked_response': { + "username": "user1", + "password": "secretpassword", + "description": "Credential obtained using hashcat", + "couchdbid": "", + 'parent': 64, + 'parent_type': 'Host', + "_rev": "", + "metadata": { + "update_time": 1514312337000, + "update_user": "", + "update_action": 0, + "creator": "", + "create_time": 1514312336000, + "update_controller_action": "", + "owner": "leonardo", + "command_id": None + }, + "owned": False, + "owner": "leonardo", + "_id": 2, + "id": 2, + "name": "dsds" + }, + 'serialized_expected_results': { + 'description': 'Credential obtained using hashcat', + 'name': 'dsds', + 'owned': False, + 'owner': 'leonardo', + 'parent': 64, + 'parent_type': 'Host', + 'password': 'secretpassword', + 'username': 'user1' + } + }] } class MockResponse: From c13294061326632a5c6145335c1c4c4257fa3375 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 26 Dec 2017 14:12:20 -0300 Subject: [PATCH 0691/1506] Add support for commands --- persistence/server/server.py | 4 +- test_cases/test_managers_mapper_manager.py | 76 +++++++++++++++++++++- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/persistence/server/server.py b/persistence/server/server.py index 785f70b8539..1d973bffb80 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -275,7 +275,7 @@ def _get_raw_services(workspace_name, **params): def _get_raw_notes(workspace_name, **params): """Take a workspace name and an arbitrary number of params and return a dictionary with the notes table.""" - request_url = _create_server_get_url(workspace_name, 'notes') + request_url = _create_server_get_url(workspace_name, 'comment', params.get('id', None)) return _get(request_url, **params) @@ -287,7 +287,7 @@ def _get_raw_credentials(workspace_name, **params): def _get_raw_commands(workspace_name, **params): - request_url = _create_server_get_url(workspace_name, 'commands') + request_url = _create_server_get_url(workspace_name, 'commands', params.get('id', None)) return _get(request_url, **params) diff --git a/test_cases/test_managers_mapper_manager.py b/test_cases/test_managers_mapper_manager.py index 7f37253f31d..cd514d8fcdf 100644 --- a/test_cases/test_managers_mapper_manager.py +++ b/test_cases/test_managers_mapper_manager.py @@ -19,7 +19,7 @@ get_service_properties, get_vuln_properties, get_vuln_web_properties, - get_credential_properties) + get_credential_properties, get_command_properties) from test_cases.factories import ( WorkspaceFactory, CommandFactory, @@ -580,6 +580,77 @@ 'password': 'secretpassword', 'username': 'user1' } + }], + Credential: [{ + 'factory': ServiceFactory, + 'api_end_point': 'credential', + 'parent': { + 'parent_type': 'Host', + 'parent_factory': HostFactory + }, + 'get_properties_function': get_credential_properties, + 'mocked_response': { + "username": "user1", + "password": "secretpassword", + "description": "Credential obtained using hashcat", + "couchdbid": "", + 'parent': 64, + 'parent_type': 'Host', + "_rev": "", + "metadata": { + "update_time": 1514312337000, + "update_user": "", + "update_action": 0, + "creator": "", + "create_time": 1514312336000, + "update_controller_action": "", + "owner": "leonardo", + "command_id": None + }, + "owned": False, + "owner": "leonardo", + "_id": 2, + "id": 2, + "name": "dsds" + }, + 'serialized_expected_results': { + 'description': 'Credential obtained using hashcat', + 'name': 'dsds', + 'owned': False, + 'owner': 'leonardo', + 'parent': 64, + 'parent_type': 'Host', + 'password': 'secretpassword', + 'username': 'user1' + } + }], + Command: [{ + 'factory': ServiceFactory, + 'api_end_point': 'commands', + 'parent': { + 'parent_type': 'Host', + 'parent_factory': HostFactory + }, + 'get_properties_function': get_command_properties, + 'mocked_response': { + "itime": 1513365824, + "ip": "192.168.20.53", + "hostname": "mandarina", + "command": "Import Nessus:", + "user": "lcubo", + "workspace": "dsadsa", + "duration": "In progress", + "params": "/home/lcubo/.faraday/report/dsadsa/nessus_report_Remote.nessus", + "_id": 1 + }, + 'serialized_expected_results': { + 'command': 'Import Nessus:', + 'duration': 'In progress', + 'hostname': 'mandarina', + 'ip': '192.168.20.53', + 'itime': 1513365824, + 'params': '/home/lcubo/.faraday/report/dsadsa/nessus_report_Remote.nessus', + 'user': 'lcubo'} }] } @@ -779,5 +850,6 @@ def mock_unsafe_io_with_server(host, test_data, server_io_function, server_expec monkeypatch.setattr(persistence.server.server, '_unsafe_io_with_server', partial(mock_unsafe_io_with_server, persisted_obj, test_data)) found_obj = mapper_manager.find(obj_class.class_signature, persisted_obj.id) serialized_obj = test_data['get_properties_function'](found_obj) - metadata = serialized_obj.pop('metadata') + if obj_class not in [Command]: + metadata = serialized_obj.pop('metadata') assert serialized_obj == test_data['serialized_expected_results'] From a9931791c561e7c25049a658dee17ee9b8e3dbf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 26 Dec 2017 16:38:03 -0300 Subject: [PATCH 0692/1506] Fix vuln updates Unskip update tests, allow null ease of resolutions, fix uniqueness validation (I had to touch server/api/base.py fot this last thing). I also commented an assertion in server/api/base.py that checked that there were no new elements in the session, I'm not sure why I added this and it was causing problems with created policy violations when updating vulns --- server/api/base.py | 8 +++- server/api/modules/vulns.py | 10 +++-- test_cases/test_api_vulnerability.py | 57 ++++++++++++++++------------ 3 files changed, 46 insertions(+), 29 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 2200c09d54a..9362f6a2007 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -241,7 +241,9 @@ def _validate_uniqueness(self, obj, object_id=None): query = query.filter(primary_key_field != object_id) for field_names in self.unique_fields: for field_name in field_names: - field = getattr(self.model_class, field_name) + # Use type(obj) instead of self.model_class to be + # compatible with polymorphic obejcts (e.g. Vulnerability) + field = getattr(type(obj), field_name) value = getattr(obj, field_name) query = query.filter( field == value) @@ -506,7 +508,9 @@ class UpdateWorkspacedMixin(UpdateMixin, CommandMixing): """Add PUT // route""" def _perform_update(self, object_id, obj, workspace_name): - assert not db.session.new + # # Make sure that if I created new objects, I had properly commited them + # assert not db.session.new + with db.session.no_autoflush: obj.workspace = self._get_workspace(workspace_name) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 7010bbc8dfd..7d887bd765b 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -93,7 +93,10 @@ class VulnerabilitySchema(AutoSchema): fields.String(), required=True) tags = PrimaryKeyRelatedField('name', dump_only=True, many=True) - easeofresolution = fields.String(attribute='ease_of_resolution', validate=OneOf(Vulnerability.EASE_OF_RESOLUTIONS),) + easeofresolution = fields.String( + attribute='ease_of_resolution', + validate=OneOf(Vulnerability.EASE_OF_RESOLUTIONS), + allow_none=True) hostnames = PrimaryKeyRelatedField('name', many=True, dump_only=True) service = fields.Nested(ServiceSchema(only=[ '_id', 'ports', 'status', 'protocol', 'name', 'version', 'summary' @@ -303,13 +306,14 @@ class VulnerabilityView(PaginatedMixin, filterset_class = VulnerabilityFilterSet unique_fields_by_class = { 'Vulnerability': [('name', 'description', 'host_id', 'service_id')], - 'VulnerabilityWeb': [('name', 'description', 'service_id', 'method', 'parameter_name', 'path', 'website')], + 'VulnerabilityWeb': [('name', 'description', 'service_id', 'method', + 'parameter_name', 'path', 'website')], } model_class_dict = { 'Vulnerability': Vulnerability, 'VulnerabilityWeb': VulnerabilityWeb, - 'VulnerabilityGeneric': VulnerabilityGeneric, # For listing objects + 'VulnerabilityGeneric': VulnerabilityGeneric, # For listing objects } schema_class_dict = { 'Vulnerability': VulnerabilitySchema, diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index bb55b4e1ef9..47235c8bc4c 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -367,7 +367,7 @@ def _create_put_data(self, "confirmed": True, "data":"", "desc": desc, - "easeofresolution":"", + "easeofresolution":None, "impact": impact, "name": name, "owned": False, @@ -384,7 +384,6 @@ def _create_put_data(self, return raw_data - @pytest.mark.skip(reason="Update not yet implemented") def test_update_vuln_from_open_to_close(self, test_client, session, host_with_hostnames): vuln = self.factory.create(status='open', host=host_with_hostnames, service=None, workspace=host_with_hostnames.workspace) session.commit() @@ -673,6 +672,27 @@ def test_create_vuln_with_invalid_ease_of_resolution(self, assert res.json['messages'].keys() == ['easeofresolution'] assert res.json['messages']['easeofresolution'][0] == u'Not a valid choice.' + def test_create_vuln_with_null_ease_of_resolution(self, + host_with_hostnames, + test_client, + session): + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='Vulnerability', + parent_id=host_with_hostnames.id, + parent_type='Host', + refs=[], + policyviolations=[], + easeofresolution=None, + ) + ws_name = host_with_hostnames.workspace.name + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), + data=raw_data) + assert res.status_code == 201, (res.status_code, res.data) + created_vuln = Vulnerability.query.get(res.json['_id']) + assert created_vuln.ease_of_resolution is None + def test_count_confirmed(self, test_client, session): for i, vuln in enumerate(self.objects[:3]): vuln.confirmed = True @@ -880,9 +900,6 @@ def test_create_with_parent_of_other_workspace( assert res.status_code == 400 assert 'Parent id not found' in res.data - @pytest.mark.skip(reason="Update not implemented yet. Also it won't work " - "instantly because it is a copy paste of other test. " - "It should be changed") @pytest.mark.parametrize("parent_type, parent_factory", [ ("Host", HostFactory), ("Service", ServiceFactory), @@ -891,26 +908,18 @@ def test_update_with_parent_of_other_workspace( self, parent_type, parent_factory, test_client, session, second_workspace, credential_factory): parent = parent_factory.create(workspace=second_workspace) - if parent_type == 'Host': - credential = credential_factory.create( - host=HostFactory.create(workspace=self.workspace), - service=None, - workspace=self.workspace) - else: - credential = credential_factory.create( - host=None, - service=ServiceFactory.create(workspace=self.workspace), - workspace=self.workspace) session.commit() assert parent.workspace_id != self.workspace.id - data = { - "username": "admin", - "password": "admin", - "name": "test", - "parent_type": parent_type, - "parent": parent.id - } - res = test_client.put(self.url(credential), data=data) + data = self._create_put_data( + name='New name', + desc='New desc', + status='closed', + parent=parent.id, + parent_type=parent_type, + refs=['ref1'], + policy_violations=['pv0'] + ) + res = test_client.put(self.url(self.first_object), data=data) assert res.status_code == 400 assert 'Parent id not found' in res.data @@ -965,4 +974,4 @@ def test_create_webvuln_multiple_times_returns_conflict(self, host_with_hostname assert res.status_code == 201 res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) - assert res.status_code == 409 \ No newline at end of file + assert res.status_code == 409 From 6658af3741ef890d340da58844ecb4cc78e386ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 26 Dec 2017 16:50:33 -0300 Subject: [PATCH 0693/1506] Improve validate_uniqueness overriding in vulns API It was setting attributes in the class instance that depended on a property of the request. This is a bad pattern, since view instances are reused and not instanced once per request. Tests success story: I first forgot to replace `self.unique_fields` for `unique_fields` in the for loop. Before committing I ran the tests and found that the conflicts on creation were not happening, and I noted that I almost introduce a regression in the code, but the test cases catched me before doing it. --- server/api/base.py | 8 ++++++-- server/api/modules/vulns.py | 5 +++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 9362f6a2007..375f38978a0 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -229,7 +229,7 @@ def _set_schema_context(self, context, **kwargs): context.update(kwargs) return context - def _validate_uniqueness(self, obj, object_id=None): + def _validate_uniqueness(self, obj, object_id=None, unique_fields=None): # TODO: Use implementation of GenericView assert obj.workspace is not None, "Object must have a " \ "workspace attribute set to call _validate_uniqueness" @@ -239,7 +239,11 @@ def _validate_uniqueness(self, obj, object_id=None): # The object already exists in DB, we want to fetch an object # different to this one but with the same unique field query = query.filter(primary_key_field != object_id) - for field_names in self.unique_fields: + if unique_fields is None: + # It is usually None, but in some case the child class may want to + # override it based on some condition dependent on the request + unique_fields = self.unique_fields + for field_names in unique_fields: for field_name in field_names: # Use type(obj) instead of self.model_class to be # compatible with polymorphic obejcts (e.g. Vulnerability) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 7d887bd765b..d5201b59cdd 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -321,8 +321,9 @@ class VulnerabilityView(PaginatedMixin, } def _validate_uniqueness(self, obj, object_id=None): - self.unique_fields = self.unique_fields_by_class[obj.__class__.__name__] - super(VulnerabilityView, self)._validate_uniqueness(obj, object_id) + unique_fields = self.unique_fields_by_class[obj.__class__.__name__] + super(VulnerabilityView, self)._validate_uniqueness( + obj, object_id, unique_fields) def _perform_create(self, data, **kwargs): data = self._parse_data(self._get_schema_instance(kwargs), From a8072fabb505a3b14dc1fc633d9bc6daca68d70c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 26 Dec 2017 16:51:06 -0300 Subject: [PATCH 0694/1506] Cleanup modelactions and fix some problems with _id and id properties * modelactions was duplicated. To maintain code simplier one of them was removed. plugins/modelactions.py was created for allowing multiple zsh terminals on commit 41de15430 * Sometimes faraday-api returns on id (on new objects like comment) and this was breaking model relationships on plugins side. --- managers/mapper_manager.py | 6 ++--- model/controller.py | 2 ++ persistence/server/models.py | 4 ++-- plugins/controller.py | 2 +- plugins/modelactions.py | 43 ------------------------------------ plugins/plugin.py | 3 ++- server/commands/reports.py | 8 ++----- 7 files changed, 12 insertions(+), 56 deletions(-) delete mode 100644 plugins/modelactions.py diff --git a/managers/mapper_manager.py b/managers/mapper_manager.py index 274ca24379b..9096bce9275 100644 --- a/managers/mapper_manager.py +++ b/managers/mapper_manager.py @@ -26,9 +26,9 @@ def createMappers(self, workpace_name): def save(self, obj, command_id=None): saved_raw_obj = create_object(self.workspace_name, obj.class_signature, obj, command_id) - if '_id' in saved_raw_obj: - return saved_raw_obj['_id'] - return False + if '_id' in saved_raw_obj or 'id' in saved_raw_obj: + return saved_raw_obj.get('_id', None) or saved_raw_obj['id'] + raise RuntimeError('Could not retrieve id from server.') def update(self, obj, command_id=None): if update_object(self.workspace_name, obj.class_signature, obj, command_id): diff --git a/model/controller.py b/model/controller.py index 44638e18994..5007093b5c5 100644 --- a/model/controller.py +++ b/model/controller.py @@ -64,6 +64,8 @@ class modelactions: DELCRED = 2050 PLUGINSTART = 3000 PLUGINEND = 3001 + LOG = 3002 + DEVLOG = 3003 __descriptions = { ADDHOST: "ADDHOST", diff --git a/persistence/server/models.py b/persistence/server/models.py index 3a2cb441487..0f2d997cd78 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -701,8 +701,8 @@ class ModelBase(object): """ def __init__(self, obj, workspace_name): self._workspace_name = workspace_name - self._server_id = obj.get('_id', '') - self.id = obj.get('id', None) + self._server_id = obj.get('_id', None) + self.id = obj.get('id', self._server_id) self.name = obj.get('name') self.description = obj.get('description', "") self.owned = obj.get('owned', False) diff --git a/plugins/controller.py b/plugins/controller.py index 890d2753cc5..75aade523eb 100644 --- a/plugins/controller.py +++ b/plugins/controller.py @@ -21,7 +21,7 @@ from plugins.plugin import PluginProcess import model.api from model.commands_history import CommandRunInformation -from plugins.modelactions import modelactions +from model.controller import modelactions from utils.logs import getLogger from config.globals import ( diff --git a/plugins/modelactions.py b/plugins/modelactions.py deleted file mode 100644 index 402f9d56108..00000000000 --- a/plugins/modelactions.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -''' -Faraday Penetration Test IDE -Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) -See the file 'doc/LICENSE' for the license information - -''' - - -class modelactions: - ADDHOST = 2000 - CADDHOST = 2001 - ADDSERVICEAPP = 2005 - CADDSERVICEAPP = 2007 - CADDSERVICEHOST = 2008 - ADDSERVICEHOST = 20008 - ADDAPPLICATION = 2009 - CADDAPPLICATION = 2010 - ADDVULNAPP = 2015 - CADDVULNAPP = 2016 - ADDVULNHOST = 2017 - CADDVULNHOST = 2018 - ADDVULNSRV = 2019 - CADDVULNSRV = 2020 - ADDNOTEAPP = 2023 - CADDNOTEAPP = 2024 - ADDNOTEHOST = 2025 - CADDNOTEHOST = 2026 - ADDNOTESRV = 2027 - CADDNOTESRV = 2028 - CADDNOTEVULN = 2030 - CADDNOTEVULN = 2031 - LOG = 2032 - DEVLOG = 2033 - ADDCREDSRV = 2035 - CADDCREDSRV = 2036 - ADDVULNWEBSRV = 2037 - CADDVULNWEBSRV = 2038 - ADDNOTENOTE = 2039 - PLUGINSTART = 3000 - PLUGINEND = 3001 diff --git a/plugins/plugin.py b/plugins/plugin.py index c8016a89b97..7673f562eea 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -28,7 +28,8 @@ Credential, Note, ) -from plugins.modelactions import modelactions +from model.controller import modelactions +#from plugins.modelactions import modelactions from config.configuration import getInstanceConfiguration CONF = getInstanceConfiguration() diff --git a/server/commands/reports.py b/server/commands/reports.py index bb46c703347..31c14f7013e 100644 --- a/server/commands/reports.py +++ b/server/commands/reports.py @@ -18,9 +18,8 @@ def import_external_reports(workspace_name=None): - - plugin_manager = PluginManager( - os.path.join(CONF.getConfigPath(), "plugins")) + plugins_path = os.path.join(CONF.getConfigPath(), "plugins") + plugin_manager = PluginManager(plugins_path) mappers_manager = MapperManager() if workspace_name: @@ -36,9 +35,6 @@ def process_workspaces(mappers_manager, plugin_manager, query): report_managers = [] controllers = [] for workspace in query.all(): - if workspace.name != 'airbnb': - continue - print(workspace.name) pending_actions = Queue() plugin_controller = PluginController( 'PluginController', From 98abebcf2bf02ce11bf0787ba9728612bb56c536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 26 Dec 2017 17:49:01 -0300 Subject: [PATCH 0695/1506] Fix bug creating host vulnerabilities in the Web UI undefined winned again :( and make an if condition never triggered, that set the parent to host --- server/www/scripts/statusReport/controllers/modalNew.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/statusReport/controllers/modalNew.js b/server/www/scripts/statusReport/controllers/modalNew.js index 8f44d7d87ab..6f11162aab8 100644 --- a/server/www/scripts/statusReport/controllers/modalNew.js +++ b/server/www/scripts/statusReport/controllers/modalNew.js @@ -139,7 +139,7 @@ angular.module('faradayApp') vm.data.parents = []; parents.forEach(function(parent) { var parent_type = "Service"; - if (Host.prototype.isPrototypeOf(vm.data.parents[0])) { + if (Host.prototype.isPrototypeOf(parents[0])) { parent_type = "Host"; } vm.data.parents.push({parent_id: parent._id, type:parent_type}); From f795cb3932136f286ae9814199e3a9a09e269f6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 26 Dec 2017 18:18:14 -0300 Subject: [PATCH 0696/1506] Fix vuln web with host parent error message of the web ui. In integration, when the user set type to vuln web and selected one or more hosts as target, a message "Error: Trying to create a Web Vulnerability with one or more Host parents. Please deselect the Host targets or click here to deselect all targets" was shown. The migration changes broke it because the api didn't return the object type for host and services. It is fixed now. Another victory of undefined, that made the browser don't throw any exceptions or warnings. --- server/api/modules/hosts.py | 3 ++- server/api/modules/services.py | 3 ++- test_cases/test_api_hosts.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index b3226d3de59..5902fd51be5 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -43,13 +43,14 @@ class HostSchema(AutoSchema): dump_only=True, # Only for now default=[]) metadata = SelfNestedField(MetadataSchema()) + type = fields.Function(lambda obj: 'Host') class Meta: model = Host fields = ('id', '_id', '_rev', 'ip', 'description', 'credentials', 'default_gateway', 'metadata', 'name', 'os', 'owned', 'owner', 'services', 'vulns', - 'hostnames' + 'hostnames', 'type', ) diff --git a/server/api/modules/services.py b/server/api/modules/services.py index 750c5df72ee..c258bcf2374 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -40,6 +40,7 @@ class ServiceSchema(AutoSchema): vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) credentials = fields.Integer(attribute='credentials_count', dump_only=True) metadata = SelfNestedField(MetadataSchema()) + type = fields.Function(lambda obj: 'Host') def load_port(self, value): # TODO migration: handle empty list and not numeric value @@ -76,7 +77,7 @@ def post_load_parent(self, data): class Meta: model = Service - fields = ('id', '_id', 'status', 'parent', + fields = ('id', '_id', 'status', 'parent', 'type', 'protocol', 'description', '_rev', 'owned', 'owner', 'credentials', 'vulns', 'name', 'version', '_id', 'port', 'ports', diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 5763ee04e81..1ef4194d6e8 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -385,6 +385,7 @@ def test_update_host(self, test_client, session): updated_host = Host.query.filter_by(id=host.id).first() assert res.json == { u'_id': host.id, + u'type': 'Host', u'_rev': u'', u'credentials': 0, u'default_gateway': None, @@ -502,4 +503,4 @@ def test_create_host_cant_assign_command_from_another_workspace(self, test_clien assert res.status_code == 400 assert res.json == {u'message': u'Command not found.'} - assert len(command.command_objects) == 0 \ No newline at end of file + assert len(command.command_objects) == 0 From fa82df3f340dc5eb95b3464ee9c42bd2fd2f2448 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 27 Dec 2017 12:47:05 -0300 Subject: [PATCH 0697/1506] Remove ignore_nplusone from a vuln api test --- test_cases/test_api_vulnerability.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 47235c8bc4c..2900e0a242c 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -756,7 +756,6 @@ def test_target(self, test_client, session, second_workspace, assert v['target'] == host.ip @pytest.mark.usefixtures('mock_envelope_list') - @pytest.mark.usefixtures('ignore_nplusone') def test_filter_by_command_id(self, test_client, session, second_workspace, workspace, From 80e33327b113e2630b17964594376e353ed9c889 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 27 Dec 2017 13:40:17 -0300 Subject: [PATCH 0698/1506] Minor fixes. owned is not a credential property. set some default values to empty string --- persistence/server/models.py | 4 ++-- plugins/plugin.py | 2 +- server/api/modules/credentials.py | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index 0f2d997cd78..d4601d81a55 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -1074,9 +1074,9 @@ def __init__(self, vuln_web, workspace_name): self.website = vuln_web.get('website') self.request = vuln_web.get('request') self.response = vuln_web.get('response') - self.method = vuln_web.get('method') + self.method = vuln_web.get('method') or '' self.pname = vuln_web.get('pname') - self.params = vuln_web.get('params') + self.params = vuln_web.get('params') or '' self.query = vuln_web.get('query') self.resolution = vuln_web.get('resolution') self.attachments = vuln_web.get('_attachments') diff --git a/plugins/plugin.py b/plugins/plugin.py index 7673f562eea..07d7689a9b0 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -321,7 +321,7 @@ def createAndAddCredToService(self, host_id, service_id, username, cred_obj = model.common.factory.createModelObject( Credential.class_signature, - username, password=password, parent_id=service_id) + username, password=password, parent_id=service_id, parent_type='Service') cred_obj._metadata.creator = self.id self.__addPendingAction(modelactions.ADDCREDSRV, cred_obj) diff --git a/server/api/modules/credentials.py b/server/api/modules/credentials.py index c9fba8a5c74..4e80ffd6cbd 100644 --- a/server/api/modules/credentials.py +++ b/server/api/modules/credentials.py @@ -20,7 +20,6 @@ class CredentialSchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') _rev = fields.String(default='', dump_only=True) - owned = fields.Boolean(default=False) owner = fields.String(dump_only=True, attribute='creator.username', default='') username = fields.String(default='') password = fields.String(default='') @@ -50,7 +49,7 @@ class Meta: fields = ('id', '_id', "_rev", 'parent', 'username', 'description', 'name', 'password', - 'owner', 'owned', 'couchdbid', + 'owner', 'couchdbid', 'parent', 'parent_type', 'metadata') From a769a8f6482135bb04de91a97eef8d65e029111e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 27 Dec 2017 14:55:52 -0300 Subject: [PATCH 0699/1506] Fix a bug when the vuln was created and updated in the same command scope --- server/api/base.py | 20 +++++++++++--- test_cases/test_api_vulnerability.py | 39 ++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 375f38978a0..8254ed0a8cf 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -453,13 +453,24 @@ def _set_command_id(self, obj, created): command = db.session.query(Command).filter(Command.id==command_id, Command.workspace==obj.workspace).first() if command is None: raise InvalidUsage('Command not found.') - command_object = CommandObject( + # if the object is created and updated in the same command + # the command object already exists + # we skip the creation. + command_object = CommandObject.query.filter_by( object_id=obj.id, object_type=obj.__class__.__name__, command=command, workspace=obj.workspace, - created_persistent=created - ) + ).first() + if created or not command_object: + command_object = CommandObject( + object_id=obj.id, + object_type=obj.__class__.__name__, + command=command, + workspace=obj.workspace, + created_persistent=created + ) + db.session.add(command) db.session.add(command_object) @@ -494,7 +505,8 @@ def put(self, object_id, **kwargs): # just in case an schema allows id as writable. data.pop('id', None) self._update_object(obj, data) - updated = self._perform_update(object_id, obj, **kwargs) + self._perform_update(object_id, obj, **kwargs) + return self._dump(obj, kwargs), 200 def _update_object(self, obj, data): diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 47235c8bc4c..04f4da2f4e6 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -263,7 +263,6 @@ def test_create_vuln_props(self, host_with_hostnames, test_client, session): if prop not in vuln_props_excluded: assert res.json[prop] == value, prop - @pytest.mark.skip(reason="Known bug") def test_create_idempotent(self, host_with_hostnames, test_client, session): """ This test makes sure that creating the same vuln twice doesn't duplicate the entry or has any other collateral effects @@ -289,7 +288,7 @@ def test_create_idempotent(self, host_with_hostnames, test_client, session): assert res.status_code == 201 assert vuln_count_previous + 1 == session.query(Vulnerability).count() res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) - assert res.status_code == 400 + assert res.status_code == 409 assert vuln_count_previous + 1 == session.query(Vulnerability).count() @@ -975,3 +974,39 @@ def test_create_webvuln_multiple_times_returns_conflict(self, host_with_hostname res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), data=raw_data) assert res.status_code == 409 + + def test_create_and_update_webvuln(self, host_with_hostnames, test_client, session): + """ + This reproduces a bug found. after creating an object with a + command, the update caused an integrity error within the same + command scope. + """ + command = CommandFactory.create(workspace=self.workspace) + service = ServiceFactory.create(workspace=self.workspace) + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='New vulnsweb', + vuln_type='VulnerabilityWeb', + parent_id=service.id, + parent_type='Service', + refs=[], + policyviolations=[], + description='helloworld', + severity='low', + ) + ws_name = host_with_hostnames.workspace.name + res = test_client.post('/v2/ws/{0}/vulns/?command_id={1}'.format(ws_name, command.id), data=raw_data) + assert res.status_code == 201 + raw_data = self._create_post_data_vulnerability( + name='Update vulnsweb', + vuln_type='VulnerabilityWeb', + parent_id=service.id, + parent_type='Service', + refs=[], + policyviolations=[], + description='Update helloworld', + severity='high', + ) + res = test_client.put('/v2/ws/{0}/vulns/{1}/?command_id={2}'.format(ws_name, res.json['_id'], command.id), + data=raw_data) + assert res.status_code == 200 \ No newline at end of file From 8757839592d8e55b29a743df38927475fa913015 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 27 Dec 2017 16:02:22 -0300 Subject: [PATCH 0700/1506] Fix create vuln from template in the web It was using the old and not fixed for the new api CWE provider, instead of the vuln models one. --- server/api/modules/vulnerability_template.py | 5 ++++- server/www/scripts/statusReport/controllers/modalNew.js | 8 ++++---- server/www/scripts/vulndb/providers/vulnModel.js | 7 ++++++- test_cases/test_api_vulnerability_template.py | 5 ++++- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/server/api/modules/vulnerability_template.py b/server/api/modules/vulnerability_template.py index aa1d412a0d6..68c75e8e8bb 100644 --- a/server/api/modules/vulnerability_template.py +++ b/server/api/modules/vulnerability_template.py @@ -33,10 +33,13 @@ class VulnerabilityTemplateSchema(AutoSchema): cwe = fields.String(dump_only=True, default='') # deprecated field, the legacy data is added to refs on import exploitation = fields.String(attribute='severity', validate=OneOf(VulnerabilityTemplate.SEVERITIES), required=True) references = fields.Method('get_references', deserialize='load_references', required=True) + refs = fields.List(fields.String(), dump_only=True, attribute='references') + desc = fields.String(dump_only=True, attribute='description') class Meta: model = VulnerabilityTemplate - fields = ('id', '_id', '_rev', 'cwe', 'description', 'exploitation', 'name', 'references', 'resolution') + fields = ('id', '_id', '_rev', 'cwe', 'description', 'desc', + 'exploitation', 'name', 'references', 'refs', 'resolution') def get_references(self, obj): return ', '.join(map(lambda ref_tmpl: ref_tmpl.name, obj.reference_template_instances)) diff --git a/server/www/scripts/statusReport/controllers/modalNew.js b/server/www/scripts/statusReport/controllers/modalNew.js index 6f11162aab8..61743e5257e 100644 --- a/server/www/scripts/statusReport/controllers/modalNew.js +++ b/server/www/scripts/statusReport/controllers/modalNew.js @@ -4,8 +4,8 @@ angular.module('faradayApp') .controller('modalNewVulnCtrl', - ['$modalInstance', '$filter', '$upload', 'EASEOFRESOLUTION', 'commonsFact', 'severities', 'workspace', 'targetFact', 'cweFact', - function($modalInstance, $filter, $upload, EASEOFRESOLUTION, commons, severities, workspace, targetFact, cweFact) { + ['$modalInstance', '$filter', '$upload', 'EASEOFRESOLUTION', 'commonsFact', 'severities', 'workspace', 'targetFact', 'vulnModelsManager', + function($modalInstance, $filter, $upload, EASEOFRESOLUTION, commons, severities, workspace, targetFact, vulnModelsManager) { var vm = this; @@ -48,7 +48,7 @@ angular.module('faradayApp') vm.host_parents = false; vm.cweList = []; - cweFact.get().then(function(data) { + vulnModelsManager.get().then(function(data) { vm.cweList = data; }); vm.cweLimit = 5; @@ -228,7 +228,7 @@ angular.module('faradayApp') vm.data.refs = refs; var policyviolations = []; - item.policyviolations.forEach(function(policyviolation) { + if(item.hasOwnProperty('policyviolations')) item.policyviolations.forEach(function(policyviolation) { policyviolations.push({value: policyviolation}); }); vm.data.policyviolations = policyviolations; diff --git a/server/www/scripts/vulndb/providers/vulnModel.js b/server/www/scripts/vulndb/providers/vulnModel.js index 9346136bac0..0d03464f993 100644 --- a/server/www/scripts/vulndb/providers/vulnModel.js +++ b/server/www/scripts/vulndb/providers/vulnModel.js @@ -7,12 +7,15 @@ angular.module('faradayApp'). function(BASEURL, configSrv, ServerAPI, $http, $q) { function VulnModel(data) { this._id = ""; + this.id = ""; this._rev = ""; this.cwe = ""; this.description = ""; + this.desc = ""; this.exploitation = ""; this.name = ""; this.references = []; + this.refs = []; this.resolution = ""; if (data) { if(data.name === undefined || data.name === "") { @@ -24,13 +27,15 @@ angular.module('faradayApp'). VulnModel.prototype = { - public_properties: ['exploitation', 'references', 'name', 'resolution', 'cwe', 'description'], + public_properties: ['exploitation', 'references', 'name', 'resolution', 'cwe', 'description', + 'desc', 'id', 'refs'], set: function(data) { var self = this; if(data._id != undefined) { self._id = data._id; + self.id = data._id; if(data._rev !== undefined) { self._rev = data._rev; }; diff --git a/test_cases/test_api_vulnerability_template.py b/test_cases/test_api_vulnerability_template.py index 3677331fd2b..8a4db0aa565 100644 --- a/test_cases/test_api_vulnerability_template.py +++ b/test_cases/test_api_vulnerability_template.py @@ -36,12 +36,14 @@ def test_backwards_json_compatibility(self, test_client, session): object_properties = [ u'exploitation', u'references', + u'refs', u'name', u'cwe', u'_rev', u'_id', u'resolution', - u'description' + u'description', + u'desc', ] expected = set(object_properties) @@ -122,6 +124,7 @@ def test_create_new_vulnerability_template_with_references(self, session, test_c res = test_client.post('/v2/vulnerability_template/', data=raw_data) assert res.status_code == 201 assert isinstance(res.json['_id'], int) + assert set(res.json['refs']) == set(['ref1', 'ref2']) assert vuln_count_previous + 1 == session.query(VulnerabilityTemplate).count() new_template = session.query(VulnerabilityTemplate).filter_by(id=res.json['_id']).first() assert new_template.references == set([u'ref1', u'ref2']) From 4aaeccc113f1e5a12bba11245e4a7b8b5ff6280e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 27 Dec 2017 16:11:27 -0300 Subject: [PATCH 0701/1506] Remove CWE provider (cweFact) It did the same as the vuln model provider but it wasn't adapted to the new api. I removed it and replaced its reference for the vulnModelsManager --- server/www/index.html | 1 - .../www/scripts/commons/controllers/modal.js | 6 +-- .../www/scripts/cwe/providers/CweService.js | 45 ------------------- .../statusReport/controllers/modalEdit.js | 6 +-- 4 files changed, 6 insertions(+), 52 deletions(-) delete mode 100644 server/www/scripts/cwe/providers/CweService.js diff --git a/server/www/index.html b/server/www/index.html index d971911a736..5aad923de74 100644 --- a/server/www/index.html +++ b/server/www/index.html @@ -111,7 +111,6 @@ - diff --git a/server/www/scripts/commons/controllers/modal.js b/server/www/scripts/commons/controllers/modal.js index 345ffad4f87..6f72983e336 100644 --- a/server/www/scripts/commons/controllers/modal.js +++ b/server/www/scripts/commons/controllers/modal.js @@ -138,9 +138,9 @@ angular.module('faradayApp') } }]); angular.module('faradayApp') - .controller('commonsModalEditCWE', ['$scope', '$modalInstance', 'msg', 'cweFact', function($scope, $modalInstance, msg, cweFact) { + .controller('commonsModalEditCWE', ['$scope', '$modalInstance', 'msg', 'vulnModelsManager', function($scope, $modalInstance, msg, vulnModelsManager) { $scope.cweList = []; - cweFact.get().then(function(data) { + vulnModelsManager.get().then(function(data) { $scope.cweList = data; }); $scope.cweLimit = 5; @@ -198,4 +198,4 @@ angular.module('faradayApp') $scope.cancel = function() { $modalInstance.dismiss(); } - }]); \ No newline at end of file + }]); diff --git a/server/www/scripts/cwe/providers/CweService.js b/server/www/scripts/cwe/providers/CweService.js deleted file mode 100644 index e0fda750f6e..00000000000 --- a/server/www/scripts/cwe/providers/CweService.js +++ /dev/null @@ -1,45 +0,0 @@ -// Faraday Penetration Test IDE -// Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) -// See the file 'doc/LICENSE' for the license information - -angular.module('faradayApp') - .factory('cweFact', ['BASEURL', '$http', '$q', 'vulnModelsManager', 'commonsFact', function(BASEURL, $http, $q, vulnModelsManager, commonsFact) { - var cweFact = {}; - - // XXX: this is still not using the server - cweFact.get = function() { - var cweList = []; - var deferred = $q.defer(); - var cwe_url = BASEURL + 'cwe/_all_docs?include_docs=true'; - $http.get(cwe_url).then(function(res) { - res.data.rows.forEach(function(obj) { - var description = ""; - - if(obj.doc.description) description += obj.doc.description; - - var c = { - id: obj.id, - cwe: obj.doc.cwe, - name: commonsFact.htmlentities(obj.doc.name), - desc: commonsFact.htmlentities(description), - resolution: obj.doc.resolution, - exploitation: obj.doc.exploitation, - refs: obj.doc.references - }; - if (typeof(obj.doc.references) == "string") { - c.refs = []; - obj.doc.references.split('\n').forEach(function(ref) { - if (ref != "") { - c.refs.push(ref); - } - }); - } - cweList.push(c); - }); - deferred.resolve(cweList); - }); - return deferred.promise; - }; - - return cweFact; - }]); diff --git a/server/www/scripts/statusReport/controllers/modalEdit.js b/server/www/scripts/statusReport/controllers/modalEdit.js index 111e004a6f2..5fd1a335700 100644 --- a/server/www/scripts/statusReport/controllers/modalEdit.js +++ b/server/www/scripts/statusReport/controllers/modalEdit.js @@ -3,8 +3,8 @@ // See the file 'doc/LICENSE' for the license information angular.module('faradayApp') - .controller('modalEditCtrl', ['$modalInstance', 'EASEOFRESOLUTION', 'STATUSES', 'commonsFact', 'severities', 'vuln', 'cweFact', - function($modalInstance, EASEOFRESOLUTION, STATUSES, commons, severities, vuln, cweFact) { + .controller('modalEditCtrl', ['$modalInstance', 'EASEOFRESOLUTION', 'STATUSES', 'commonsFact', 'severities', 'vuln', 'vulnModelsManager', + function($modalInstance, EASEOFRESOLUTION, STATUSES, commons, severities, vuln, vulnModelsManager) { var vm = this; @@ -32,7 +32,7 @@ angular.module('faradayApp') vm.icons = {}; vm.cweList = []; - cweFact.get().then(function(data) { + vulnModelsManager.get().then(function(data) { vm.cweList = data; }); vm.cweLimit = 5; From 3c10c9616445dcb0227cef02b8f325978a668586 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 28 Dec 2017 15:12:16 -0300 Subject: [PATCH 0702/1506] Fix a bug on initdb: a method retuned two values --- server/commands/initdb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index 29bfb86ae4b..1bb106d0108 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -1,3 +1,4 @@ +import getpass import string import os @@ -90,7 +91,7 @@ def run(self): 'Please enter the {blue} database name {white} (press enter to use "faraday"): '.format( blue=Fore.BLUE, white=Fore.WHITE)) or 'faraday' current_psql_output = TemporaryFile() - process_status = self._create_database(database_name, username, current_psql_output) + database_name, process_status = self._create_database(database_name, username, current_psql_output) current_psql_output.seek(0) self._check_psql_output(current_psql_output, process_status) else: From 463db3b194e42c43d413895629da71cf06c821bb Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 29 Dec 2017 10:19:00 -0300 Subject: [PATCH 0703/1506] Create admin user on initdb --- managers/reports_managers.py | 2 +- server/commands/initdb.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/managers/reports_managers.py b/managers/reports_managers.py index cd855bd9ae8..c0fe8c96902 100755 --- a/managers/reports_managers.py +++ b/managers/reports_managers.py @@ -131,7 +131,7 @@ def syncReports(self): filenames = [] for root, dirs, files in os.walk(self._report_path, False): - + # skip processed and unprocessed directories if root == self._report_path: for name in files: filenames.append(os.path.join(root, name)) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index 1bb106d0108..ff44622a2b8 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -7,6 +7,8 @@ from tempfile import TemporaryFile from subprocess import Popen, PIPE +from sqlalchemy import create_engine + try: # py2.7 from configparser import ConfigParser, NoSectionError, NoOptionError @@ -103,11 +105,19 @@ def run(self): current_psql_output.close() conn_string = self._save_config(config, username, password, database_name, hostname) self._create_tables(conn_string) + self._create_admin_user(conn_string) except KeyboardInterrupt: current_psql_output.close() print('User cancelled.') sys.exit(1) + def _create_admin_user(self, conn_string): + engine = create_engine(conn_string) + random_password = self.generate_random_pw(12) + engine.execute("INSERT INTO \"user\" (username, password, is_ldap, active) VALUES ('admin', '{0}', false, true);".format(random_password)) + print("Admin username created with {red} password{white}: {random_password}".format(random_password=random_password, white=Fore.WHITE, red=Fore.RED)) + + def _configure_existing_postgres_user(self): username = raw_input('Please enter the postgresql username: ') password = getpass.getpass('Please enter the postgresql password: ') From c282f02f9b1fe4750b8854e113078d4ded8bea16 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 29 Dec 2017 10:19:23 -0300 Subject: [PATCH 0704/1506] Improve wait time for getID --- persistence/server/models.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index d4601d81a55..4c387cf43d4 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -726,8 +726,15 @@ def setID(self, id): self.id_available.set() def getID(self): - if self.id is None: - self.id_available.wait(timeout=10) + # getId will wait until the id is not None + timeout = 1 + retries = 1 + max_retries = 6 + while retries <= max_retries and self.id is None: + self.id_available.wait(timeout=timeout) + print('Retrying getID timeout {0}'.format(timeout)) + timeout = timeout << retries - 1 + retries += 1 return self.id @staticmethod From 597b2f8abefda9058fdc8e6b30ffccacd49f0e23 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 29 Dec 2017 10:29:27 -0300 Subject: [PATCH 0705/1506] Encode to ascii the responde --- persistence/server/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index 4c387cf43d4..582808a5d1d 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -1080,7 +1080,7 @@ def __init__(self, vuln_web, workspace_name): self.path = vuln_web.get('path') self.website = vuln_web.get('website') self.request = vuln_web.get('request') - self.response = vuln_web.get('response') + self.response = vuln_web.get('response').encode('ascii', 'ignore') self.method = vuln_web.get('method') or '' self.pname = vuln_web.get('pname') self.params = vuln_web.get('params') or '' From 56ea038f906d03e33c1662ccf85e58a029a5bdaa Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 29 Dec 2017 12:20:02 -0300 Subject: [PATCH 0706/1506] Fix encoding issues and close open file. --- persistence/server/models.py | 2 +- plugins/plugin.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index 582808a5d1d..4c387cf43d4 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -1080,7 +1080,7 @@ def __init__(self, vuln_web, workspace_name): self.path = vuln_web.get('path') self.website = vuln_web.get('website') self.request = vuln_web.get('request') - self.response = vuln_web.get('response').encode('ascii', 'ignore') + self.response = vuln_web.get('response') self.method = vuln_web.get('method') or '' self.pname = vuln_web.get('pname') self.params = vuln_web.get('params') or '' diff --git a/plugins/plugin.py b/plugins/plugin.py index 07d7689a9b0..09acde0fced 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -126,8 +126,11 @@ def getCompletitionSuggestionsList(self, current_input): def processOutput(self, term_output): output = term_output if self.has_custom_output() and os.path.isfile(self.get_custom_file_path()): - output = open(self.get_custom_file_path(), 'r').read() - self.parseOutputString(output) + with open(self.get_custom_file_path(), 'r', encoding='utf8').read() as output: + self.parseOutputString(output) + else: + self.parseOutputString(output) + def processReport(self, filepath): if os.path.isfile(filepath): From 586ce441a912b9baed2280d5423ad8aa47fbbb17 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 29 Dec 2017 15:56:37 -0300 Subject: [PATCH 0707/1506] Fix burp encoding issue --- plugins/plugin.py | 9 +++++---- plugins/repo/burp/plugin.py | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/plugins/plugin.py b/plugins/plugin.py index 09acde0fced..e8c008224b7 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -126,16 +126,17 @@ def getCompletitionSuggestionsList(self, current_input): def processOutput(self, term_output): output = term_output if self.has_custom_output() and os.path.isfile(self.get_custom_file_path()): - with open(self.get_custom_file_path(), 'r', encoding='utf8').read() as output: - self.parseOutputString(output) + self._parse_filename(self.get_custom_file_path()) else: self.parseOutputString(output) + def _parse_filename(self, filename): + with open(filename, 'rb') as output: + self.parseOutputString(output.read()) def processReport(self, filepath): if os.path.isfile(filepath): - output = open(filepath, 'r').read() - self.parseOutputString(output) + self._parse_filename(filepath) def parseOutputString(self, output): """ diff --git a/plugins/repo/burp/plugin.py b/plugins/repo/burp/plugin.py index 1084eb7f1d1..ed7bbd68b04 100644 --- a/plugins/repo/burp/plugin.py +++ b/plugins/repo/burp/plugin.py @@ -189,9 +189,10 @@ def decode_binary_node(self, path): return "" encoded = distutils.util.strtobool(subnode.get('base64', 'false')) if encoded: - return subnode.text.decode('base64', 'strict') + res = subnode.text.decode('base64', 'strict') else: - return subnode.text + res = subnode.text + return "".join([ch for ch in res if ord(ch) <= 128]) def get_text_from_subnode(self, subnode_xpath_expr): """ From 0e12bdb172bd5f7a269ec2ef66a1935e00c287c6 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 29 Dec 2017 15:57:11 -0300 Subject: [PATCH 0708/1506] Update unit tests --- test_cases/plugins/test_acunetix.py | 10 ++++------ test_cases/plugins/test_burp.py | 7 +++---- test_cases/plugins/test_nessus.py | 14 ++++++-------- test_cases/plugins/test_nexpose_full.py | 5 ++--- test_cases/plugins/test_ping.py | 1 - test_cases/plugins/test_telnet.py | 1 - test_cases/plugins/test_whois.py | 1 - test_cases/test_api_credentials.py | 1 - test_cases/test_server_io.py | 15 ++++++--------- 9 files changed, 21 insertions(+), 34 deletions(-) diff --git a/test_cases/plugins/test_acunetix.py b/test_cases/plugins/test_acunetix.py index 6e80b31510a..29541f14ed3 100644 --- a/test_cases/plugins/test_acunetix.py +++ b/test_cases/plugins/test_acunetix.py @@ -27,8 +27,6 @@ Host, Service, ModelBase) -from plugins.modelactions import modelactions - class TestAcunetixParser: @@ -59,19 +57,19 @@ def test_Plugin_creates_apropiate_objects(self, monkeypatch): action = self.plugin._pending_actions.get(block=True) actions[action[0]].append(action[1]) - assert actions.keys() == [2000, 20008, 2027, 2037, 2039] + assert actions.keys() == [2000, 20008, 2027, 2040, 2038] assert len(actions[2000]) == 1 assert actions[2000][0].name == "5.175.17.140" assert len(actions[20008]) == 1 assert len(actions[2027]) == 1 - assert len(actions[2037]) == 52 - assert len(actions[2039]) == 1 + assert len(actions[2040]) == 1 + assert len(actions[2038]) == 52 assert actions[20008][0].ports == [80] assert actions[20008][0].name == 'http' assert actions[20008][0].protocol == 'tcp' - assert "ASP.NET error message" in map(lambda vuln_web: vuln_web.name, actions[2037]) + assert "ASP.NET error message" in map(lambda vuln_web: vuln_web.name, actions[2038]) if __name__ == '__main__': diff --git a/test_cases/plugins/test_burp.py b/test_cases/plugins/test_burp.py index a38a350c4e4..fb76613f59d 100644 --- a/test_cases/plugins/test_burp.py +++ b/test_cases/plugins/test_burp.py @@ -25,7 +25,6 @@ Host, Service, ModelBase) -from plugins.modelactions import modelactions class TestBurp: @@ -51,11 +50,11 @@ def test_Plugin_creates_adecuate_objects(self, monkeypatch): actions[action[0]].append(action[1]) assert actions[2000][0].name == "200.20.20.201" - assert actions.keys() == [2000, 20008, 2027, 2037, 2039] + assert actions.keys() == [2000, 20008, 2027, 2040, 2038] assert len(actions[20008]) == 14 assert len(actions[2027]) == 14 - assert len(actions[2037]) == 14 - assert len(actions[2039]) == 14 + assert len(actions[2040]) == 14 + assert len(actions[2038]) == 14 assert all('http' == name for name in map(lambda service: service.name, actions[20008])) assert all([80] == ports for ports in map(lambda service: service.ports, actions[20008])) diff --git a/test_cases/plugins/test_nessus.py b/test_cases/plugins/test_nessus.py index dc389cf5685..6c5ca3e0da6 100644 --- a/test_cases/plugins/test_nessus.py +++ b/test_cases/plugins/test_nessus.py @@ -7,9 +7,8 @@ See the file 'doc/LICENSE' for the license information ''' - -import unittest import sys +import unittest from Queue import Queue from collections import defaultdict @@ -25,8 +24,6 @@ Host, Service, ModelBase) -from plugins.modelactions import modelactions -import test_common class TestNessusParser: @@ -51,13 +48,14 @@ def test_Plugin_Calls_createAndAddHost(self, monkeypatch): actions[action[0]].append(action[1]) assert actions[2000][0].name == "12.233.108.201" - assert actions.keys() == [2017, 20008, 2027, 2000, 2037, 2039] + assert actions.keys() == [2017, 20008, 2027, 2000, 2038, 2040] assert len(actions[20008]) == 1 assert len(actions[2027]) == 1 - assert len(actions[2037]) == 1 - assert len(actions[2039]) == 1 + assert len(actions[2038]) == 1 + assert len(actions[2040]) == 1 - assert actions[2037][0].name == "Nessus SYN scanner" + assert actions[2040][0].name == "preprod.boardvantage.net" + assert actions[2038][0].name == "Nessus SYN scanner" assert actions[20008][0].ports == [443] assert actions[20008][0].name == 'https?' diff --git a/test_cases/plugins/test_nexpose_full.py b/test_cases/plugins/test_nexpose_full.py index 7b4c5aeade8..f0e9976665b 100644 --- a/test_cases/plugins/test_nexpose_full.py +++ b/test_cases/plugins/test_nexpose_full.py @@ -28,7 +28,6 @@ Host, Service, ModelBase) -from plugins.modelactions import modelactions class TestNexpose: @@ -53,12 +52,12 @@ def test_Plugin_creates_apropiate_objects(self, monkeypatch): actions[action[0]].append(action[1]) assert actions[2000][0].name == "192.168.1.1" - assert actions.keys() == [2000, 2017, 2019, 2037, 20008] + assert actions.keys() == [2000, 2017, 2019, 2038, 20008] assert len(actions[2000]) == 8 assert len(actions[20008]) == 20 assert len(actions[2027]) == 0 - assert len(actions[2037]) == 403 + assert len(actions[2038]) == 403 assert len(actions[2039]) == 0 diff --git a/test_cases/plugins/test_ping.py b/test_cases/plugins/test_ping.py index bcd98458ff0..c19e4ff338b 100644 --- a/test_cases/plugins/test_ping.py +++ b/test_cases/plugins/test_ping.py @@ -25,7 +25,6 @@ Host, Service, ModelBase) -from plugins.modelactions import modelactions class TestCmdPingPlugin: diff --git a/test_cases/plugins/test_telnet.py b/test_cases/plugins/test_telnet.py index 0962517faad..56868f65ad3 100644 --- a/test_cases/plugins/test_telnet.py +++ b/test_cases/plugins/test_telnet.py @@ -25,7 +25,6 @@ Host, Service, ModelBase) -from plugins.modelactions import modelactions class TestCmdPingPlugin: diff --git a/test_cases/plugins/test_whois.py b/test_cases/plugins/test_whois.py index 5759993ffd0..a7364a95319 100644 --- a/test_cases/plugins/test_whois.py +++ b/test_cases/plugins/test_whois.py @@ -25,7 +25,6 @@ Host, Service, ModelBase) -from plugins.modelactions import modelactions class TestCmdPingPlugin: diff --git a/test_cases/test_api_credentials.py b/test_cases/test_api_credentials.py index 7452cd80289..9d67b186da8 100644 --- a/test_cases/test_api_credentials.py +++ b/test_cases/test_api_credentials.py @@ -30,7 +30,6 @@ def test_get_list_backwards_compatibility(self, test_client, session, second_wor u'description', u'metadata', u'name', - u'owned', u'owner', u'password', u'username', diff --git a/test_cases/test_server_io.py b/test_cases/test_server_io.py index ee5aeb67a35..4de4476fe41 100644 --- a/test_cases/test_server_io.py +++ b/test_cases/test_server_io.py @@ -121,17 +121,16 @@ def test_faraday_dictionary_dispatcher_result(self): @patch('persistence.server.server._get_raw_hosts') @patch('persistence.server.server._get_raw_vulns') - @patch('persistence.server.server._get_raw_interfaces') @patch('persistence.server.server._get_raw_services') @patch('persistence.server.server._get_raw_notes') @patch('persistence.server.server._get_raw_credentials') @patch('persistence.server.server._get_raw_commands') - def test_faraday_dictionary_dispatcher_calls(self, mock_hosts, mock_vulns, mock_interfaces, - mock_services, mock_notes, mock_credentials, mock_commands): + def test_faraday_dictionary_dispatcher_calls(self, mock_hosts, mock_vulns, + mock_services, mock_notes, + mock_credentials, mock_commands): # NOTE: if you finds any bugs here, i have the suspipcion that mock_host is actually mock_commands # i mean that the parameters names are wrong. I'd check for that. Good luck. server._get_faraday_ready_dictionaries('a', 'hosts', 'whatever') - server._get_faraday_ready_dictionaries('a', 'interfaces', 'whatever') server._get_faraday_ready_dictionaries('a', 'vulns', 'whatever') server._get_faraday_ready_dictionaries('a', 'services', 'whatever') server._get_faraday_ready_dictionaries('a', 'notes', 'whatever') @@ -139,7 +138,6 @@ def test_faraday_dictionary_dispatcher_calls(self, mock_hosts, mock_vulns, mock_ server._get_faraday_ready_dictionaries('a', 'commands', 'whatever') mock_hosts.assert_called_once_with('a') mock_vulns.assert_called_once_with('a') - mock_interfaces.assert_called_once_with('a') mock_services.assert_called_once_with('a') mock_notes.assert_called_once_with('a') mock_credentials.assert_called_once_with('a') @@ -147,14 +145,13 @@ def test_faraday_dictionary_dispatcher_calls(self, mock_hosts, mock_vulns, mock_ @patch('persistence.server.server.get_hosts', return_value='hosts') @patch('persistence.server.server.get_vulns', return_value='vulns') - @patch('persistence.server.server.get_interfaces', return_value='interfaces') @patch('persistence.server.server.get_services', return_value='services') @patch('persistence.server.server.get_credentials', return_value='CREDENTIAL') @patch('persistence.server.server.get_notes', return_value='NOTE') @patch('persistence.server.server.get_commands', return_value='COMMAND') - def test_get_objects(self, not_command, not_note, not_credential, not_service, - not_interface, not_vuln, not_host): - obj_sign_to_mock = {'hosts': not_host, 'vulns': not_vuln, 'interfaces': not_interface, + def test_get_objects(self, not_command, not_note, + not_credential, not_service, not_vuln, not_host): + obj_sign_to_mock = {'hosts': not_host, 'vulns': not_vuln, 'services': not_service, 'credentials': not_credential, 'notes': not_note, 'commands': not_command} for obj_sign in obj_sign_to_mock.keys(): From 2158cb7e6d1abbf46b264f8e0f44868c8288cf7a Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 2 Jan 2018 12:08:48 -0300 Subject: [PATCH 0709/1506] Add new api configuration to user.xml To configure the faraday client it was added: http://localhost:5985 admin secret --- config/configuration.py | 62 ++++++++++++++++++------------------ persistence/server/server.py | 2 +- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/config/configuration.py b/config/configuration.py index 91c678a7223..f34a3ae51ab 100644 --- a/config/configuration.py +++ b/config/configuration.py @@ -39,9 +39,9 @@ CONST_PERSISTENCE_PATH = "persistence_path" CONST_PERSPECTIVE_VIEW = "perspective_view" CONST_REPO_PASSWORD = "repo_password" -CONST_SERVER_URI = "couch_uri" -CONST_COUCH_REPLICS = "couch_replics" -CONST_COUCH_ISREPLICATED = "couch_is_replicated" +CONST_API_URL = "api_url" +CONST_API_USERNAME = "api_username" +CONST_API_PASSWORD = "api_password" CONST_REPO_URL = "repo_url" CONST_REPO_USER = "repo_user" CONST_REPORT_PATH = "report_path" @@ -145,9 +145,9 @@ def _getConfig(self): self._persistence_path = self._getValue(tree, CONST_PERSISTENCE_PATH) self._perspective_view = self._getValue(tree, CONST_PERSISTENCE_PATH) self._repo_password = self._getValue(tree, CONST_REPO_PASSWORD) - self._server_uri = self._getValue(tree, CONST_SERVER_URI, default = "") - self._couch_replics = self._getValue(tree, CONST_COUCH_REPLICS, default = "") - self._couch_is_replicated = bool(self._getValue(tree, CONST_COUCH_ISREPLICATED, default = False)) + self._api_url = self._getValue(tree, CONST_API_URL) + self._api_username = self._getValue(tree, CONST_API_USERNAME) + self._api_password = self._getValue(tree, CONST_API_PASSWORD) self._repo_url = self._getValue(tree, CONST_REPO_URL) self._repo_user = self._getValue(tree, CONST_REPO_USER) self._report_path = self._getValue(tree, CONST_REPORT_PATH) @@ -245,16 +245,7 @@ def getPerspectiveView(self): return self._perspective_view def getServerURI(self): - if self._server_uri and self._server_uri.endswith('/'): - return self._server_uri[:-1] - else: - return self._server_uri - - def getCouchReplics(self): - return self._couch_replics - - def getCouchIsReplicated(self): - return self._couch_is_replicated + return self._api_url def getDBSessionCookies(self): return self._session_cookies @@ -304,6 +295,15 @@ def getTktTemplate(self): def getMergeStrategy(self): return self._merge_strategy + def getAPIUrl(self): + return self._api_url + + def getAPIUsername(self): + return self._api_username + + def getAPIPassword(self): + return self._api_password + def setLastWorkspace(self, workspaceName): self._last_workspace = workspaceName @@ -404,14 +404,14 @@ def setShellMaximized(self, val): def setVersion(self, val): self._version = val - def setCouchUri(self, uri): - self._server_uri = uri + def setAPIUrl(self, url): + self._server_uri = url - def setCouchIsReplicated(self, is_it): - self._couch_is_replicated = is_it + def setAPIUsername(self, username): + self._username = username - def setCouchReplics(self, urls): - self._couch_replics = urls + def setAPIPassword(self, password): + self._password = password def setPluginSettings(self, settings): self._plugin_settings = settings @@ -557,17 +557,17 @@ def saveConfig(self, xml_file="~/.faraday/config/user.xml"): LAST_WORKSPACE.text = self.getLastWorkspace() ROOT.append(LAST_WORKSPACE) - SERVER_URI = Element(CONST_SERVER_URI) - SERVER_URI.text = self.getServerURI() - ROOT.append(SERVER_URI) + SERVER_URL = Element(CONST_API_URL) + SERVER_URL.text = self.getServerURI() + ROOT.append(SERVER_URL) - COUCH_IS_REPLICATED = Element(CONST_COUCH_ISREPLICATED) - COUCH_IS_REPLICATED.text = str(self.getCouchIsReplicated()) - ROOT.append(COUCH_IS_REPLICATED) + SERVER_USERNAME = Element(CONST_API_USERNAME) + SERVER_USERNAME.text = self.getAPIUsername() + ROOT.append(SERVER_USERNAME) - COUCH_REPLICS = Element(CONST_COUCH_REPLICS) - COUCH_REPLICS.text = self.getCouchReplics() - ROOT.append(COUCH_REPLICS) + SERVER_PASSWORD = Element(CONST_API_PASSWORD) + SERVER_PASSWORD.text = self.getAPIPassword() + ROOT.append(SERVER_PASSWORD) VERSION = Element(CONST_VERSION) VERSION.text = self.getVersion() diff --git a/persistence/server/server.py b/persistence/server/server.py index 1d973bffb80..9809fafadc7 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -72,7 +72,7 @@ def _conf(): # Fplugin run in other instance, so this dont generate any trouble. if not CONF.getDBSessionCookies(): server_url = CONF.getServerURI() if FARADAY_UP else SERVER_URL - cookie = login_user(server_url, AUTH_USER, AUTH_PASS) + cookie = login_user(server_url, CONF.getAPIUsername(), CONF.getAPIPassword()) CONF.setDBSessionCookies(cookie) return CONF From 8121ddd3fcbf8d9c1ca1f98d319749c1739c1080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 2 Jan 2018 15:36:31 -0300 Subject: [PATCH 0710/1506] Undefer credential count in host API It was causing (strangely not detected) nplusone issues in the host list --- server/api/modules/hosts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index ff37be683aa..506913652e7 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -85,7 +85,8 @@ class HostsView(PaginatedMixin, schema_class = HostSchema unique_fields = [('ip', )] filterset_class = HostFilterSet - get_undefer = [Host.open_service_count, + get_undefer = [Host.credentials_count, + Host.open_service_count, Host.vulnerability_count] get_joinedloads = [Host.hostnames] From 83887d7be2ea1e7de24a68b250602355fb2e509d Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 2 Jan 2018 18:44:22 -0300 Subject: [PATCH 0711/1506] Add control to home and update test to return 401 instead of 403 * API was retuning 403 when user wasnot logged in, this was changed to return 401. * Added a controller on the home to check if the user is logged in since this is the default view for the root url. * Updated unit test to assert against 401 status code --- server/app.py | 2 +- server/www/index.html | 1 + server/www/scripts/app.js | 3 ++- .../scripts/commons/controllers/homeCtrl.js | 15 ++++++++++++++ test_cases/test_server.py | 20 +++++++++---------- 5 files changed, 29 insertions(+), 12 deletions(-) create mode 100644 server/www/scripts/commons/controllers/homeCtrl.js diff --git a/server/app.py b/server/app.py index c19563679fd..4053dc6315b 100644 --- a/server/app.py +++ b/server/app.py @@ -98,7 +98,7 @@ def default_login_required(): view = app.view_functions.get(flask.request.endpoint) logged_in = 'user_id' in flask.session if (not logged_in and not getattr(view, 'is_public', False)): - flask.abort(403) + flask.abort(401) g.user = None if logged_in: diff --git a/server/www/index.html b/server/www/index.html index d971911a736..7ac9c8e9ea8 100644 --- a/server/www/index.html +++ b/server/www/index.html @@ -98,6 +98,7 @@ + diff --git a/server/www/scripts/app.js b/server/www/scripts/app.js index a4fc3f519c3..a8935360bf5 100644 --- a/server/www/scripts/app.js +++ b/server/www/scripts/app.js @@ -284,7 +284,8 @@ faradayApp.config(['$routeProvider', 'ngClipProvider', '$uibTooltipProvider', title: ' Forbidden |' }). otherwise({ - templateUrl: 'scripts/commons/partials/home.html' + templateUrl: 'scripts/commons/partials/home.html', + controller: 'homeCtrl' }); }]); diff --git a/server/www/scripts/commons/controllers/homeCtrl.js b/server/www/scripts/commons/controllers/homeCtrl.js new file mode 100644 index 00000000000..2accb641923 --- /dev/null +++ b/server/www/scripts/commons/controllers/homeCtrl.js @@ -0,0 +1,15 @@ +// Faraday Penetration Test IDE +// Copyright (C) 2018 Infobyte LLC (http://www.infobytesec.com/) +// See the file 'doc/LICENSE' for the license information + +angular.module('faradayApp') + .controller('homeCtrl', + ['$location', 'loginSrv', + function($location, loginSrv) { + loginSrv.isAuthenticated().then(function(auth){ + if(!auth) { + $location.path('/login'); + } + return deferred.reject(response); + }); + }]); \ No newline at end of file diff --git a/test_cases/test_server.py b/test_cases/test_server.py index 0d3994dce38..6c5b85bed2a 100644 --- a/test_cases/test_server.py +++ b/test_cases/test_server.py @@ -38,17 +38,17 @@ class TestAuthentication(BaseAPITestCase, unittest.TestCase): """Tests related to allow/dissallow access depending of whether the user is logged in or not""" - def test_403_when_getting_an_existent_view_and_not_logged(self): + def test_401_when_getting_an_existent_view_and_not_logged(self): res = self.app.get('/') - self.assertEqual(res.status_code, 403) + self.assertEqual(res.status_code, 401) - def test_403_when_posting_an_existent_view_and_not_logged(self): + def test_401_when_posting_an_existent_view_and_not_logged(self): res = self.app.post('/', 'data') - self.assertEqual(res.status_code, 403) + self.assertEqual(res.status_code, 401) - def test_403_when_accessing_a_non_existent_view_and_not_logged(self): + def test_401_when_accessing_a_non_existent_view_and_not_logged(self): res = self.app.post('/dfsdfsdd', 'data') - self.assertEqual(res.status_code, 403) + self.assertEqual(res.status_code, 401) def test_200_when_not_logged_but_endpoint_is_public(self): endpoint.is_public = True @@ -56,7 +56,7 @@ def test_200_when_not_logged_but_endpoint_is_public(self): self.assertEqual(res.status_code, 200) del endpoint.is_public - def test_403_when_logged_user_is_inactive(self): + def test_401_when_logged_user_is_inactive(self): with self.flask_app.app_context(): # Without this line the test breaks. Taken from # http://pythonhosted.org/Flask-Testing/#testing-with-sqlalchemy @@ -64,13 +64,13 @@ def test_403_when_logged_user_is_inactive(self): self.assertTrue(self.flask_app.user_datastore.deactivate_user(self.user)) res = self.app.get('/') - self.assertEqual(res.status_code, 403) + self.assertEqual(res.status_code, 401) - def test_403_when_logged_user_is_deleted(self): + def test_401_when_logged_user_is_deleted(self): with self.flask_app.app_context(): self.flask_app.user_datastore.delete_user(self.user) res = self.app.get('/') - self.assertEqual(res.status_code, 403) + self.assertEqual(res.status_code, 401) class TestAuthenticationPytest(BaseAPITestCase): From bdc2712e970d32582fe0ce180b580447a3edc3a7 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 2 Jan 2018 18:53:03 -0300 Subject: [PATCH 0712/1506] Add auth check on commercial controller --- server/www/scripts/commons/controllers/commercialCtrl.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/www/scripts/commons/controllers/commercialCtrl.js b/server/www/scripts/commons/controllers/commercialCtrl.js index 1b18700c0cf..ed1352b3cb9 100644 --- a/server/www/scripts/commons/controllers/commercialCtrl.js +++ b/server/www/scripts/commons/controllers/commercialCtrl.js @@ -6,6 +6,12 @@ angular.module('faradayApp') .controller('commercialCtrl', ['$scope', '$location', function($scope, $location) { + loginSrv.isAuthenticated().then(function(auth){ + if(!auth) { + $location.path('/login'); + } + return deferred.reject(response); + }); if ($location.path().split("/")[1] === "executive") { $scope.header = "executive report"; } else if ($location.path().split("/")[1] === "comparison") { From b31aeafb8735ca7e6554d8a9f2ba193383febf9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 2 Jan 2018 19:01:44 -0300 Subject: [PATCH 0713/1506] Remove severity from vuln unique fields in the importer We changed the model so severity isn't revelant when deciding if two vulns are the same or not --- server/importer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/importer.py b/server/importer.py index 06c903735fc..5b20bc45793 100644 --- a/server/importer.py +++ b/server/importer.py @@ -527,7 +527,6 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation VulnerabilityWeb, name=document.get('name'), description=document.get('desc'), - severity=severity, service_id=parent.id, method=method, parameter_name=pname, @@ -539,7 +538,6 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation if document['type'] == 'Vulnerability': vuln_params = { 'name': document.get('name'), - 'severity': severity, 'workspace': workspace, 'description': document.get('desc') } @@ -552,6 +550,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation Vulnerability, **vuln_params ) + vulnerability.severity = severity vulnerability.confirmed = document.get('confirmed', False) or False vulnerability.data = document.get('data') vulnerability.ease_of_resolution = document.get('easeofresolution') if document.get('easeofresolution') else None From 1a08e58807a90fbc964b4647c9c95b1b0e683ffc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 2 Jan 2018 19:12:49 -0300 Subject: [PATCH 0714/1506] Show warning when many queries done in the same requests This will help diasnosing not detected nplusone issues. It should be rare that a unique request does 15 or more SQL queries. --- server/app.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/server/app.py b/server/app.py index 68c12627fa6..bbaa42aaa0e 100644 --- a/server/app.py +++ b/server/app.py @@ -18,6 +18,7 @@ import flask from flask import Flask, session, g from flask.json import JSONEncoder +from flask_sqlalchemy import get_debug_queries from flask_security import ( Security, SQLAlchemyUserDatastore, @@ -29,8 +30,6 @@ from server.utils.logger import LOGGING_HANDLERS logger = logging.getLogger(__name__) -logger = logging.getLogger(__name__) - def setup_storage_path(): default_path = join(expanduser("~"), '.faraday/storage') @@ -105,6 +104,19 @@ def default_login_required(): user = User.query.filter_by(id=session["user_id"]).first() g.user = user + @app.after_request + def log_queries_count(response): + queries = get_debug_queries() + max_query_time = max(q.duration for q in queries) + if len(queries) > 15: + logger.warn("Too many queries done (%s) in endpoint %s. " + "Maximum query time: %.2f", + len(queries), flask.request.endpoint, max_query_time) + # from collections import Counter + # print '\n\n\n'.join( + # map(str,Counter(q.statement for q in queries).most_common())) + return response + def create_app(db_connection_string=None, testing=None): app = Flask(__name__) @@ -115,6 +127,7 @@ def create_app(db_connection_string=None, testing=None): app.config['SECURITY_POST_LOGIN_VIEW'] = '/_api/session' app.config['SECURITY_POST_LOGOUT_VIEW'] = '/_api/login' app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False + app.config['SQLALCHEMY_RECORD_QUERIES'] = True # app.config['SQLALCHEMY_ECHO'] = True app.config['SECURITY_PASSWORD_SCHEMES'] = [ 'bcrypt', # This should be the default value From 553bfd07d5ccb7ed8b8491674568a615f73a08da Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 3 Jan 2018 11:02:36 -0300 Subject: [PATCH 0715/1506] Fix a bug when ex was raised sometimes answer was undefined --- persistence/server/server.py | 7 +++++-- persistence/server/server_io_exceptions.py | 5 +++-- plugins/manager.py | 2 -- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/persistence/server/server.py b/persistence/server/server.py index 9809fafadc7..58d68320805 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -25,6 +25,7 @@ import os import json +import logging try: import urlparse @@ -47,6 +48,7 @@ # NOTE: Change is you want to use this module by itself. # If FARADAY_UP is False, SERVER_URL must be a valid faraday server url +logger = logging.getLogger(__name__) FARADAY_UP = True SERVER_URL = "http://127.0.0.1:5985" AUTH_USER = "" @@ -175,6 +177,7 @@ def _unsafe_io_with_server(server_io_function, server_expected_response, Return the response from the server. """ + answer = None try: answer = server_io_function(server_url, **payload) if answer.status_code == 409: @@ -185,11 +188,11 @@ def _unsafe_io_with_server(server_io_function, server_expected_response, raise Unauthorized(answer) if answer.status_code != server_expected_response: raise requests.exceptions.RequestException(response=answer) - except requests.exceptions.RequestException: + except requests.exceptions.RequestException as ex: + logger.debug(ex) raise CantCommunicateWithServerError(server_io_function, server_url, payload, answer) return answer - def _parse_json(response_object): """Takes a response object and return its response as a dictionary.""" try: diff --git a/persistence/server/server_io_exceptions.py b/persistence/server/server_io_exceptions.py index 6d78ec979b4..6c432294cb0 100644 --- a/persistence/server/server_io_exceptions.py +++ b/persistence/server/server_io_exceptions.py @@ -21,16 +21,17 @@ def __str__(self): class CantCommunicateWithServerError(ServerRequestException): - def __init__(self, function, server_url, payload, response): + def __init__(self, function, server_url, payload, response=None): self.function = function self.server_url = server_url self.payload = payload self.response = response def __str__(self): + response_text = self.response and self.response.text or '' return ("Couldn't get a valid response from the server when requesting " "to URL {0} and function {1}. Response was {2}".format(self.server_url, - self.function, self.response.text)) + self.function, response_text)) class ConflictInDatabase(ServerRequestException): def __init__(self, answer): diff --git a/plugins/manager.py b/plugins/manager.py index 612560f985e..7d34b538467 100644 --- a/plugins/manager.py +++ b/plugins/manager.py @@ -108,8 +108,6 @@ def _loadPlugins(self, plugin_repo_path): module_filename, traceback.format_exc()) getLogger(self).debug(msg) getLogger(self).warn(e) - else: - pass def getPlugins(self): plugins = self._instancePlugins() From 853a78658a503bf96174089fcebd5bf74a45d375 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 3 Jan 2018 11:02:57 -0300 Subject: [PATCH 0716/1506] If no queries use 0 as max value --- server/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/app.py b/server/app.py index bfbfddc8929..0bb202b75cc 100644 --- a/server/app.py +++ b/server/app.py @@ -107,7 +107,7 @@ def default_login_required(): @app.after_request def log_queries_count(response): queries = get_debug_queries() - max_query_time = max(q.duration for q in queries) + max_query_time = max([q.duration for q in queries] or [0]) if len(queries) > 15: logger.warn("Too many queries done (%s) in endpoint %s. " "Maximum query time: %.2f", From 2641529ad19af95eaad208166a618fb804a99c4b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 3 Jan 2018 12:17:01 -0300 Subject: [PATCH 0717/1506] Use username and password from user.xml --- faraday.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/faraday.py b/faraday.py index 07135b03379..91feebca763 100755 --- a/faraday.py +++ b/faraday.py @@ -617,10 +617,8 @@ def main(): setConf() checkCouchUrl() checkVersion() - + CONF = getInstanceConfiguration() if args.login: - CONF = getInstanceConfiguration() - if not CONF.getServerURI(): couchURI = raw_input("Enter the Faraday server [http://127.0.0.1:5985]: ") or "http://127.0.0.1:5985" @@ -632,6 +630,10 @@ def main(): sys.exit(-1) doLoginLoop() + else: + session_cookie = login_user(CONF.getServerURI(), CONF.getAPIUsername(), CONF.getAPIPassword()) + if session_cookie: + CONF.setDBSessionCookies(session_cookie) check_faraday_version() From 30d0cfe757a367d7a53d59ffd5ad9f611b89f699 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 3 Jan 2018 12:17:32 -0300 Subject: [PATCH 0718/1506] Remove unused and empty method --- gui/gtk/application.py | 2 -- managers/workspace_manager.py | 7 +------ 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/gui/gtk/application.py b/gui/gtk/application.py index cfd3b9ff45f..e8355cb2c97 100644 --- a/gui/gtk/application.py +++ b/gui/gtk/application.py @@ -415,14 +415,12 @@ def reload_workspaces_no_connection(self): """Very similar to reload_workspaces, but doesn't resource the workspace_manager to avoid asking for information to a database we can't access.""" - self.workspace_manager.closeWorkspace() self.ws_sidebar.clear_sidebar() def reload_workspaces(self): """Close workspace, resources the workspaces available, clears the sidebar of the old workspaces and injects all the new ones in there too""" - self.workspace_manager.closeWorkspace() self.ws_sidebar.clear_sidebar() self.ws_sidebar.refresh_sidebar() diff --git a/managers/workspace_manager.py b/managers/workspace_manager.py index ca4dbf76725..811ecb7bc15 100644 --- a/managers/workspace_manager.py +++ b/managers/workspace_manager.py @@ -54,7 +54,6 @@ def createWorkspace(self, name, desc, start_date=int(time.time() * 1000), "http://john:password@127.0.0.1:5984")) except Exception as e: raise WorkspaceException(str(e)) - self.closeWorkspace() self.mappersManager.createMappers(name) self.setActiveWorkspace(workspace) notification_center.workspaceChanged(workspace) @@ -67,7 +66,7 @@ def openWorkspace(self, name): if name not in get_workspaces_names(): raise WorkspaceException( "Workspace %s wasn't found" % name) - self.closeWorkspace() + try: workspace = get_workspace(name) except Unauthorized: @@ -85,10 +84,6 @@ def openWorkspace(self, name): notification_center.workspaceChanged(workspace) return workspace - def closeWorkspace(self): - # TODO: DELETE - pass - def removeWorkspace(self, name): if name in self.getWorkspacesNames(): try: From 5fcf7895499223eee5685cd75a6a64078a21e466 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 3 Jan 2018 12:18:01 -0300 Subject: [PATCH 0719/1506] Add filename, fileno and funcname to logs --- utils/logs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/logs.py b/utils/logs.py index eaebd99ef86..1df99c6e101 100644 --- a/utils/logs.py +++ b/utils/logs.py @@ -28,7 +28,7 @@ def setUpLogger(debug=False): logger.setLevel(level) formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + '%(asctime)s - %(name)s - %(levelname)s [%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s') ch = logging.StreamHandler() ch.setFormatter(formatter) From 61d1ceca10aacb7d1cf5b8517b615e5a0e365a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 3 Jan 2018 14:48:21 -0300 Subject: [PATCH 0720/1506] Fix dates in many web ui views The only server-side thing I modified is the command start date --- server/api/modules/commandsrun.py | 2 +- .../scripts/dashboard/partials/modal-services-by-host.html | 2 +- server/www/scripts/hosts/partials/list.html | 3 +-- server/www/scripts/services/partials/list.html | 3 +-- .../statusReport/partials/ui-grid/columns/datecolumn.html | 4 ++-- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/server/api/modules/commandsrun.py b/server/api/modules/commandsrun.py index 9e6fe616e76..a5282ea6341 100644 --- a/server/api/modules/commandsrun.py +++ b/server/api/modules/commandsrun.py @@ -32,7 +32,7 @@ def load_itime(self, value): return datetime.datetime.fromtimestamp(value) def get_itime(self, obj): - return time.mktime(obj.start_date.utctimetuple()) + return time.mktime(obj.start_date.utctimetuple()) * 1000 def get_duration(self, obj): if obj.end_date and obj.start_date: diff --git a/server/www/scripts/dashboard/partials/modal-services-by-host.html b/server/www/scripts/dashboard/partials/modal-services-by-host.html index a5b9690870c..0298a553bb0 100644 --- a/server/www/scripts/dashboard/partials/modal-services-by-host.html +++ b/server/www/scripts/dashboard/partials/modal-services-by-host.html @@ -34,7 +34,7 @@

    Services for {{name}} ({{services.length}} total)

    Operating System: {{host.os}}
    -
    Created
    +
    Created
    diff --git a/server/www/scripts/hosts/partials/list.html b/server/www/scripts/hosts/partials/list.html index e42a8a3b1e2..9f024608a87 100644 --- a/server/www/scripts/hosts/partials/list.html +++ b/server/www/scripts/hosts/partials/list.html @@ -102,8 +102,7 @@

    not yet - - + diff --git a/server/www/scripts/services/partials/list.html b/server/www/scripts/services/partials/list.html index 4181c6b9fb0..56345fd6f7a 100644 --- a/server/www/scripts/services/partials/list.html +++ b/server/www/scripts/services/partials/list.html @@ -217,8 +217,7 @@

    Description

    Created - - +
    diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/datecolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/datecolumn.html index 538f42d16eb..5dbb32839aa 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/datecolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/datecolumn.html @@ -1,5 +1,5 @@
    -
    {{row.entity.metadata.create_time * 1000 | date:"MM/dd/yyyy"}} +
    {{row.entity.metadata.create_time | date:"MM/dd/yyyy"}}
    -
    {{row.entity.metadata.create_time.split(" ")[0] * 1000 | date:"MM/dd/yyyy"}}
    +
    {{row.entity.metadata.create_time.split(" ")[0] | date:"MM/dd/yyyy"}}
    From 8a226e2416f604bad8af51fb2d98e9e58d76a56a Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 3 Jan 2018 15:10:13 -0300 Subject: [PATCH 0721/1506] Delete unused code --- server/api/modules/doc.py | 104 ----------------- server/app.py | 2 - server/dao/__init__.py | 4 - server/dao/base.py | 40 ------- server/dao/command.py | 69 ------------ server/dao/credential.py | 91 --------------- server/dao/host.py | 138 ----------------------- server/dao/service.py | 115 ------------------- server/dao/vuln.py | 216 ----------------------------------- server/dao/workspace.py | 105 ----------------- server/database.py | 230 -------------------------------------- 11 files changed, 1114 deletions(-) delete mode 100644 server/api/modules/doc.py delete mode 100644 server/dao/__init__.py delete mode 100644 server/dao/base.py delete mode 100644 server/dao/command.py delete mode 100644 server/dao/credential.py delete mode 100644 server/dao/host.py delete mode 100644 server/dao/service.py delete mode 100644 server/dao/vuln.py delete mode 100644 server/dao/workspace.py delete mode 100644 server/database.py diff --git a/server/api/modules/doc.py b/server/api/modules/doc.py deleted file mode 100644 index 08293557a10..00000000000 --- a/server/api/modules/doc.py +++ /dev/null @@ -1,104 +0,0 @@ -# Faraday Penetration Test IDE -# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) -# See the file 'doc/LICENSE' for the license information -import json - -import flask -from flask import Blueprint - -import server.database -import server.utils.logger - -from server.utils.web import ( - validate_workspace, - build_bad_request_response, - get_basic_auth -) -from server.couchdb import get_user_from_session -from restkit.errors import RequestFailed, ResourceError - -logger = server.utils.logger.get_logger(__name__) -doc_api = Blueprint('doc_api', __name__) - - -@doc_api.route('/ws//doc/', methods=['GET']) -def get_document(workspace, doc_id): - validate_workspace(workspace) - ws = server.database.get(workspace) - couchdb_conn = ws.couchdb - response = couchdb_conn.get_document(doc_id) - return flask.jsonify(response) - - -@doc_api.route('/ws//doc/', methods=['PUT']) -def add_or_update_document(workspace, doc_id): - validate_workspace(workspace) - - try: - document = json.loads(flask.request.data) - except ValueError: - return build_bad_request_response('invalid json') - - document['_id'] = doc_id # document dictionary does not have id, add it - ws = server.database.get(workspace) - couchdb_conn = ws.couchdb - is_update_request = bool(document.get('_rev', False)) - - # change user in metadata based on session information - user = get_user_from_session(flask.request.cookies, get_basic_auth()) - if 'owner' in document.get('metadata', {}): - document['metadata']['owner'] = user - if 'update_user' in document.get('metadata', {}): - document['metadata']['update_user'] = user - - try: - response = couchdb_conn.save_doc(document) - except RequestFailed as e: - response = flask.jsonify(json.loads(e.msg)) - response.status_code = e.status_int - return response - except ResourceError as e: - response = flask.jsonify({'error': e.message}) - response.status_code = e.status_int - return response - - if response.get('ok', False): - doc_importer = server.database.DocumentImporter(ws.connector) - if is_update_request: - doc_importer.update_entity_from_doc(document) - else: - doc_importer.add_entity_from_doc(document) - - return flask.jsonify(response) - - -@doc_api.route('/ws//doc/', methods=['DELETE']) -def delete_document_and_children(workspace, doc_id): - - def delete_document(doc_id, doc_rev): - try: - response = couchdb_conn.delete_doc({'_id': doc_id, '_rev': doc_rev}) - - except RequestFailed as e: - response = flask.jsonify(json.loads(e.msg)) - response.status_code = e.status_int - return response - except ResourceError as e: - response = flask.jsonify({'error': e.message}) - response.status_code = e.status_int - return response - if response.get('ok', False): - doc_importer = server.database.DocumentImporter(ws.connector) - doc_importer.delete_entity_from_doc_id(doc_id) - - return flask.jsonify(response) - - validate_workspace(workspace) - ws = server.database.get(workspace) - couchdb_conn = ws.couchdb - docs_to_delete = couchdb_conn.get_documents_starting_with_id(doc_id) - docs_ids_to_delete = filter(lambda x: x is not None, (map(lambda d: d.get('id'), docs_to_delete))) - docs_revs_to_delete = map(lambda d: d['doc']['_rev'], docs_to_delete) - responses = map(delete_document, docs_ids_to_delete, docs_revs_to_delete) - - return responses[0] diff --git a/server/app.py b/server/app.py index c66f8e0bb95..7963c4248de 100644 --- a/server/app.py +++ b/server/app.py @@ -52,7 +52,6 @@ def register_blueprints(app): from server.api.modules.info import info_api from server.api.modules.commandsrun import commandsrun_api from server.api.modules.credentials import credentials_api - from server.api.modules.doc import doc_api from server.api.modules.hosts import host_api from server.api.modules.licenses import license_api from server.api.modules.services import services_api @@ -64,7 +63,6 @@ def register_blueprints(app): from server.api.modules.comments import comment_api app.register_blueprint(commandsrun_api) app.register_blueprint(credentials_api) - app.register_blueprint(doc_api) app.register_blueprint(host_api) app.register_blueprint(info_api) app.register_blueprint(license_api) diff --git a/server/dao/__init__.py b/server/dao/__init__.py deleted file mode 100644 index 50a8239b3f9..00000000000 --- a/server/dao/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Faraday Penetration Test IDE -# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) -# See the file 'doc/LICENSE' for the license information - diff --git a/server/dao/base.py b/server/dao/base.py deleted file mode 100644 index 1cbf0bf7fe8..00000000000 --- a/server/dao/base.py +++ /dev/null @@ -1,40 +0,0 @@ -# Faraday Penetration Test IDE -# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) -# See the file 'doc/LICENSE' for the license information - -import server.database -import server.utils.logger - -from server.models import db, Workspace, EntityMetadata - - -class FaradayDAO(object): - MAPPED_ENTITY = None - COLUMNS_MAP = {} - - def __init__(self, workspace): - self._logger = server.utils.logger.get_logger(self) - self._session = db.session - self._couchdb = None - self.workspace = workspace - if not getattr(workspace, 'name', None): - self.workspace = db.session.query(Workspace).filter_by(name=workspace) - - def get_all(self): - self.__check_valid_operation() - return self._session.query(self.MAPPED_ENTITY).filter_by(workspace=self.workspace) - - def __check_valid_operation(self): - if self.MAPPED_ENTITY is None: - raise RuntimeError('Invalid operation') - - def get_by_couchdb_id(self, couchdb_id): - self.__check_valid_operation() - query = self._session.query(self.MAPPED_ENTITY)\ - .join(EntityMetadata)\ - .filter(EntityMetadata.couchdb_id == couchdb_id) - return query.one() - - def save(self, obj): - self._session.add(obj) - self._session.commit() diff --git a/server/dao/command.py b/server/dao/command.py deleted file mode 100644 index 9ef1ec1ddd7..00000000000 --- a/server/dao/command.py +++ /dev/null @@ -1,69 +0,0 @@ -# Faraday Penetration Test IDE -# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) -# See the file 'doc/LICENSE' for the license information - -from sqlalchemy.orm.query import Bundle - -from server.dao.base import FaradayDAO -from server.models import Command, EntityMetadata -from server.utils.database import apply_search_filter, paginate - - -class CommandDAO(FaradayDAO): - MAPPED_ENTITY = Command - COLUMNS_MAP = { - 'couchid': [EntityMetadata.couchdb_id] - } - STRICT_FILTERING = ["couchid"] - - def list(self, search=None, page=0, page_size=0, command_filter={}): - results = self.__query_database(search, page, page_size, command_filter) - - rows = [self.__get_command_data(result.command) for result in results] - - result = { - 'commands': rows - } - - return result - - def __query_database(self, search=None, page=0, page_size=0, command_filter={}): - command_bundle = Bundle('command', - Command.itime, - Command.ip, - Command.hostname, - Command.command, - Command.user, - Command.duration, - Command.workspace_id, - Command.params, - EntityMetadata.couchdb_id) - - query = self._session.query(command_bundle)\ - .outerjoin(EntityMetadata, EntityMetadata.id == Command.entity_metadata_id) - - # Apply filtering options to the query - query = query.filter(Command.workspace == self.workspace) - query = apply_search_filter(query, self.COLUMNS_MAP, None, command_filter, self.STRICT_FILTERING) - - if page_size: - query = paginate(query, page, page_size) - - results = query.all() - - return results - - def __get_command_data(self, command): - return { - 'id': command.couchdb_id, - 'key': command.couchdb_id, - 'value': { - "_id": command.couchdb_id, - "itime": command.itime, - "ip": command.ip, - "hostname": command.hostname, - "command": command.command, - "user": command.user, - "workspace": command.workspace_id, - "duration": command.duration, - "params": command.params}} diff --git a/server/dao/credential.py b/server/dao/credential.py deleted file mode 100644 index 3d8c123dccd..00000000000 --- a/server/dao/credential.py +++ /dev/null @@ -1,91 +0,0 @@ -# Faraday Penetration Test IDE -# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) -# See the file 'doc/LICENSE' for the license information - -from sqlalchemy.orm.query import Bundle -from sqlalchemy import not_ - -from server.dao.base import FaradayDAO -from server.models import Credential, EntityMetadata -from server.utils.database import apply_search_filter - - -class CredentialDAO(FaradayDAO): - - MAPPED_ENTITY = Credential - - COLUMNS_MAP = { - 'couchid': [EntityMetadata.couchdb_id], - 'username': [Credential.username], - 'password': [Credential.password], - 'service_id': [], - 'host_id': []} - - STRICT_FILTERING = ["couchid"] - - def list(self, search=None, cred_filter={}): - results = self.__query_database(search, cred_filter) - - rows = [self.__get_cred_data(result.cred) for result in results] - result = { - 'rows': rows - } - - return result - - def __query_database(self, search=None, cred_filter={}): - creds_bundle = Bundle( - 'cred', Credential.username, Credential.password, - Credential.name, EntityMetadata.couchdb_id, - Credential.description, Credential.owned, - EntityMetadata.revision, EntityMetadata.update_time, - EntityMetadata.update_user, EntityMetadata.create_time, - EntityMetadata.update_action, EntityMetadata.creator, - EntityMetadata.update_controller_action, EntityMetadata.owner, - EntityMetadata.command_id) - - query = self._session.query(creds_bundle)\ - .outerjoin(EntityMetadata, EntityMetadata.id == Credential.entity_metadata_id) - - query = query.filter(Credential.workspace == self.workspace) - # Apply filtering options to the query - query = apply_search_filter(query, self.COLUMNS_MAP, search, cred_filter, self.STRICT_FILTERING) - - # I apply a custom filter for search by hostId and serviceId. - # 'LIKE' for search by serviceId.%, that return only credentials started with serviceId. - if cred_filter.get('service_id') is not None: - query = query.filter(EntityMetadata.couchdb_id.like(cred_filter.get('service_id') + ".%")) - - # 'LIKE' for search by hostId.%, with that LIKE we receive credentials of services also. - # I need another like for filter credentials of services (%.%.%) - if cred_filter.get('host_id') is not None: - query = query.filter( - EntityMetadata.couchdb_id.like(cred_filter.get('host_id') + ".%")).filter( - not_(EntityMetadata.couchdb_id.like("%.%.%"))) - - results = query.all() - return results - - def __get_cred_data(self, cred): - return { - 'id': cred.couchdb_id, - 'key': cred.couchdb_id, - 'value': { - '_id': cred.couchdb_id, - 'username': cred.username, - 'password': cred.password, - 'owner': cred.owner, - 'owned': cred.owned, - 'description': cred.description, - 'name': cred.name, - 'metadata': { - 'update_time': cred.update_time, - 'update_user': cred.update_user, - 'update_action': cred.update_action, - 'creator': cred.creator, - 'create_time': cred.create_time, - 'update_controller_action': cred.update_controller_action, - 'owner': cred.owner, - 'command_id': cred.command_id - }, - 'couchid': cred.couchdb_id}} diff --git a/server/dao/host.py b/server/dao/host.py deleted file mode 100644 index b90e55ef08d..00000000000 --- a/server/dao/host.py +++ /dev/null @@ -1,138 +0,0 @@ -# Faraday Penetration Test IDE -# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) -# See the file "doc/LICENSE" for the license information - -from server.dao.base import FaradayDAO -from server.utils.database import ( - paginate, - sort_results, - apply_search_filter, - get_count -) - -from sqlalchemy import distinct -from sqlalchemy.orm.query import Bundle -from sqlalchemy.sql import func -from server.models import ( - Host, - Service, - Vulnerability, - EntityMetadata, - Credential -) - - -class HostDAO(FaradayDAO): - - MAPPED_ENTITY = Host - - COLUMNS_MAP = { - "couchid": [EntityMetadata.couchdb_id], - "ip": [Host.ip], - "service": [Service.name], - "services": ["open_services_count"], - "vulns": ["vuln_count"], - "os": [Host.os], - "owned": [Host.owned], - "command_id": [EntityMetadata.command_id], - } - - STRICT_FILTERING = ["service", "couchid", "command_id"] - - def list(self, search=None, page=0, page_size=0, order_by=None, order_dir=None, host_filter={}): - - results, count = self.__query_database(search, page, page_size, order_by, order_dir, host_filter) - rows = [self.__get_host_data(result.host) for result in results] - - result = { - "total_rows": count, - "rows": rows - } - - return result - - def __query_database(self, search=None, page=0, page_size=0, order_by=None, order_dir=None, host_filter={}): - - host_bundle = Bundle( - "host", Host.id, Host.ip, Host.os, Host.description, Host.owned, - Host.default_gateway_ip, Host.default_gateway_mac, - EntityMetadata.couchdb_id, EntityMetadata.revision, - EntityMetadata.update_time, EntityMetadata.update_user, - EntityMetadata.update_action, EntityMetadata.creator, - EntityMetadata.create_time, EntityMetadata.update_controller_action, - EntityMetadata.owner, EntityMetadata.command_id, - func.count(distinct(Vulnerability.id)).label("vuln_count"), - func.count(distinct(Service.id)).label("open_services_count"), - func.count(distinct(Credential.id)).label("credentials_count")) - - query = self._session.query(host_bundle)\ - .outerjoin(EntityMetadata, EntityMetadata.id == Host.entity_metadata_id)\ - .outerjoin(Vulnerability, Host.id == Vulnerability.host_id)\ - .outerjoin(Service, (Host.id == Service.host_id) & (Service.status.in_(("open", "running", "opened"))))\ - .outerjoin(Credential, (Credential.host_id == Host.id) ) \ - .filter(Credential.service_id == None) \ - .group_by(Host.id, EntityMetadata.id) - - query = query.filter(Host.workspace == self.workspace) - # Apply pagination, sorting and filtering options to the query - query = sort_results(query, self.COLUMNS_MAP, order_by, order_dir, default=Host.id) - query = apply_search_filter(query, self.COLUMNS_MAP, search, host_filter, self.STRICT_FILTERING) - count = get_count(query, count_col=Host.id) - - if page_size: - query = paginate(query, page, page_size) - - results = query.all() - - return results, count - - def __get_host_data(self, host): - - return { - "id": host.couchdb_id, - "key": host.couchdb_id, - "_id": host.id, - "value": { - "_id": host.couchdb_id, - "_rev": host.revision, - "ip": host.ip, - "os": host.os, - "owned": host.owned, - "owner": host.owner, - "description": host.description, - "default_gateway": [host.default_gateway_ip, host.default_gateway_mac], - "metadata": { - "update_time": host.update_time, - "update_user": host.update_user, - "update_action": host.update_action, - "creator": host.creator, - "create_time": host.create_time, - "update_controller_action": host.update_controller_action, - "owner": host.owner, - "command_id": host.command_id - }, - "vulns": host.vuln_count, - "services": host.open_services_count, - "credentials": host.credentials_count - }} - - def count(self, group_by=None): - total_count = self._session.query(func.count(Host.id)).filter_by(workspace=self.workspace).scalar() - - # Return total amount of services if no group-by field was provided - result_count = {"total_count": total_count} - if group_by is None: - return result_count - - # Otherwise return the amount of services grouped by the field specified - # Strict restriction is applied for this entity - if group_by not in ["ip", "os"]: - return None - - col = HostDAO.COLUMNS_MAP.get(group_by)[0] - query = self._session.query(col, func.count()).group_by(col) - res = query.all() - - result_count["groups"] = [{group_by: value, "count": count} for value, count in res] - - return result_count diff --git a/server/dao/service.py b/server/dao/service.py deleted file mode 100644 index f39cd32b2fc..00000000000 --- a/server/dao/service.py +++ /dev/null @@ -1,115 +0,0 @@ -# Faraday Penetration Test IDE -# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) -# See the file 'doc/LICENSE' for the license information - -from sqlalchemy import distinct -from sqlalchemy.sql import func -from sqlalchemy.orm.query import Bundle - -from server.dao.base import FaradayDAO -from server.models import ( - Host, - Service, - EntityMetadata, - Vulnerability, - Credential -) -from server.utils.database import apply_search_filter - - -class ServiceDAO(FaradayDAO): - MAPPED_ENTITY = Service - COLUMNS_MAP = { - "couchid": [EntityMetadata.couchdb_id], - "command_id": [EntityMetadata.command_id], - 'id': [Service.id], - "name": [Service.name], - "protocol": [Service.protocol], - "version": [Service.version], - "status": [Service.status], - "owned": [Service.owned], - "hostid": [Host.id], - "hostIdCouchdb": [] - } - - STRICT_FILTERING = ["couchid", 'id', 'hostid'] - - def list(self, service_filter={}): - service_bundle = Bundle('service', - Service.id, Service.name, Service.description, Service.protocol, - Service.status, Service.port, Service.version, Service.owned, - func.count(distinct(Vulnerability.id)).label('vuln_count'), EntityMetadata.couchdb_id,\ - EntityMetadata.revision, EntityMetadata.update_time, EntityMetadata.update_user,\ - EntityMetadata.update_action, EntityMetadata.creator, EntityMetadata.create_time,\ - EntityMetadata.update_controller_action, EntityMetadata.owner, EntityMetadata.command_id, - func.count(distinct(Credential.id)).label("credentials_count")) - - query = self._session.query(service_bundle).\ - group_by(Service.id, EntityMetadata.id).\ - outerjoin(EntityMetadata, EntityMetadata.id == Service.entity_metadata_id).\ - outerjoin(Vulnerability, Service.id == Vulnerability.service_id).group_by(Service.id).\ - outerjoin(Credential, (Credential.service_id == Service.id) and (Credential.host_id == None)) - - query = query.filter(Service.workspace == self.workspace) - query = apply_search_filter(query, self.COLUMNS_MAP, None, service_filter, self.STRICT_FILTERING) - - # 'LIKE' for search services started by hostId.%.% - if service_filter.get('hostIdCouchdb') is not None: - query = query.filter( - EntityMetadata.couchdb_id.like(service_filter.get('hostIdCouchdb') + ".%.%")) - - raw_services = query.all() - services = [self.__get_service_data(r.service) for r in raw_services] - result = {'services': services} - return result - - def __get_service_data(self, service): - return { - 'id': service.couchdb_id, - 'key': service.couchdb_id, - '_id': service.id, - 'value': { - '_id': service.couchdb_id, - '_rev': service.revision, - 'name': service.name, - 'description': service.description, - 'metadata': { - 'update_time': service.update_time, - 'update_user': service.update_user, - 'update_action': service.update_action, - 'creator': service.creator, - 'create_time': service.create_time, - 'update_controller_action': service.update_controller_action, - 'owner': service.owner, - 'command_id': service.command_id - }, - 'protocol': service.protocol, - 'status': service.status, - 'port': [int(i) for i in service.port.split(',') if service.port], - 'version': service.version, - 'owned': service.owned, - 'owner': service.owner, - 'credentials': service.credentials_count - }, - 'vulns': service.vuln_count} - - def count(self, group_by=None): - total_count = self._session.query(func.count(Service.id)).filter_by(workspace=self.workspace).scalar() - - # Return total amount of services if no group-by field was provided - if group_by is None: - return {'total_count': total_count} - - # Otherwise return the amount of services grouped by the field specified - if group_by not in ServiceDAO.COLUMNS_MAP: - return None - - col = ServiceDAO.COLUMNS_MAP.get(group_by)[0] - query = self._session.query(col, func.count())\ - .filter(Service.status.in_(('open', 'running')))\ - .group_by(col) - - res = query.all() - - return {'total_count': total_count, - 'groups': [{group_by: value, 'count': count} for value, count in res]} diff --git a/server/dao/vuln.py b/server/dao/vuln.py deleted file mode 100644 index c91b28fe804..00000000000 --- a/server/dao/vuln.py +++ /dev/null @@ -1,216 +0,0 @@ -# Faraday Penetration Test IDE -# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) -# See the file 'doc/LICENSE' for the license information - -import json - -from server.dao.base import FaradayDAO -from server.utils.database import ( - GroupConcat, - paginate, - sort_results, - apply_search_filter, - get_count -) - -from sqlalchemy import case -from sqlalchemy.sql import func -from sqlalchemy.orm.query import Bundle -from server.models import ( - Host, - Service, - Vulnerability, - EntityMetadata -) - - - -class VulnerabilityDAO(FaradayDAO): - MAPPED_ENTITY = Vulnerability - COLUMNS_MAP = { - "couchid": [EntityMetadata.couchdb_id], - "id": [Vulnerability.id], - "date": [EntityMetadata.create_time], # TODO: fix search for this field - "confirmed": [Vulnerability.confirmed], - "name": [Vulnerability.name], - "severity": [Vulnerability.severity], - "service": [Service.port, Service.protocol, Service.name], - "target": [Host.ip], - "desc": [Vulnerability.description], - "resolution": [Vulnerability.resolution], - "data": [Vulnerability.data], - "owner": [EntityMetadata.owner], - "ease_of_resolution": [Vulnerability.ease_of_resolution], - "type": [EntityMetadata.document_type], - "status": [Vulnerability.status], - "tags": [], - "evidence": [], - "impact": [], - "hostid": [Host.id], - "serviceid": [Service.id], - "web": [], - "issuetracker": [], - "creator": [EntityMetadata.creator], - "command_id": [EntityMetadata.command_id] - } - - STRICT_FILTERING = ["type", "service", "couchid", "hostid", "serviceid", 'id', 'status', 'command_id'] - - def list(self, search=None, page=0, page_size=0, order_by=None, order_dir=None, vuln_filter={}): - results, count = self.__query_database(search, page, page_size, order_by, order_dir, vuln_filter) - vuln_list = [self.__get_vuln_data(v, s, h, hn) for v, s, h, hn in results] - - response = { - 'vulnerabilities': vuln_list, - 'count': count - } - - return response - - def __query_database(self, search=None, page=0, page_size=0, order_by=None, order_dir=None, vuln_filter={}): - # Instead of using SQLAlchemy ORM facilities to fetch rows, we bundle involved columns for - # organizational and MAINLY performance reasons. Doing it this way, we improve retrieving - # times from large workspaces almost 2x. - vuln_bundle = Bundle('vuln', Vulnerability.id.label('server_id'), Vulnerability.name.label('v_name'),\ - Vulnerability.confirmed, Vulnerability.data,\ - Vulnerability.description, Vulnerability.ease_of_resolution, Vulnerability.impact_accountability,\ - Vulnerability.impact_availability, Vulnerability.impact_confidentiality, Vulnerability.impact_integrity,\ - Vulnerability.resolution, Vulnerability.severity, Vulnerability.status,\ - EntityMetadata.couchdb_id, EntityMetadata.revision, EntityMetadata.create_time, EntityMetadata.creator,\ - EntityMetadata.owner, EntityMetadata.update_action, EntityMetadata.update_controller_action,\ - EntityMetadata.update_time, EntityMetadata.update_user, EntityMetadata.document_type, EntityMetadata.command_id, \ - Vulnerability.attachments) - service_bundle = Bundle('service', Service.name.label('s_name'), Service.port, Service.protocol, Service.id) - host_bundle = Bundle('host', Host.ip) - - # IMPORTANT: OUTER JOINS on those tables is IMPERATIVE. Changing them could result in loss of - # data. For example, on vulnerabilities not associated with any service and instead to its host - # directly. - query = self._session.query(vuln_bundle, - service_bundle, - host_bundle, - GroupConcat(Interface.hostnames))\ - .group_by(Vulnerability.id, EntityMetadata.id, Service.id, Host.id)\ - .outerjoin(EntityMetadata, EntityMetadata.id == Vulnerability.entity_metadata_id)\ - .outerjoin(Service, Service.id == Vulnerability.service_id)\ - .outerjoin(Host, Host.id == Vulnerability.host_id)\ - .filter(Vulnerability.workspace == self.workspace) - - # Apply pagination, sorting and filtering options to the query - query = self.__specialized_sort(query, order_by, order_dir) - query = apply_search_filter(query, self.COLUMNS_MAP, search, vuln_filter, self.STRICT_FILTERING) - count = get_count(query,count_col=Vulnerability.id) - - if page_size: - query = paginate(query, page, page_size) - - results = query.all() - - return results, count - - def __specialized_sort(self, query, order_by, order_dir): - """ Before using sort_results(), handle special ordering cases - for some fields """ - if order_by == 'severity': - # For severity only, we choose a risk-based ordering - # instead of a lexicographycally one - column_map = { - 'severity': [case( - {'unclassified': 0, - 'info': 1, - 'low': 2, - 'med': 3, - 'high': 4, - 'critical': 5}, - value=Vulnerability.severity - )] - } - else: - column_map = self.COLUMNS_MAP - - return sort_results(query, column_map, order_by, order_dir, default=Vulnerability.id) - - def __get_vuln_data(self, vuln, service, host, hostnames): - def get_own_id(couchdb_id): - return couchdb_id.split('.')[-1] - - def get_parent_id(couchdb_id): - return '.'.join(couchdb_id.split('.')[:-1]) - - return { - 'id': vuln.couchdb_id, - 'key': vuln.couchdb_id, - '_id': vuln.server_id, - 'value': { - '_id': vuln.couchdb_id, - '_rev': vuln.revision, - 'confirmed': vuln.confirmed, - 'data': vuln.data, - 'desc': vuln.description, - 'description': vuln.description, - 'ease_of_resolution': vuln.ease_of_resolution, - 'impact': { - 'accountability': vuln.impact_accountability, - 'availability': vuln.impact_availability, - 'confidentiality': vuln.impact_confidentiality, - 'integrity': vuln.impact_integrity - }, - 'issuetracker': {}, - 'metadata': { - 'create_time': vuln.create_time, - 'creator': vuln.creator, - 'owner': vuln.owner, - 'update_action': vuln.update_action, - 'update_controller_action': vuln.update_controller_action, - 'update_time': vuln.update_time, - 'update_user': vuln.update_user, - 'command_id': vuln.command_id - }, - '_attachments': json.loads(vuln.attachments), - 'name': vuln.v_name, - 'obj_id': get_own_id(vuln.couchdb_id), - 'owned': vuln.owned, - 'owner': vuln.owner, - 'parent': get_parent_id(vuln.couchdb_id), - 'status': vuln.status, - 'resolution': vuln.resolution, - 'severity': vuln.severity, - 'tags': [], - 'type': vuln.document_type, - 'target': Host.ip, - 'hostnames': hostnames.split(',') if hostnames else '', - 'service': "(%s/%s) %s" % (service.port, service.protocol, service.s_name) if service.port else '' - }} - - def count(self, group_by=None, search=None, vuln_filter={}): - query = self._session.query(Vulnerability.vuln_type, func.count())\ - .filter_by(workspace=self.workspace)\ - .group_by(Vulnerability.vuln_type) - query = apply_search_filter(query, self.COLUMNS_MAP, search, vuln_filter) - total_count = dict(query.all()) - - # Return total amount of services if no group-by field was provided - result_count = {'total_count': sum(total_count.values()), - 'web_vuln_count': total_count.get('VulnerabilityWeb', 0), - 'vuln_count': total_count.get('Vulnerability', 0)} - - if group_by is None: - return result_count - - # Otherwise return the amount of services grouped by the field specified - # Don't perform group-by counting on fields with less or more than 1 column mapped to it - if group_by not in VulnerabilityDAO.COLUMNS_MAP or\ - len(VulnerabilityDAO.COLUMNS_MAP.get(group_by)) != 1: - return None - - col = VulnerabilityDAO.COLUMNS_MAP.get(group_by)[0] - vuln_bundle = Bundle('vuln', col) - query = self._session.query(vuln_bundle, func.count())\ - .group_by(col) - - query = apply_search_filter(query, self.COLUMNS_MAP, search, vuln_filter, self.STRICT_FILTERING) - result = query.all() - - result_count['groups'] = [{group_by: value[1], 'count': count} for value, count in result] - - return result_count diff --git a/server/dao/workspace.py b/server/dao/workspace.py deleted file mode 100644 index 799018aded5..00000000000 --- a/server/dao/workspace.py +++ /dev/null @@ -1,105 +0,0 @@ -# Faraday Penetration Test IDE -# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) -# See the file "doc/LICENSE" for the license information - -from server.dao.base import FaradayDAO -from server.utils.database import ( - apply_search_filter, - get_count, - paginate -) -from server.utils import logger - -from sqlalchemy.orm.query import Bundle -from server.models import db, Workspace - - -class WorkspaceDAO(object): - - MAPPED_ENTITY = Workspace - - COLUMNS_MAP = { - "active": [Workspace.active], - "customer": [Workspace.customer], - "description": [Workspace.description], - "end_date": [Workspace.end_date], - "name": [Workspace.name], - "public": [Workspace.public], - # "scope": [Workspace.scope], - "start_date": [Workspace.start_date] - } - - def __init__(self): - self._logger = logger.get_logger(self) - self._session = db.session - self._couchdb = None - - def get_all(self): - self.__check_valid_operation() - return self._session.query(self.MAPPED_ENTITY) - - def __check_valid_operation(self): - if self.MAPPED_ENTITY is None: - raise RuntimeError('Invalid operation') - - def save(self, obj): - self._session.add(obj) - self._session.commit() - - def list(self, search=None, page=0, page_size=0, order_by=None, - order_dir=None, workspace_filter={}): - - results, count = self.__query_database(search, page, page_size, - order_by, order_dir, - workspace_filter) - rows = [self.__get_workspace_data(result.workspace) - for result in results] - - result = { - "total_rows": count, - "rows": rows - } - - return result - - def __query_database(self, search=None, page=0, page_size=0, - order_by=None, order_dir=None, workspace_filter={}): - - workspace_bundle = Bundle('workspace', - Workspace.active, - Workspace.customer, - Workspace.description, - Workspace.end_date, - Workspace.name, - Workspace.public, - # Workspace.scope, - Workspace.start_date, - ) - - STRICT_FILTERING = ["name", "customer", "disabled", - "public", "update_date"] - query = self._session.query(workspace_bundle) - - query = apply_search_filter(query, self.COLUMNS_MAP, search, - workspace_filter, self.STRICT_FILTERING) - count = get_count(query, count_col=Workspace.name) - - if page_size: - query = paginate(query, page, page_size) - - results = query.all() - - return results, count - - def __get_workspace_data(self, workspace): - - return { - "active": workspace.active, - "customer": workspace.customer, - "description": workspace.description, - "end_date": workspace.end_date, - "name": workspace.name, - "public": workspace.public, - # "scope": workspace.scope, - "start_date": workspace.start_date - } diff --git a/server/database.py b/server/database.py deleted file mode 100644 index 343b79b27cf..00000000000 --- a/server/database.py +++ /dev/null @@ -1,230 +0,0 @@ -# Faraday Penetration Test IDE -# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) -# See the file 'doc/LICENSE' for the license information -import server.models -import server.config -import server.couchdb -import server.utils.logger - -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm.exc import MultipleResultsFound - -logger = server.utils.logger.get_logger(__name__) - - -class DocumentImporter(object): - def __init__(self, db_conn, post_processing_change_cbk=None): - self.__db_conn = db_conn - self.__db_conf = Configuration(db_conn) - self.__post_processing_change_cbk = post_processing_change_cbk - - def add_entity_from_doc(self, document): - """ - ISSUES: - * Other entities related to this new document may be not already - include into the database (ie: these documents are added on future - changes) - """ - entity = server.models.FaradayEntity.parse(self.__db_conn.session, document) - if entity is None: - return False - - entity.add_relationships_from_db(self.__db_conn.session) - self.__db_conn.session.add(entity) - - try: - self.__db_conn.session.commit() - logger.info(u'New {} ({}) was added in Workspace {}'.format( - entity.entity_metadata.document_type, - getattr(entity, 'name', ''), - self.__db_conn.db_name)) - - except IntegrityError, e: - # For now, we silently rollback because it is an excepted - # scenario when we create documents from the server and its - # change notification arrives - self.__db_conn.session.rollback() - return False - - return True - - def delete_entity_from_doc_id(self, document_id): - """ - ISSUES: - * Delete child entities. Have not found cases where this is a problem. So far, - clients are deleting all CouchDBs documents properly, and if they don't, the - DBs still are consistent. Maybe use SQLAlchemy's cascades if this become a - problem. Status: Somewhat OK - - * Doc ID maps to multiple elements. This could happen since the ID is a hash - based in a few entity's properties which can be replicated. Status: TODO - """ - entity = self.__get_modified_entity(document_id) - if entity is not None: - self.__db_conn.session.delete(entity) - self.__db_conn.session.commit() - logger.info(u'A {} ({}) was deleted in Workspace {}'.format( - entity.entity_metadata.document_type, - getattr(entity, 'name', ''), - self.__db_conn.db_name)) - return True - - logger.debug(u'Document ({}) was not present in database to delete'.format(document_id)) - return False - - def update_entity_from_doc(self, document): - """ - ISSUES: - * Updated relationships are not taken into account. Status: TODO - """ - entity = self.__get_modified_entity(document.get('_id')) - if entity is not None: - entity.update_from_document(document) - entity.entity_metadata.update_from_document(document) - self.__db_conn.session.commit() - logger.info(u'A {} ({}) was updated in Workspace {}'.format( - entity.entity_metadata.document_type, - getattr(entity, 'name', ''), - self.__db_conn.db_name)) - return True - - logger.debug(u'Document ({}) was not present in database to update'.format(document.get('_id'))) - return False - - def __get_modified_entity(self, document_id): - metadata = self.get_document_metadata(document_id) - if metadata is None: - logger.info(u'Doc {} was not found in the database'.format(document_id)) - return None - - # Obtain the proper table on which to perform the entity operation - entity_cls = server.models.FaradayEntity.get_entity_class_from_type( - metadata.document_type) - - # TODO(mrocha): Add error handling here when no or more than one entities where found. - entity = self.__db_conn.session.query(entity_cls)\ - .join(server.models.EntityMetadata)\ - .filter(server.models.EntityMetadata.couchdb_id == document_id)\ - .one() - return entity - - def get_document_metadata(self, document_id): - metadata = None - try: - metadata = self.__db_conn.session.query(server.models.EntityMetadata)\ - .filter(server.models.EntityMetadata.couchdb_id == document_id)\ - .one_or_none() - except MultipleResultsFound: - logger.warning(u'Multiple entities were found for doc {}.'\ - 'Ignoring change'.format(document_id)) - return metadata - - -class Configuration(object): - # TODO(mrocha): use enums in database metadata table - # instead of constants defined here - LAST_SEQ_CONFIG = 'last_seq' - MIGRATION_SUCCESS = 'migration' - SCHEMA_VERSION = 'version' - - def __init__(self, db_conn): - self.__db_conn = db_conn - - def setup_new_database(self, from_seq=0): - self.set_last_seq(from_seq) - self.set_migration_status(False) - self.set_schema_version() - - def get_last_seq(self): - config = self.__get_config(Configuration.LAST_SEQ_CONFIG) - if config is not None: - return int(config.value) - else: - return 0 - - def set_last_seq(self, last_seq): - self.__set_config(Configuration.LAST_SEQ_CONFIG, last_seq) - - - def was_migration_successful(self): - config = self.__get_config(Configuration.MIGRATION_SUCCESS) - return (config is not None and config.value == 'true') - - def get_schema_version(self): - config = self.__get_config(Configuration.SCHEMA_VERSION) - return (config.value if config is not None else None) - - def set_migration_status(self, was_successful): - self.__set_config(Configuration.MIGRATION_SUCCESS, 'true' if was_successful else 'false') - - def set_schema_version(self): - self.__set_config(Configuration.SCHEMA_VERSION, server.models.SCHEMA_VERSION) - - def __get_config(self, option): - try: - result = self.__db_conn.session\ - .query(server.models.DatabaseMetadata)\ - .filter(server.models.DatabaseMetadata.option == option)\ - .one_or_none() - - except MultipleResultsFound: - msg = u'Database {} should not have the option {} defined multiple times'.format(self.__db_conn.db_name, option) - logger.error(msg) - raise RuntimeError(msg) - - return result - - def __set_config(self, option, value): - config = self.__get_config(option) - if config is None: - config = server.models.DatabaseMetadata(option=option) - config.value = value - - self.__db_conn.session.merge(config) - self.__db_conn.session.commit() - - -# -# Profile queries performance on debug mode -# Debug utility extracted from http://docs.sqlalchemy.org/en/latest/faq/performance.html -# -if server.config.is_debug_mode(): - from sqlalchemy import event - from sqlalchemy.engine import Engine - import time - - @event.listens_for(Engine, "before_cursor_execute") - def before_cursor_execute(conn, cursor, statement, parameters, context, executemany): - context._query_start_time = time.time() - logger.debug(u"Start Query:\n{}".format(statement)) - logger.debug(u"Parameters:\n{!r}".format(parameters)) - - @event.listens_for(Engine, "after_cursor_execute") - def after_cursor_execute(conn, cursor, statement, - parameters, context, executemany): - total = time.time() - context._query_start_time - logger.debug(u"Query Complete. Total Time: {:.02f}ms".format(total*1000)) - - -# -# Exception definitions -# -class WorkspaceNotFound(Exception): - def __init__(self, workspace_name): - super(WorkspaceNotFound, self).__init__('Workspace "%s" not found' % workspace_name) - - -# -# Connection to a database common to all workspaces -# - -def setup_common(db_path='sqlite:////tmp/test.db'): - common_engine = create_engine(db_path) - common_session = scoped_session(sessionmaker(autocommit=False, - autoflush=False, - bind=common_engine)) - server.models.CommonBase.metadata.bind = common_engine - from server.models import CommonBase - CommonBase.metadata.create_all(bind=common_engine) - CommonBase.query = common_session.query_property() - return common_session From 498950c4ac095da2c968b1599cee648fd99ecccc Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 3 Jan 2018 15:18:22 -0300 Subject: [PATCH 0722/1506] Verify login with username. add name to generated admin --- persistence/server/server.py | 2 +- server/commands/initdb.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/persistence/server/server.py b/persistence/server/server.py index dbc37784999..4d63f59ac53 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -1499,7 +1499,7 @@ def is_authenticated(uri, cookies): resp = requests.get(uri + "/_api/session", cookies=cookies, timeout=1) if resp.status_code != 403: user_info = resp.json() - return bool(user_info.get('name', {})) + return bool(user_info.get('username', {})) else: return False except requests.adapters.ConnectionError: diff --git a/server/commands/initdb.py b/server/commands/initdb.py index ff44622a2b8..82e6e0728e0 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -114,7 +114,7 @@ def run(self): def _create_admin_user(self, conn_string): engine = create_engine(conn_string) random_password = self.generate_random_pw(12) - engine.execute("INSERT INTO \"user\" (username, password, is_ldap, active) VALUES ('admin', '{0}', false, true);".format(random_password)) + engine.execute("INSERT INTO \"user\" (username, name, password, is_ldap, active) VALUES ('admin', 'Administrator', '{0}', false, true);".format(random_password)) print("Admin username created with {red} password{white}: {random_password}".format(random_password=random_password, white=Fore.WHITE, red=Fore.RED)) From c1eb08eca57d4ae7aa7a1b5e5f1565dd7034c745 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 3 Jan 2018 15:18:45 -0300 Subject: [PATCH 0723/1506] Host uses ip instead of name --- server/events.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/server/events.py b/server/events.py index 5c996449cd0..2788207a91c 100644 --- a/server/events.py +++ b/server/events.py @@ -14,11 +14,12 @@ def new_object_event(mapper, connection, instance): # Since we don't have jet a model for workspace we # retrieve the name from the connection string workspace_name = repr(connection.engine).split('.')[0].split('/')[-1] + name = getattr(instance, 'ip', None) or instance.name msg = { 'id': instance.id, 'action': 'CREATE', 'type': instance.__class__.__name__, - 'name': instance.name, + 'name': name, 'workspace': workspace_name } changes_queue.put(msg) @@ -26,11 +27,12 @@ def new_object_event(mapper, connection, instance): def delete_object_event(mapper, connection, instance): workspace_name = repr(connection.engine).split('.')[0].split('/')[-1] + name = getattr(instance, 'ip', None) or instance.name msg = { 'id': instance.id, 'action': 'DELETE', 'type': instance.__class__.__name__, - 'name': instance.name, + 'name': name, 'workspace': workspace_name } changes_queue.put(msg) @@ -38,11 +40,12 @@ def delete_object_event(mapper, connection, instance): def update_object_event(mapper, connection, instance): workspace_name = repr(connection.engine).split('.')[0].split('/')[-1] + name = getattr(instance, 'ip', None) or instance.name msg = { 'id': instance.id, 'action': 'UPDATE', 'type': instance.__class__.__name__, - 'name': instance.name, + 'name': name, 'workspace': workspace_name } changes_queue.put(msg) From fe1a7629eb96614f4433073fad87d7d0b9cb3fbb Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 3 Jan 2018 15:19:11 -0300 Subject: [PATCH 0724/1506] Code cleanup: remove unused imports and remove invalid imports --- server/importer.py | 1 - test_cases/test_server.py | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/server/importer.py b/server/importer.py index 5b20bc45793..99224eba600 100644 --- a/server/importer.py +++ b/server/importer.py @@ -25,7 +25,6 @@ import server.config import server.couchdb -import server.database import server.models import server.utils.logger from server.models import ( diff --git a/test_cases/test_server.py b/test_cases/test_server.py index 6c5b85bed2a..f143cf0a169 100644 --- a/test_cases/test_server.py +++ b/test_cases/test_server.py @@ -1,14 +1,11 @@ import os import sys import unittest -import tempfile import pytest sys.path.append(os.path.abspath(os.getcwd())) -from server.app import create_app -from flask_security import Security, SQLAlchemyUserDatastore + from server.models import db, User, Role -from server.database import setup_common def endpoint(): From 014c455990c6af4f1a2730fab8cbc7c9ef976d8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 3 Jan 2018 15:21:52 -0300 Subject: [PATCH 0725/1506] Allow sorting by metadata fields (update time, create time, etc) --- server/api/base.py | 10 ++++++++++ test_cases/test_api_hosts.py | 24 +++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/server/api/base.py b/server/api/base.py index 8254ed0a8cf..d312904f0cb 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -308,6 +308,16 @@ def _get_order_field(self, **kwargs): # Check that the field is in the schema to prevent unwanted fields # value leaking schema = self._get_schema_instance(kwargs) + + # Add metadata nested field + try: + metadata_field = schema.fields.pop('metadata') + except KeyError: + pass + else: + for (key, value) in metadata_field.target_schema.fields.items(): + schema.fields['metadata.' + key] = value + try: field_instance = schema.fields[order_field] except KeyError: diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index f40c570e75d..c1ad25e8964 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -475,6 +475,28 @@ def test_sort_by_services(self, test_client, session, second_workspace, assert res.status_code == 200 assert [h['_id'] for h in res.json['data']] == expected_ids + @pytest.mark.usefixtures("mock_envelope_list") + def test_sort_by_update_time(self, test_client, session, second_workspace, + host_factory): + """ + This test doesn't test only the hosts view, but all the ones that + expose a object with metadata. + Think twice if you are thinking in removing it + """ + expected = host_factory.create_batch(10, workspace=second_workspace) + session.commit() + for i in range(len(expected)): + if i % 2 == 0: # Only update some hosts + host = expected.pop(0) + host.description = 'i was updated' + session.add(host) + session.commit() + expected.append(host) # Put it on the end + res = test_client.get(self.url(workspace=second_workspace) + + '?sort=metadata.update_time&sort_dir=asc') + assert res.status_code == 200, res.data + assert [h['_id'] for h in res.json['data']] == [h.id for h in expected] + def test_create_a_host_twice_returns_conflict(self, test_client): res = test_client.post(self.url(), data={ "ip": "127.0.0.1", @@ -526,4 +548,4 @@ def test_create_host_cant_assign_command_from_another_workspace(self, test_clien assert res.status_code == 400 assert res.json == {u'message': u'Command not found.'} - assert len(command.command_objects) == 0 \ No newline at end of file + assert len(command.command_objects) == 0 From e802243c972a1ff40fb5c6909380189a75f989f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 3 Jan 2018 16:40:23 -0300 Subject: [PATCH 0726/1506] Fix page numbers in hosts widget of the dashboard --- server/www/scripts/dashboard/controllers/compoundCtrl.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/dashboard/controllers/compoundCtrl.js b/server/www/scripts/dashboard/controllers/compoundCtrl.js index 3aebabeddf2..f4ea518a25b 100644 --- a/server/www/scripts/dashboard/controllers/compoundCtrl.js +++ b/server/www/scripts/dashboard/controllers/compoundCtrl.js @@ -34,7 +34,7 @@ angular.module('faradayApp') var loadHosts = function() { hostsManager.getHosts( - $scope.workspace, $scope.currentPage - 1, + $scope.workspace, $scope.currentPage, $scope.pageSize, $scope.expression, $scope.sortField, $scope.sortDirection) .then(function(batch) { @@ -144,4 +144,4 @@ angular.module('faradayApp') dashboardSrv.registerCallback(loadHosts); init(); - }]); \ No newline at end of file + }]); From dbcbbafa8eed5e01738dae94de7f76b1f543dd6d Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 3 Jan 2018 17:28:43 -0300 Subject: [PATCH 0727/1506] Fix: services api returns an int on ports field --- persistence/server/models.py | 7 ++++++- test_cases/plugins/test_burp.py | 2 +- test_cases/test_managers_mapper_manager.py | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index 4c387cf43d4..17361ba7a46 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -879,7 +879,12 @@ class Service(ModelBase): def __init__(self, service, workspace_name): ModelBase.__init__(self, service, workspace_name) self.protocol = service['protocol'] - self.ports = [int(port) for port in service['ports']] + if type(service['ports']) == int: + # the new api returns an integer in ports + self.ports = [service['ports']] + else: + # plugin creates a list of strings with the ports + self.ports = map(int, service['ports']) self.version = service['version'] self.status = service['status'] self.vuln_amount = int(service.get('vulns', 0)) diff --git a/test_cases/plugins/test_burp.py b/test_cases/plugins/test_burp.py index fb76613f59d..f70214adcf2 100644 --- a/test_cases/plugins/test_burp.py +++ b/test_cases/plugins/test_burp.py @@ -57,7 +57,7 @@ def test_Plugin_creates_adecuate_objects(self, monkeypatch): assert len(actions[2038]) == 14 assert all('http' == name for name in map(lambda service: service.name, actions[20008])) - assert all([80] == ports for ports in map(lambda service: service.ports, actions[20008])) + assert all([80] == ports for ports in map(lambda service: service.getPorts(), actions[20008])) assert all('tcp' == protocol for protocol in map(lambda service: service.protocol, actions[20008])) assert all('open' for status in map(lambda service: service.status, actions[20008])) diff --git a/test_cases/test_managers_mapper_manager.py b/test_cases/test_managers_mapper_manager.py index cd514d8fcdf..0075041817b 100644 --- a/test_cases/test_managers_mapper_manager.py +++ b/test_cases/test_managers_mapper_manager.py @@ -80,7 +80,7 @@ 'owned': False, 'owner': 'leo', 'protocol': 'tcp', - 'ports': [60], + 'ports': 60, 'version': '2', 'status': 'open', 'vulns': 0, @@ -521,7 +521,7 @@ "id": 1, "credentials": 0, "_id": 1, - "ports": [21], + "ports": 21, "name": "ssh" }, 'serialized_expected_results': { From b7ad1c39ee8462c348cbf91f7732dea2f607aeae Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 3 Jan 2018 17:29:40 -0300 Subject: [PATCH 0728/1506] Remove unused parameter since --- persistence/server/models.py | 3 +-- persistence/server/server.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index 17361ba7a46..45d9c04f75c 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -177,8 +177,7 @@ def get_changes_stream(workspace_name): of name workspace_name. The change stream will have heartbeet set to 1000. """ - since = server.get_workspace(workspace_name)['last_seq'] - return server.get_changes_stream(workspace_name, since=since, + return server.get_changes_stream(workspace_name, heartbeat='1000') diff --git a/persistence/server/server.py b/persistence/server/server.py index 4d63f59ac53..5fafa8125fb 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -526,11 +526,11 @@ def _websockets_changes(workspace_name, **extra_params): # cha cha cha chaaaanges! -def get_changes_stream(workspace_name, since=0, heartbeat='1000', stream_provider=_websockets_changes, **extra_params): +def get_changes_stream(workspace_name, heartbeat='1000', stream_provider=_websockets_changes, **extra_params): """ stream_provider: A function that returns an instance of a Stream provider """ - return stream_provider(workspace_name, since=since, feed='continuous', + return stream_provider(workspace_name, feed='continuous', heartbeat=heartbeat, **extra_params) def get_workspaces_names(): From 6867129a83f20403b215b4a07cd32e150e2ad120 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 3 Jan 2018 17:30:05 -0300 Subject: [PATCH 0729/1506] We are not using sqlite db per workspace --- server/events.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/server/events.py b/server/events.py index 2788207a91c..35429c09bb8 100644 --- a/server/events.py +++ b/server/events.py @@ -13,40 +13,37 @@ def new_object_event(mapper, connection, instance): # Since we don't have jet a model for workspace we # retrieve the name from the connection string - workspace_name = repr(connection.engine).split('.')[0].split('/')[-1] name = getattr(instance, 'ip', None) or instance.name msg = { 'id': instance.id, 'action': 'CREATE', 'type': instance.__class__.__name__, 'name': name, - 'workspace': workspace_name + 'workspace': instance.workspace.name } changes_queue.put(msg) def delete_object_event(mapper, connection, instance): - workspace_name = repr(connection.engine).split('.')[0].split('/')[-1] name = getattr(instance, 'ip', None) or instance.name msg = { 'id': instance.id, 'action': 'DELETE', 'type': instance.__class__.__name__, 'name': name, - 'workspace': workspace_name + 'workspace': instance.workspace.name } changes_queue.put(msg) def update_object_event(mapper, connection, instance): - workspace_name = repr(connection.engine).split('.')[0].split('/')[-1] name = getattr(instance, 'ip', None) or instance.name msg = { 'id': instance.id, 'action': 'UPDATE', 'type': instance.__class__.__name__, 'name': name, - 'workspace': workspace_name + 'workspace': instance.workspace.name } changes_queue.put(msg) From 296c040b61c2146d5a377cd12101556b8febdb8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 3 Jan 2018 18:12:34 -0300 Subject: [PATCH 0730/1506] Add search option in hosts API --- RELEASE.md | 1 + server/api/modules/hosts.py | 14 ++++++++++++++ test_cases/test_api_hosts.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index c23a478c926..08867217105 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -17,6 +17,7 @@ TBA: * Communication object was removed and replaced with Comment * Show credentials count in summarized report on the dashboard * Remove vuln template CWE fields, join it with references +* Allow to search hosts by hostname November 17, 2017: --- diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 506913652e7..0197eebf8a3 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -118,6 +118,20 @@ def _update_object(self, obj, data): return super(HostsView, self)._update_object(obj, data) + def _filter_query(self, query): + query = super(HostsView, self)._filter_query(query) + search_term = flask.request.args.get('search', None) + if search_term is not None: + like_term = '%' + search_term + '%' + match_ip = Host.ip.ilike(like_term) + match_service_name = Host.services.any( + Service.name.ilike(like_term)) + match_hostname = Host.hostnames.any(Hostname.name.ilike(like_term)) + query = query.filter(match_ip | + match_service_name | + match_hostname) + return query + def _envelope_list(self, objects, pagination_metadata=None): hosts = [] for host in objects: diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index c1ad25e8964..4bd43cb8cf1 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -296,6 +296,41 @@ def test_filter_by_service(self, test_client, session, workspace, expected_host_ids = set(host.id for host in hosts) assert shown_hosts_ids == expected_host_ids + def test_search_ip(self, test_client, session, workspace, host_factory): + host = host_factory.create(ip="longname", + workspace=workspace) + session.commit() + res = test_client.get(self.url() + '?search=ONGNAM') + assert res.status_code == 200 + assert len(res.json['rows']) == 1 + assert res.json['rows'][0]['id'] == host.id + + @pytest.mark.usefixtures('host_services') + def test_search_service_name(self, test_client, session, workspace, + service_factory): + expected_hosts = [self.hosts[2], self.hosts[4]] + for host in expected_hosts: + service_factory.create(host=host, name="GOPHER 5", + workspace=workspace) + session.commit() + res = test_client.get(self.url() + '?search=gopher') + assert res.status_code == 200 + shown_hosts_ids = set(obj['id'] for obj in res.json['rows']) + expected_host_ids = set(host.id for host in expected_hosts) + assert shown_hosts_ids == expected_host_ids + + @pytest.mark.usefixtures('host_with_hostnames') + def test_search_by_hostname(self, test_client, session, workspace): + expected_hosts = [self.hosts[2], self.hosts[4]] + for host in expected_hosts: + host.set_hostnames(['staging.twitter.com']) + session.commit() + res = test_client.get(self.url() + '?search=twitter') + assert res.status_code == 200 + shown_hosts_ids = set(obj['id'] for obj in res.json['rows']) + expected_host_ids = set(host.id for host in expected_hosts) + assert shown_hosts_ids == expected_host_ids + def test_host_with_open_vuln_count_verification(self, test_client, session, workspace, host_factory, vulnerability_factory, From 8cd8c7803968ee3b6df57c5b0f0766749fa6abdd Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 4 Jan 2018 12:17:04 -0300 Subject: [PATCH 0731/1506] last_seq removed --- server/couchdb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/couchdb.py b/server/couchdb.py index 5cf10c89708..8428e1acc59 100644 --- a/server/couchdb.py +++ b/server/couchdb.py @@ -157,7 +157,6 @@ def get_workspace(workspace_name, cookies, credentials): workspace = _get_workspace_doc(workspace_name, cookies, credentials).json() ws_info_url = get_couchdb_url() + ('/%s' % (workspace_name)) response = requests.get(ws_info_url, verify=False, cookies=cookies, auth=credentials) - workspace['last_seq'] = response.json()['update_seq'] return workspace From 177ede65a8039f0af7af998bdb775a43cb692848 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 4 Jan 2018 12:17:18 -0300 Subject: [PATCH 0732/1506] Avoid duplicate msg on websockets --- server/events.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/events.py b/server/events.py index 35429c09bb8..cc14955584e 100644 --- a/server/events.py +++ b/server/events.py @@ -37,6 +37,11 @@ def delete_object_event(mapper, connection, instance): def update_object_event(mapper, connection, instance): + delta = instance.update_date - instance.create_date + if delta.seconds < 30: + # sometimes apis will commit to db to have fk. + # this will avoid duplicate messages on websockets + return name = getattr(instance, 'ip', None) or instance.name msg = { 'id': instance.id, From c7733054ef948c464551813a4d4b5823376289f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 4 Jan 2018 12:29:22 -0300 Subject: [PATCH 0733/1506] Fix typo and trailing slash bug in workspaces view --- server/www/scripts/commons/providers/server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index 70fb560163d..0308ac49df3 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -57,7 +57,7 @@ angular.module("faradayApp") }; var createDbUrl = function(wsName = "") { - return APIURL + "ws/" + wsName; + return APIURL + "ws/" + wsName + (wsName ? "/" : ""); } var createDeleteUrl = createPutUrl; @@ -489,7 +489,7 @@ angular.module("faradayApp") ServerAPI.updateWorkspace = function(workspace) { var putUrl = createDbUrl(workspace.name); - return send_data(dbUrl, workspace, true, "PUT"); + return send_data(putUrl, workspace, true, "PUT"); } ServerAPI.deleteWorkspace = function(wsName) { From e5886eba61df6963d27f6b12b976ae31767b2d90 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 4 Jan 2018 12:51:38 -0300 Subject: [PATCH 0734/1506] Fix gtk shell --- persistence/server/server.py | 6 ++++-- persistence/server/utils.py | 1 + plugins/controller.py | 3 ++- test_cases/test_managers_mapper_manager.py | 7 +++++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/persistence/server/server.py b/persistence/server/server.py index 5fafa8125fb..cfeb88bd62a 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -1356,7 +1356,7 @@ def update_credential(workspace_name, command_id, id, name, username, password, password=password, type="Cred") -def create_command(workspace_name, command, duration=None, hostname=None, +def create_command(workspace_name, command, tool, duration=None, hostname=None, ip=None, itime=None, params=None, user=None): """Creates a command. @@ -1375,6 +1375,7 @@ def create_command(workspace_name, command, duration=None, hostname=None, """ return _save_to_server(workspace_name, command=command, + tool=tool, duration=duration, hostname=hostname, ip=ip, @@ -1384,7 +1385,7 @@ def create_command(workspace_name, command, duration=None, hostname=None, workspace=workspace_name, type="CommandRunInformation") -def update_command(workspace_name, command_id, command, duration=None, hostname=None, +def update_command(workspace_name, command_id, command, tool, duration=None, hostname=None, ip=None, itime=None, params=None, user=None): """Updates a command. @@ -1405,6 +1406,7 @@ def update_command(workspace_name, command_id, command, duration=None, hostname= return _update_in_server(workspace_name, command_id, command=command, + tool=tool, duration=duration, hostname=hostname, ip=ip, diff --git a/persistence/server/utils.py b/persistence/server/utils.py index 7957d221542..27ea17bcc1d 100644 --- a/persistence/server/utils.py +++ b/persistence/server/utils.py @@ -130,6 +130,7 @@ def get_credential_properties(credential): def get_command_properties(command): return { 'command': command.command, + 'tool': command.command, 'user': command.user, 'ip': command.ip, 'hostname': command.hostname, diff --git a/plugins/controller.py b/plugins/controller.py index 75aade523eb..623b5dee83f 100644 --- a/plugins/controller.py +++ b/plugins/controller.py @@ -186,7 +186,8 @@ def processCommandInput(self, pid, cmd, pwd): 'itime': time.time(), 'command': cmd.split()[0], 'params': ' '.join(cmd.split()[1:])}) - self._mapper_manager.save(cmd_info) + cmd_info.setID(self._mapper_manager.save(cmd_info)) + self._active_plugins[pid] = plugin, cmd_info return plugin.id, modified_cmd_string diff --git a/test_cases/test_managers_mapper_manager.py b/test_cases/test_managers_mapper_manager.py index 0075041817b..fed8b9a4f23 100644 --- a/test_cases/test_managers_mapper_manager.py +++ b/test_cases/test_managers_mapper_manager.py @@ -252,6 +252,7 @@ 'ip': "192.168.124.1", 'hostname': "mandarina", 'command': "Import Nexpose:", + 'tool': "Import Nexpose:", 'user': "leonardo", 'workspace': "airbnb", 'duration': 0.164561, @@ -260,6 +261,7 @@ }, 'expected_payload':{ 'command': 'Import Nexpose:', + 'tool': "Import Nexpose:", 'duration': 0.164561, 'hostname': 'mandarina', 'ip': '192.168.124.1', @@ -637,6 +639,7 @@ "ip": "192.168.20.53", "hostname": "mandarina", "command": "Import Nessus:", + "tool": "Import Nessus:", "user": "lcubo", "workspace": "dsadsa", "duration": "In progress", @@ -645,6 +648,7 @@ }, 'serialized_expected_results': { 'command': 'Import Nessus:', + "tool": "Import Nessus:", 'duration': 'In progress', 'hostname': 'mandarina', 'ip': '192.168.20.53', @@ -668,6 +672,9 @@ class TestMapperManager(): @pytest.mark.parametrize("obj_class, many_test_data", OBJ_DATA.items()) def test_save_without_command(self, obj_class, many_test_data, monkeypatch, session): + """ + This test verifies that the request made to the api are the expected ones + """ workspace = WorkspaceFactory.create(name='test') session.commit() mapper_manager = MapperManager() From 1be14761694d8eeb8d656ee3501e6db4441d9eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 4 Jan 2018 14:17:13 -0300 Subject: [PATCH 0735/1506] Editable workspace names --- RELEASE.md | 1 + .../www/scripts/commons/providers/server.js | 4 ++-- .../workspaces/controllers/workspaces.js | 10 ++++++---- .../workspaces/partials/modalEdit.html | 20 ++++++++++++++++++- .../workspaces/providers/workspaces.js | 4 ++-- 5 files changed, 30 insertions(+), 9 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 08867217105..d2ab6e8941d 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -18,6 +18,7 @@ TBA: * Show credentials count in summarized report on the dashboard * Remove vuln template CWE fields, join it with references * Allow to search hosts by hostname +* Workspace names can be changed from the Web UI November 17, 2017: --- diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index 0308ac49df3..3dc01b59e67 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -487,8 +487,8 @@ angular.module("faradayApp") return send_data(dbUrl, data, true, "POST"); } - ServerAPI.updateWorkspace = function(workspace) { - var putUrl = createDbUrl(workspace.name); + ServerAPI.updateWorkspace = function(workspace, wsName) { + var putUrl = createDbUrl(wsName || workspace.name); return send_data(putUrl, workspace, true, "PUT"); } diff --git a/server/www/scripts/workspaces/controllers/workspaces.js b/server/www/scripts/workspaces/controllers/workspaces.js index 5bd0cfa907b..edf209b5b08 100644 --- a/server/www/scripts/workspaces/controllers/workspaces.js +++ b/server/www/scripts/workspaces/controllers/workspaces.js @@ -77,7 +77,8 @@ angular.module('faradayApp') $scope.onSuccessEdit = function(workspace){ for(var i = 0; i < $scope.workspaces.length; i++) { - if($scope.workspaces[i].name == workspace.name){ + if($scope.workspaces[i]._id == workspace._id){ + $scope.workspaces[i].name = workspace.name; $scope.workspaces[i]._rev = workspace._rev; $scope.workspaces[i].description = workspace.description; if ($scope.workspaces[i].duration === undefined) @@ -117,7 +118,7 @@ angular.module('faradayApp') $scope.onFailInsert); }; - $scope.update = function(ws){ + $scope.update = function(ws, wsName){ if(typeof(ws.duration.start_date) == "number") { start_date = ws.duration.start_date; } else if(ws.duration.start_date) { @@ -140,7 +141,7 @@ angular.module('faradayApp') "scope": ws.scope, "type": ws.type }; - workspacesFact.update(workspace).then(function(workspace) { + workspacesFact.update(workspace, wsName).then(function(workspace) { $scope.onSuccessEdit(workspace); }); }; @@ -175,6 +176,7 @@ angular.module('faradayApp') }); if(workspace){ + var oldName = workspace.name; var modal = $uibModal.open({ templateUrl: 'scripts/workspaces/partials/modalEdit.html', controller: 'workspacesModalEdit', @@ -188,7 +190,7 @@ angular.module('faradayApp') modal.result.then(function(workspace) { if(workspace != undefined){ - $scope.update(workspace); + $scope.update(workspace, oldName); } }); } else { diff --git a/server/www/scripts/workspaces/partials/modalEdit.html b/server/www/scripts/workspaces/partials/modalEdit.html index c43ffe8de75..2389069f3f4 100644 --- a/server/www/scripts/workspaces/partials/modalEdit.html +++ b/server/www/scripts/workspaces/partials/modalEdit.html @@ -5,13 +5,31 @@
    - +

    Edit Workspace: {{workspace.name}}

    +
    +
    + + + +
    + +
    +     +
    +
    diff --git a/server/www/scripts/workspaces/providers/workspaces.js b/server/www/scripts/workspaces/providers/workspaces.js index c8c2e028cd2..79dfb44c797 100644 --- a/server/www/scripts/workspaces/providers/workspaces.js +++ b/server/www/scripts/workspaces/providers/workspaces.js @@ -87,9 +87,9 @@ angular.module('faradayApp') return ret; }; - workspacesFact.update = function(workspace) { + workspacesFact.update = function(workspace, wsName) { var deferred = $q.defer(); - ServerAPI.updateWorkspace(workspace).then(function(data){ + ServerAPI.updateWorkspace(workspace, wsName).then(function(data){ workspace._rev = data.rev; deferred.resolve(workspace); }); From db4204f76a2b77de00bc5c921fd1e32920f0493e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 4 Jan 2018 15:11:15 -0300 Subject: [PATCH 0736/1506] Make workspace scope writable in the API --- server/api/modules/hosts.py | 4 ---- server/api/modules/workspaces.py | 18 +++++++++++++++++- server/models.py | 13 +++++++++++-- test_cases/test_api_workspace.py | 31 +++++++++++++++++++++++++------ 4 files changed, 53 insertions(+), 13 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 0197eebf8a3..631037a6104 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -112,10 +112,6 @@ def _update_object(self, obj, data): else: obj.set_hostnames(hostnames) - # Required to make the assert pass. This actually makes two requests - # to the DB - db.session.commit() - return super(HostsView, self)._update_object(obj, data) def _filter_query(self, query): diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index 7b96360ff54..7aaa248720d 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -12,6 +12,7 @@ from server.utils.logger import get_logger from server.schemas import ( JSTimestampField, + MutableField, PrimaryKeyRelatedField, SelfNestedField, ) @@ -57,7 +58,10 @@ class WorkspaceSchema(AutoSchema): stats = SelfNestedField(WorkspaceSummarySchema()) duration = SelfNestedField(WorkspaceDurationSchema()) _id = fields.Integer(dump_only=True, attribute='id') - scope = PrimaryKeyRelatedField('name', many=True, dump_only=True) + scope = MutableField( + PrimaryKeyRelatedField('name', many=True, dump_only=True), + fields.List(fields.String) + ) class Meta: model = Workspace @@ -88,5 +92,17 @@ def _get_base_query(self): only_confirmed = False return Workspace.query_with_count(only_confirmed) + def _perform_create(self, data, **kwargs): + scope = data.pop('scope', []) + workspace = super(WorkspaceView, self)._perform_create(data, **kwargs) + workspace.set_scope(scope) + db.session.commit() + return workspace + + def _update_object(self, obj, data): + scope = data.pop('scope', []) + obj.set_scope(scope) + return super(WorkspaceView, self)._update_object(obj, data) + WorkspaceView.register(workspace_api) diff --git a/server/models.py b/server/models.py index dbc68c95e0f..c16b8880e56 100644 --- a/server/models.py +++ b/server/models.py @@ -235,7 +235,8 @@ def set_hostnames(self, new_hostnames): child_field='name') -def set_children_objects(instance, value, parent_field, child_field='id'): +def set_children_objects(instance, value, parent_field, child_field='id', + workspaced=True): """ Override some kind of children of instance. This is useful in one to many relationships. It takes care of deleting not used children, @@ -245,6 +246,7 @@ def set_children_objects(instance, value, parent_field, child_field='id'): :param value: list of childs (values of the child_field) :param parent_field: the parent field's relationship to the children name :param child_field: the "lookup field" of the children model + :param workspaced: indicates if the parent model has a workspace """ # Get the class of the children. Inspired in # https://stackoverflow.com/questions/6843144/how-to-find-sqlalchemy-remote-side-objects-class-or-class-name-without-db-queri @@ -269,7 +271,8 @@ def set_children_objects(instance, value, parent_field, child_field='id'): # it already exists continue kwargs = {child_field: new_child} - kwargs['workspace'] = instance.workspace + if workspaced: + kwargs['workspace'] = instance.workspace current_value.append(children_model(**kwargs)) @@ -1087,6 +1090,12 @@ def query_with_count(cls, only_confirmed): ), ) + def set_scope(self, new_scope): + return set_children_objects(self, new_scope, + parent_field='scope', + child_field='name', + workspaced=False) + class Scope(Metadata): __tablename__ = 'scope' diff --git a/test_cases/test_api_workspace.py b/test_cases/test_api_workspace.py index ed20a41cebc..40db0400270 100644 --- a/test_cases/test_api_workspace.py +++ b/test_cases/test_api_workspace.py @@ -1,7 +1,7 @@ import time import pytest -from server.models import Workspace +from server.models import Workspace, Scope from server.api.modules.workspaces import WorkspaceView from test_cases.test_api_non_workspaced_base import ReadWriteAPITests from test_cases import factories @@ -104,10 +104,29 @@ def test_create_with_description(self, session, test_client): assert res.json['description'] == description def test_create_with_scope(self, session, test_client): - description = 'darkside' - raw_data = {'name': 'something', 'description': description} - workspace_count_previous = session.query(Workspace).count() + desired_scope = [ + 'www.google.com', + '127.0.0.1' + ] + raw_data = {'name': 'something', 'description': 'test', + 'scope': desired_scope} res = test_client.post('/v2/ws/', data=raw_data) assert res.status_code == 201 - assert workspace_count_previous + 1 == session.query(Workspace).count() - assert res.json['description'] == description \ No newline at end of file + assert set(res.json['scope']) == set(desired_scope) + workspace = Workspace.query.get(res.json['id']) + assert set(s.name for s in workspace.scope) == set(desired_scope) + + def test_update_with_scope(self, session, test_client, workspace): + session.add(Scope(name='test.com', workspace=workspace)) + session.add(Scope(name='www.google.com', workspace=workspace)) + desired_scope = [ + 'www.google.com', + '127.0.0.1' + ] + raw_data = {'name': 'something', 'description': 'test', + 'scope': desired_scope} + res = test_client.put('/v2/ws/{}/'.format(workspace.name), + data=raw_data) + assert res.status_code == 200 + assert set(res.json['scope']) == set(desired_scope) + assert set(s.name for s in workspace.scope) == set(desired_scope) From 3d0769535b95b0200ccb815e1c7138f3f3abc932 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 4 Jan 2018 16:23:44 -0300 Subject: [PATCH 0737/1506] Change scope field widget in edit workspace modal --- .../scripts/workspaces/controllers/workspaces.js | 3 +++ .../workspaces/controllers/workspacesModalEdit.js | 7 ++++++- .../scripts/workspaces/partials/modalEdit.html | 15 +++++++++------ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/server/www/scripts/workspaces/controllers/workspaces.js b/server/www/scripts/workspaces/controllers/workspaces.js index edf209b5b08..a5191b5b0ec 100644 --- a/server/www/scripts/workspaces/controllers/workspaces.js +++ b/server/www/scripts/workspaces/controllers/workspaces.js @@ -55,6 +55,9 @@ angular.module('faradayApp') $scope.onSuccessGet = function(workspace){ workspace.selected = false; + workspace.scope = workspace.scope.map(function(scope){ + return {key: scope} + }); $scope.workspaces.push(workspace); }; diff --git a/server/www/scripts/workspaces/controllers/workspacesModalEdit.js b/server/www/scripts/workspaces/controllers/workspacesModalEdit.js index 9d0c1bb8d97..bd4cb167da9 100644 --- a/server/www/scripts/workspaces/controllers/workspacesModalEdit.js +++ b/server/www/scripts/workspaces/controllers/workspacesModalEdit.js @@ -25,6 +25,11 @@ angular.module('faradayApp') }; $scope.today(); + $scope.newScope = function($event){ + $scope.workspace.scope.push({key:''}); + $event.preventDefault(); + } + $scope.clear = function () { $scope.dt = null; }; @@ -45,4 +50,4 @@ angular.module('faradayApp') }; init(); - }]); \ No newline at end of file + }]); diff --git a/server/www/scripts/workspaces/partials/modalEdit.html b/server/www/scripts/workspaces/partials/modalEdit.html index 2389069f3f4..3022dc535a1 100644 --- a/server/www/scripts/workspaces/partials/modalEdit.html +++ b/server/www/scripts/workspaces/partials/modalEdit.html @@ -60,14 +60,17 @@
    End Date

    -
    +
    - - +
    Scope
    + Add scope target +
    +
    +
    + + +
    From f84b7d70966f9e4e9cf50cb45e5cbc31aef0e4bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 4 Jan 2018 16:42:49 -0300 Subject: [PATCH 0738/1506] Make workspace editing work with scope in the web ui --- .../scripts/workspaces/controllers/workspaces.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/workspaces/controllers/workspaces.js b/server/www/scripts/workspaces/controllers/workspaces.js index a5191b5b0ec..2d9d0dd4328 100644 --- a/server/www/scripts/workspaces/controllers/workspaces.js +++ b/server/www/scripts/workspaces/controllers/workspaces.js @@ -121,7 +121,7 @@ angular.module('faradayApp') $scope.onFailInsert); }; - $scope.update = function(ws, wsName){ + $scope.update = function(ws, wsName, finally_){ if(typeof(ws.duration.start_date) == "number") { start_date = ws.duration.start_date; } else if(ws.duration.start_date) { @@ -145,7 +145,10 @@ angular.module('faradayApp') "type": ws.type }; workspacesFact.update(workspace, wsName).then(function(workspace) { + if (finally_) finally_(workspace); $scope.onSuccessEdit(workspace); + }, function(workspace){ + if (finally_) finally_(workspace); }); }; @@ -193,7 +196,16 @@ angular.module('faradayApp') modal.result.then(function(workspace) { if(workspace != undefined){ - $scope.update(workspace, oldName); + + // The API expects list of strings in scope + var old_scope = workspace.scope; + workspace.scope = workspace.scope.map(function(scope){ + return scope.key + }).filter(Boolean); + + $scope.update(workspace, oldName, function(workspace){ + workspace.scope = old_scope; + }); } }); } else { From 3c223856d90848cb27add3f13ae9e714c2428b54 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 4 Jan 2018 17:17:17 -0300 Subject: [PATCH 0739/1506] Improve test_server_url and add unit tests to info api. Fix gtk url configuration dialog --- gui/gtk/application.py | 10 +++--- persistence/server/server.py | 7 ++-- test_cases/test_api_info.py | 21 +++++++++++ test_cases/test_persistence_server_server.py | 37 ++++++++++++++++++++ 4 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 test_cases/test_api_info.py create mode 100644 test_cases/test_persistence_server_server.py diff --git a/gui/gtk/application.py b/gui/gtk/application.py index 977b65532d0..c1e4dd9bf6f 100644 --- a/gui/gtk/application.py +++ b/gui/gtk/application.py @@ -336,7 +336,7 @@ def force_change_couch_url(self, button=None, dialog=None): preference_window.run() - def connect_to_couch(self, server_uri, parent=None): + def connect_to_couch(self, server_url, parent=None): """Tries to connect to a CouchDB on a specified Couch URI. Returns the success status of the operation, False for not successful, True for successful @@ -344,14 +344,14 @@ def connect_to_couch(self, server_uri, parent=None): if parent is None: parent = self.window - if not self.serverIO.test_server_url(server_uri): + if not self.serverIO.test_server_url(server_url): errorDialog(parent, "Could not connect to Faraday Server.", ("Are you sure it is running and that you can " "connect to it? \n Make sure your username and " "password are still valid.")) success = False - elif server_uri.startswith("https://"): - if not checkSSL(server_uri): + elif server_url.startswith("https://"): + if not checkSSL(server_url): errorDialog(self.window, "The SSL certificate validation has failed") success = False @@ -364,7 +364,7 @@ def connect_to_couch(self, server_uri, parent=None): "client you are runnung. Version numbers must match!") success = False return success - CONF.setCouchUri(server_uri) + CONF.setAPIUrl(server_url) CONF.saveConfig() self.reload_workspaces() self.open_last_workspace() diff --git a/persistence/server/server.py b/persistence/server/server.py index cfeb88bd62a..92bbac16b90 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -1528,9 +1528,10 @@ def test_server_url(url_to_test): False otherwise. """ try: - _get("{0}/v2/_api/info".format(url_to_test)) - test_okey = True - except: + resp = _get("{0}/_api/v2/info".format(url_to_test)) + return 'Faraday Server' in resp + except Exception as ex: + logger.exception(ex) test_okey = False return test_okey diff --git a/test_cases/test_api_info.py b/test_cases/test_api_info.py new file mode 100644 index 00000000000..d3c7c5b25e4 --- /dev/null +++ b/test_cases/test_api_info.py @@ -0,0 +1,21 @@ +import os +import pytest + + +@pytest.mark.usefixtures('logged_user') +class TestAPIInfoEndpoint: + + def test_api_info(self, test_client): + current_dir = os.path.dirname(os.path.realpath(__file__)) + # this is a bug on the info api! + # we require faraday to be a package since we can't import + # from base path when our current working dir is test_cases. + if 'test_cases' in current_dir: + faraday_base = os.path.join(current_dir, '..') + os.chdir(faraday_base) + + response = test_client.get('v2/info') + assert response.status_code == 200 + assert response.json['Faraday Server'] == 'Running' + # to avoid side effects + os.chdir(current_dir) \ No newline at end of file diff --git a/test_cases/test_persistence_server_server.py b/test_cases/test_persistence_server_server.py new file mode 100644 index 00000000000..66d807ae66f --- /dev/null +++ b/test_cases/test_persistence_server_server.py @@ -0,0 +1,37 @@ +import os + +import responses +import requests + +import pytest + +from persistence.server import server + + +@pytest.mark.usefixtures('logged_user') +class TestServerFuncions: + + @responses.activate + def test_test_server_url_server_runnimg(self, test_client): + """ + The name of the method is correct we are testing the function: + test_server_url + + """ + mocked_response = { + "Faraday Server": "Running", + "Version":"2.7.1" + } + + responses.add(responses.GET, 'http://localhost/_api/v2/info', + json=mocked_response, status=200) + assert server.test_server_url('http://localhost') + + @responses.activate + def test_test_server_url_another_http_returns_404(self, test_client): + responses.add(responses.GET, 'http://localhost/_api/v2/info', + status=404) + assert not server.test_server_url('http://localhost') + + def test_test_server_url_aserver_down(self, test_client): + assert not server.test_server_url('http://localhost') From 7fda5d07dd6349e983670635c60e5cef515e1910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 4 Jan 2018 18:19:26 -0300 Subject: [PATCH 0740/1506] Make workspace creation work with scope in the web ui --- .../scripts/workspaces/controllers/workspaces.js | 9 +++++++-- .../workspaces/controllers/workspacesModalNew.js | 9 +++++++-- .../www/scripts/workspaces/partials/modalNew.html | 13 ++++++++----- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/server/www/scripts/workspaces/controllers/workspaces.js b/server/www/scripts/workspaces/controllers/workspaces.js index 2d9d0dd4328..046c27550cc 100644 --- a/server/www/scripts/workspaces/controllers/workspaces.js +++ b/server/www/scripts/workspaces/controllers/workspaces.js @@ -167,7 +167,12 @@ angular.module('faradayApp') }); $scope.modal.result.then(function(workspace) { - workspace = $scope.create(workspace.name, workspace.description, workspace.start_date, workspace.end_date, workspace.scope); + // The API expects list of strings in scope + api_scope = workspace.scope.map(function(scope){ + return scope.key + }).filter(Boolean); + + workspace = $scope.create(workspace.name, workspace.description, workspace.start_date, workspace.end_date, api_scope); $scope.insert(workspace); }); @@ -272,7 +277,7 @@ angular.module('faradayApp') $scope.create = function(wname, wdesc, start_date, end_date, scope){ if(end_date) end_date = end_date.getTime(); else end_date = ""; if(start_date) start_date = start_date.getTime(); else start_date = ""; - workspace = { + var workspace = { "_id": wname, "customer": "", "name": wname, diff --git a/server/www/scripts/workspaces/controllers/workspacesModalNew.js b/server/www/scripts/workspaces/controllers/workspacesModalNew.js index df54d1b0039..a289839792d 100644 --- a/server/www/scripts/workspaces/controllers/workspacesModalNew.js +++ b/server/www/scripts/workspaces/controllers/workspacesModalNew.js @@ -14,7 +14,7 @@ angular.module('faradayApp') $scope.workspace = { "description": "", "name": "", - "scope": "" + "scope": [{key: ''}] }; }; @@ -35,6 +35,11 @@ angular.module('faradayApp') if(isStart) $scope.openedStart = true; else $scope.openedEnd = true; }; + $scope.newScope = function($event){ + $scope.workspace.scope.push({key:''}); + $event.preventDefault(); + } + $scope.okNew = function(){ $modalInstance.close($scope.workspace); }; @@ -44,4 +49,4 @@ angular.module('faradayApp') }; init(); - }]); \ No newline at end of file + }]); diff --git a/server/www/scripts/workspaces/partials/modalNew.html b/server/www/scripts/workspaces/partials/modalNew.html index 8196dd94adc..1ce320b44e6 100644 --- a/server/www/scripts/workspaces/partials/modalNew.html +++ b/server/www/scripts/workspaces/partials/modalNew.html @@ -71,11 +71,14 @@

    New Workspace

    - - +
    Scope
    + Add scope target +
    +
    +
    + + +
    From 5d0559d3cf8ad31cdec4a67e169eb71ac764d3fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 5 Jan 2018 12:02:55 -0300 Subject: [PATCH 0741/1506] Show a blank text input when editing a workspace with no scope --- server/www/scripts/workspaces/controllers/workspaces.js | 1 + 1 file changed, 1 insertion(+) diff --git a/server/www/scripts/workspaces/controllers/workspaces.js b/server/www/scripts/workspaces/controllers/workspaces.js index 046c27550cc..6a84c1adfb4 100644 --- a/server/www/scripts/workspaces/controllers/workspaces.js +++ b/server/www/scripts/workspaces/controllers/workspaces.js @@ -58,6 +58,7 @@ angular.module('faradayApp') workspace.scope = workspace.scope.map(function(scope){ return {key: scope} }); + if (workspace.scope.length == 0) workspace.scope.push({key: ''}); $scope.workspaces.push(workspace); }; From a787435a7513b85b7ece8d753c8ffc4fbd87d426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 5 Jan 2018 12:11:53 -0300 Subject: [PATCH 0742/1506] Fix editing scope of recently created workspaces --- server/www/scripts/workspaces/controllers/workspaces.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/www/scripts/workspaces/controllers/workspaces.js b/server/www/scripts/workspaces/controllers/workspaces.js index 6a84c1adfb4..fb2d7c1ddcf 100644 --- a/server/www/scripts/workspaces/controllers/workspaces.js +++ b/server/www/scripts/workspaces/controllers/workspaces.js @@ -64,6 +64,10 @@ angular.module('faradayApp') $scope.onSuccessInsert = function(workspace){ $scope.wss.push(workspace.name); + workspace.scope = workspace.scope.map(function(scope){ + return {key: scope} + }); + if (workspace.scope.length == 0) workspace.scope.push({key: ''}); $scope.workspaces.push(workspace); }; From 8b962ab4ca416d717adc5e8a2377caca2d01dad6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 5 Jan 2018 12:23:25 -0300 Subject: [PATCH 0743/1506] Keep alphabetical sort when creating workspaces --- server/www/scripts/workspaces/controllers/workspaces.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/www/scripts/workspaces/controllers/workspaces.js b/server/www/scripts/workspaces/controllers/workspaces.js index fb2d7c1ddcf..6df425c1437 100644 --- a/server/www/scripts/workspaces/controllers/workspaces.js +++ b/server/www/scripts/workspaces/controllers/workspaces.js @@ -68,7 +68,12 @@ angular.module('faradayApp') return {key: scope} }); if (workspace.scope.length == 0) workspace.scope.push({key: ''}); - $scope.workspaces.push(workspace); + + // Ulgy hack to keep the workspaces sorted alphabetically + for (var i = 0; i < $scope.workspaces.length; i++) { + if ($scope.workspaces[i].name > workspace.name) break; + } + $scope.workspaces.splice(i, 0, workspace); }; $scope.onFailInsert = function(error){ From 093d13c179ea61dfa207b0c063f6546fcea7ddce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 5 Jan 2018 13:16:55 -0300 Subject: [PATCH 0744/1506] Fix workspaces API bug The stats weren't shown after creating or updating a workspace. Tests success story: I overwritten the _dump method to re-query the object but I wasn't correctly handling the case when many objects are serialized (many=True). The test case testing listing objects via the API caught this error. --- server/api/modules/workspaces.py | 7 +++++++ test_cases/test_api_workspace.py | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index 7aaa248720d..884b33f5647 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -104,5 +104,12 @@ def _update_object(self, obj, data): obj.set_scope(scope) return super(WorkspaceView, self)._update_object(obj, data) + def _dump(self, obj, route_kwargs, **kwargs): + # When the object was created or updated it doesn't have the stats + # loaded so I have to query it again + if not kwargs.get('many') and obj.vulnerability_total_count is None: + obj = self._get_object(obj.name) + return super(WorkspaceView, self)._dump(obj, route_kwargs, **kwargs) + WorkspaceView.register(workspace_api) diff --git a/test_cases/test_api_workspace.py b/test_cases/test_api_workspace.py index 40db0400270..9998f6d6db4 100644 --- a/test_cases/test_api_workspace.py +++ b/test_cases/test_api_workspace.py @@ -103,6 +103,30 @@ def test_create_with_description(self, session, test_client): assert workspace_count_previous + 1 == session.query(Workspace).count() assert res.json['description'] == description + @pytest.mark.parametrize("stat_name", [ + 'credentials', 'services', 'web_vulns', 'code_vulns', 'std_vulns', + 'total_vulns' + ]) + def test_create_stat_is_zero(self, test_client, stat_name): + raw_data = {'name': 'something', 'description': ''} + res = test_client.post('/v2/ws/', data=raw_data) + assert res.status_code == 201 + assert res.json['stats'][stat_name] == 0 + + def test_update_stats(self, workspace, session, test_client, + vulnerability_factory, + vulnerability_web_factory): + vulnerability_factory.create_batch(10, workspace=workspace) + vulnerability_web_factory.create_batch(5, workspace=workspace) + session.commit() + raw_data = {'name': 'something', 'description': ''} + res = test_client.put('/v2/ws/{}/'.format(workspace.name), + data=raw_data) + assert res.status_code == 200 + assert res.json['stats']['web_vulns'] == 5 + assert res.json['stats']['std_vulns'] == 10 + assert res.json['stats']['total_vulns'] == 15 + def test_create_with_scope(self, session, test_client): desired_scope = [ 'www.google.com', From 269eb2466f04e04ae5457df6590840b2890e2abe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 5 Jan 2018 13:22:26 -0300 Subject: [PATCH 0745/1506] Show stats (objects count) after creating a workspace --- server/www/scripts/workspaces/controllers/workspaces.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/www/scripts/workspaces/controllers/workspaces.js b/server/www/scripts/workspaces/controllers/workspaces.js index 6df425c1437..f2711da5bdf 100644 --- a/server/www/scripts/workspaces/controllers/workspaces.js +++ b/server/www/scripts/workspaces/controllers/workspaces.js @@ -69,6 +69,8 @@ angular.module('faradayApp') }); if (workspace.scope.length == 0) workspace.scope.push({key: ''}); + $scope.objects[workspace.name] = workspace.stats; + // Ulgy hack to keep the workspaces sorted alphabetically for (var i = 0; i < $scope.workspaces.length; i++) { if ($scope.workspaces[i].name > workspace.name) break; @@ -126,6 +128,7 @@ angular.module('faradayApp') $scope.insert = function(workspace){ delete workspace.selected; workspacesFact.put(workspace).then(function(resp){ + workspace.stats = resp.data.stats; $scope.onSuccessInsert(workspace) }, $scope.onFailInsert); From fe9169e616776587a648579c1c071254c45853e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 5 Jan 2018 13:35:01 -0300 Subject: [PATCH 0746/1506] Fix workspace proggress widget on the dashboard A name conflict broke this in a previous commit --- server/www/scripts/workspaces/providers/workspaces.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/workspaces/providers/workspaces.js b/server/www/scripts/workspaces/providers/workspaces.js index 79dfb44c797..958401f8e6b 100644 --- a/server/www/scripts/workspaces/providers/workspaces.js +++ b/server/www/scripts/workspaces/providers/workspaces.js @@ -46,8 +46,8 @@ angular.module('faradayApp') var deferred = $q.defer(); ServerAPI.getWorkspace(workspace_name).then(function(workspace) { deferred.resolve({ - "start_date": workspace.data.duration.start_date, - "end_date": workspace.data.duration.end_date + "start": workspace.data.duration.start_date, + "end": workspace.data.duration.end_date }); }); return deferred.promise; From 963621bc3ccc9eacc2151ea637b717bc6728ec6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 5 Jan 2018 13:46:36 -0300 Subject: [PATCH 0747/1506] Add release information for #4459 --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index d2ab6e8941d..38f57dfd040 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -19,6 +19,7 @@ TBA: * Remove vuln template CWE fields, join it with references * Allow to search hosts by hostname * Workspace names can be changed from the Web UI +* Changed the scope field of a workspace from a free text input to a list of targets November 17, 2017: --- From d83acf1f7f8a9c4820d8f1e8e4a0fc050a15587c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 5 Jan 2018 18:16:08 -0300 Subject: [PATCH 0748/1506] Fix a problem when saving command end date --- managers/reports_managers.py | 8 +- plugins/controller.py | 94 +++++++++++++++----- plugins/plugin.py | 7 +- test_cases/test_config_configuration.py | 0 test_cases/test_persistence_server_server.py | 2 +- utils/common.py | 1 - 6 files changed, 82 insertions(+), 30 deletions(-) create mode 100644 test_cases/test_config_configuration.py diff --git a/managers/reports_managers.py b/managers/reports_managers.py index c0fe8c96902..22916551bc2 100755 --- a/managers/reports_managers.py +++ b/managers/reports_managers.py @@ -5,14 +5,15 @@ See the file 'doc/LICENSE' for the license information ''' -from threading import Thread + import os import re import time import logging import traceback -from multiprocessing import Process +from threading import Thread + from utils.logs import getLogger try: @@ -138,8 +139,9 @@ def syncReports(self): for filename in filenames: name = os.path.basename(filename) - # If plugin not is detected... move to unprocessed + # PluginCommiter will rename the file to processed or unprocessed + # when the plugin finishes if self.processor.processReport(filename) is False: logger.info('Plugin not detected. Moving {0} to unprocessed'.format(filename)) os.rename( diff --git a/plugins/controller.py b/plugins/controller.py index 623b5dee83f..60378b93027 100644 --- a/plugins/controller.py +++ b/plugins/controller.py @@ -7,17 +7,14 @@ See the file 'doc/LICENSE' for the license information ''' -import threading - import os import time -import Queue import shlex -import errno import logging -from multiprocessing import JoinableQueue -from Queue import Queue, Empty +from threading import Thread +from multiprocessing import JoinableQueue, Process +from config.configuration import getInstanceConfiguration from plugins.plugin import PluginProcess import model.api from model.commands_history import CommandRunInformation @@ -27,11 +24,54 @@ from config.globals import ( CONST_FARADAY_HOME_PATH, CONST_FARADAY_ZSH_OUTPUT_PATH) +CONF = getInstanceConfiguration() logger = logging.getLogger(__name__) -class PluginController(threading.Thread): +class PluginCommiter(Thread): + + def __init__(self, output_queue, output, pending_actions, plugin, command, mapper_manager): + super(PluginCommiter, self).__init__() + self.output_queue = output_queue + self.pending_actions = pending_actions + self.stop = False + self.plugin = plugin + self.command = command + self.mapper_manager = mapper_manager + self.output = output + self._report_path = os.path.join(CONF.getReportPath(), command.workspace) + self._report_ppath = os.path.join(self._report_path, "process") + self._report_upath = os.path.join(self._report_path, "unprocessed") + + def stop(self): + self.stop = True + + def commit(self): + logger.debug('Plugin end. Commiting to faraday server.') + self.pending_actions.put( + (modelactions.PLUGINEND, self.plugin.id, self.command.getID())) + self.command.duration = time.time() - self.command.itime + self.mapper_manager.update(self.command) + + def run(self): + try: + self.output_queue.join() + self.commit() + if os.path.isfile(self.output): + # sometimes output is a filepath + name = os.path.basename(self.output) + os.rename(self.output, + os.path.join(self._report_ppath, name)) + except Exception as ex: + logger.exception(ex) + logger.info('Something failed, moving file to unprocessed') + os.rename(self.output, + os.path.join(self._report_upath, name)) + + + +class PluginController(Thread): """ TODO: Doc string. """ @@ -102,7 +142,18 @@ def getAvailablePlugins(self): def stop(self): self.stop = True - def processOutput(self, plugin, output, command_id, isReport=False): + def processOutput(self, plugin, output, command, isReport=False): + """ + Process the output of the plugin. This will start the PluginProcess + and also PluginCommiter (thread) that will informa to faraday server + when the command finished. + + :param plugin: Plugin to execute + :param output: read output from plugin or term + :param command_id: command id that started the plugin + :param isReport: Report or output from shell + :return: None + """ output_queue = JoinableQueue() plugin.set_actions_queue(self.pending_actions) @@ -113,17 +164,21 @@ def processOutput(self, plugin, output, command_id, isReport=False): "Created plugin_process (%d) for plugin instance (%d)" % (id(plugin_process), id(plugin))) - # TODO: stop this processes + self.pending_actions.put((modelactions.PLUGINSTART, plugin.id, command.getID())) + + output_queue.put((output, command.getID())) + plugin_commiter = PluginCommiter( + output_queue, + output, + self.pending_actions, + plugin, + command, + self._mapper_manager + ) + plugin_commiter.start() + # This process is stopped when plugin commiter joins output queue plugin_process.start() - print('Plugin controller ', self.pending_actions) - self.pending_actions.put((modelactions.PLUGINSTART, plugin.id, command_id)) - - output_queue.put((output, command_id)) - output_queue.join() - - self.pending_actions.put((modelactions.PLUGINEND, plugin.id, command_id)) - def _processAction(self, action, parameters): """ decodes and performs the action given @@ -224,9 +279,8 @@ def processReport(self, plugin, filepath, ws_name=None): if plugin in self._plugins: logger.info('Processing report with plugin {0}'.format(plugin)) - self.processOutput(self._plugins[plugin], filepath, cmd_info.getID(), True) - cmd_info.duration = time.time() - cmd_info.itime - self._mapper_manager.update(cmd_info) + with open(filepath, 'rb') as output: + self.processOutput(self._plugins[plugin], output.read(), cmd_info, True) return True return False diff --git a/plugins/plugin.py b/plugins/plugin.py index e8c008224b7..8dc98ca138e 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -365,7 +365,7 @@ def __init__(self, plugin_instance, output_queue, isReport=False): :param output_queue: queue with raw ouput of that the plugin needs. :param isReport: output data was read from file. """ - Thread.__init__(self) + super(PluginProcess, self).__init__() self.output_queue = output_queue self.plugin = plugin_instance self.isReport = isReport @@ -385,10 +385,7 @@ def run(self): if output is not None: model.api.devlog('%s: %s' % (proc_name, "New Output")) try: - if self.isReport: - self.plugin.processReport(output) - else: - self.plugin.processOutput(output) + self.plugin.processOutput(output) except Exception as ex: model.api.devlog("Plugin raised an exception:") model.api.devlog(traceback.format_exc()) diff --git a/test_cases/test_config_configuration.py b/test_cases/test_config_configuration.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test_cases/test_persistence_server_server.py b/test_cases/test_persistence_server_server.py index 66d807ae66f..743e298bc81 100644 --- a/test_cases/test_persistence_server_server.py +++ b/test_cases/test_persistence_server_server.py @@ -28,7 +28,7 @@ def test_test_server_url_server_runnimg(self, test_client): assert server.test_server_url('http://localhost') @responses.activate - def test_test_server_url_another_http_returns_404(self, test_client): + def test_test_server_url_another_http_returns_404(self): responses.add(responses.GET, 'http://localhost/_api/v2/info', status=404) assert not server.test_server_url('http://localhost') diff --git a/utils/common.py b/utils/common.py index 0aad70a7396..ee3dc23b7d9 100644 --- a/utils/common.py +++ b/utils/common.py @@ -6,7 +6,6 @@ ''' import hashlib import uuid -import time import socket import struct import sys From a4c931c2b82a43a5fe19519705993d8e4bf487dc Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 5 Jan 2018 18:21:16 -0300 Subject: [PATCH 0749/1506] Add disable-polling parameter --- manage.py | 5 +++-- server/commands/reports.py | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/manage.py b/manage.py index 5e2a6c3fbd0..9c0163f87ea 100755 --- a/manage.py +++ b/manage.py @@ -29,14 +29,15 @@ def check_faraday_server(url): @click.command() @click.option('--debug/--no-debug', default=False) @click.option('--workspace', default=None) -def process_reports(debug, workspace): +@click.option('--disable-polling', default=False) +def process_reports(debug, workspace, disable_polling): setUpLogger(debug) configuration = _conf() url = '{0}/_api/v2/info'.format(configuration.getServerURI() if FARADAY_UP else SERVER_URL) with app.app_context(): try: check_faraday_server(url) - import_external_reports(workspace) + import_external_reports(workspace, disable_polling) except OperationalError as ex: print('{0}'.format(ex)) print('Please verify database is running or configuration on server.ini!') diff --git a/server/commands/reports.py b/server/commands/reports.py index 31c14f7013e..c1db8981c9c 100644 --- a/server/commands/reports.py +++ b/server/commands/reports.py @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) -def import_external_reports(workspace_name=None): +def import_external_reports(workspace_name=None, disable_polling=False): plugins_path = os.path.join(CONF.getConfigPath(), "plugins") plugin_manager = PluginManager(plugins_path) mappers_manager = MapperManager() @@ -27,11 +27,11 @@ def import_external_reports(workspace_name=None): else: query = Workspace.query - process_workspaces(mappers_manager, plugin_manager, query) + process_workspaces(mappers_manager, plugin_manager, query, disable_polling) #controller._pending_actions.join() -def process_workspaces(mappers_manager, plugin_manager, query): +def process_workspaces(mappers_manager, plugin_manager, query, disable_polling): report_managers = [] controllers = [] for workspace in query.all(): @@ -52,7 +52,7 @@ def process_workspaces(mappers_manager, plugin_manager, query): 0.1, workspace.name, plugin_controller, - polling=False + polling=not disable_polling ) report_managers.append(report_manager) report_manager.start() From e65ceec4655aa94556ceeb6287fe5b7108f29400 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 5 Jan 2018 18:30:11 -0300 Subject: [PATCH 0750/1506] Allow the user to choose wether to display the hostnames column --- server/www/scripts/hosts/controllers/hosts.js | 1 + server/www/scripts/hosts/partials/list.html | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/hosts/controllers/hosts.js b/server/www/scripts/hosts/controllers/hosts.js index 3bb77a5283e..8e543cd5c8e 100644 --- a/server/www/scripts/hosts/controllers/hosts.js +++ b/server/www/scripts/hosts/controllers/hosts.js @@ -9,6 +9,7 @@ angular.module('faradayApp') var init = function() { $scope.selectall_hosts = false; + $scope.showHostnames = true; // hosts list $scope.hosts = []; $scope.totalHosts = 0; diff --git a/server/www/scripts/hosts/partials/list.html b/server/www/scripts/hosts/partials/list.html index c84bbb1cde8..bfadda405a6 100644 --- a/server/www/scripts/hosts/partials/list.html +++ b/server/www/scripts/hosts/partials/list.html @@ -44,6 +44,9 @@

    +
    + Show hostnames +
    @@ -54,7 +57,7 @@

    Name - + Hostnames @@ -88,7 +91,7 @@

    {{host.name}} - + {{hostname}}
    From 148fc04aa21554d9fce84a5e7ea43e1b5bf6ce1c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 8 Jan 2018 18:52:58 -0300 Subject: [PATCH 0751/1506] Add unique create of comments for plugins --- persistence/server/server.py | 4 +-- persistence/server/utils.py | 2 +- server/api/base.py | 2 +- server/api/modules/comments.py | 50 ++++++++++++++++++++++++++++------ test_cases/test_api_comment.py | 23 +++++++++++++++- 5 files changed, 67 insertions(+), 14 deletions(-) diff --git a/persistence/server/server.py b/persistence/server/server.py index 92bbac16b90..da739ae42ac 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -115,11 +115,12 @@ def _create_server_get_url(workspace_name, object_name=None, object_id=None): def _create_server_post_url(workspace_name, obj_type, command_id): server_api_url = _create_server_api_url() object_end_point_name = OBJECT_TYPE_END_POINT_MAPPER[obj_type] + if obj_type == 'comment': + object_end_point_name = object_end_point_name.strip('/') + '_unique/' post_url = '{0}/ws/{1}/{2}/'.format(server_api_url, workspace_name, object_end_point_name) if command_id: get_params = {'command_id': command_id} post_url += '?' + urlencode(get_params) - print(post_url) return post_url @@ -130,7 +131,6 @@ def _create_server_put_url(workspace_name, obj_type, obj_id, command_id): if command_id: get_params = {'command_id': command_id} put_url += '?' + urlencode(get_params) - print(put_url) return put_url diff --git a/persistence/server/utils.py b/persistence/server/utils.py index 27ea17bcc1d..b8cba8155b9 100644 --- a/persistence/server/utils.py +++ b/persistence/server/utils.py @@ -108,7 +108,7 @@ def get_vuln_web_properties(vuln_web): def get_note_properties(note): note_dict = { - 'text': note.getText(), + 'text': '{0}\n{1}'.format(note.getName(), note.getText()), 'object_id': note.getObjectID(), 'object_type': note.getObjectType() } diff --git a/server/api/base.py b/server/api/base.py index 8254ed0a8cf..297dabf760f 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -176,7 +176,7 @@ def handle_conflict(err): @app.errorhandler(409) def handle_conflict(err): # webargs attaches additional metadata to the `data` attribute - exc = getattr(err, 'exc') + exc = getattr(err, 'exc', None) or getattr(err, 'description', None) if exc: # Get validations from the ValidationError object messages = exc.messages diff --git a/server/api/modules/comments.py b/server/api/modules/comments.py index 1ff6c2fb884..4047944c23d 100644 --- a/server/api/modules/comments.py +++ b/server/api/modules/comments.py @@ -1,16 +1,16 @@ # Faraday Penetration Test IDE # Copyright (C) 2017 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information -from flask import request, Blueprint -from marshmallow import fields +from flask import abort, Blueprint +from marshmallow import fields, ValidationError from marshmallow.validate import OneOf -from server.models import db, Host, Service +from server.models import db, Host, Service, CommandObject from server.api.base import ( AutoSchema, ReadWriteWorkspacedView, - InvalidUsage) + InvalidUsage, CreateWorkspacedMixin, GenericWorkspacedView) from server.models import Comment comment_api = Blueprint('comment_api', __name__) @@ -27,10 +27,7 @@ class Meta: ) -class CommentView(ReadWriteWorkspacedView): - route_base = 'comment' - model_class = Comment - schema_class = CommentSchema +class CommentCreateMixing(CreateWorkspacedMixin): def _perform_create(self, data, workspace_name): model = { @@ -45,7 +42,42 @@ def _perform_create(self, data, workspace_name): raise InvalidUsage('Can\'t comment inexistent object') if obj.workspace != workspace: raise InvalidUsage('Can\'t comment object of another workspace') - return super(CommentView, self)._perform_create(data, workspace_name) + return super(CommentCreateMixing, self)._perform_create(data, workspace_name) + + +class CommentView(CommentCreateMixing, ReadWriteWorkspacedView): + route_base = 'comment' + model_class = Comment + schema_class = CommentSchema + + +class UniqueCommentView(GenericWorkspacedView, CommentCreateMixing): + """ + This view is used by the plugin engine to avoid duplicate comments + when the same plugin and data was ran multiple times. + """ + route_base = 'comment_unique' + model_class = Comment + schema_class = CommentSchema + + def _perform_create(self, data, workspace_name): + res = super(UniqueCommentView, self)._perform_create(data, workspace_name) + + comment = db.session.query(Comment).filter_by( + text=data['text'], + object_type=data['object_type'], + object_id=data['object_id'], + workspace=self._get_workspace(workspace_name) + ).first() + if comment is not None: + abort(409, ValidationError( + { + 'message': 'Comment already exists', + 'object': self.schema_class().dump(comment).data, + } + )) + return res CommentView.register(comment_api) +UniqueCommentView.register(comment_api) diff --git a/test_cases/test_api_comment.py b/test_cases/test_api_comment.py index 0fc9b51ebd8..05fda625f40 100644 --- a/test_cases/test_api_comment.py +++ b/test_cases/test_api_comment.py @@ -63,4 +63,25 @@ def test_cannot_create_comment_of_another_workspace_object(self, test_client, se raw_comment = self._create_raw_comment('service', 456464556) res = test_client.post(self.url(workspace=self.workspace), data=raw_comment) assert res.status_code == 400 - assert res.json == {u'message': u"Can't comment inexistent object"} \ No newline at end of file + assert res.json == {u'message': u"Can't comment inexistent object"} + + + def test_create_unique_comment_for_plugins(self, session, test_client): + """ + + + """ + service = ServiceFactory.create(workspace=self.workspace) + session.commit() + initial_comment_count = len(session.query(Comment).all()) + raw_comment = self._create_raw_comment('service', service.id) + res = test_client.post(self.url(workspace=self.workspace), + data=raw_comment) + assert res.status_code == 201 + assert len(session.query(Comment).all()) == initial_comment_count + 1 + + url = self.url(workspace=self.workspace).strip('/') + '_unique/' + res = test_client.post(url, data=raw_comment) + assert res.status_code == 409 + assert 'object' in res.json + assert type(res.json) == dict \ No newline at end of file From 6a2af1957159785e8bbe9319139ef3404a5e17b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 8 Jan 2018 19:44:59 -0300 Subject: [PATCH 0752/1506] Fix bug when quitting the faraday client Use a daemonic report manager thread --- managers/reports_managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/managers/reports_managers.py b/managers/reports_managers.py index 22916551bc2..08a642f9e6d 100755 --- a/managers/reports_managers.py +++ b/managers/reports_managers.py @@ -67,9 +67,9 @@ def onlinePlugin(self, cmd): class ReportManager(Thread): def __init__(self, timer, ws_name, plugin_controller, polling=True): Thread.__init__(self) + self.setDaemon(True) self.polling = polling self.ws_name = ws_name - self.daemon = False self.timer = timer self._stop = False self._report_path = os.path.join(CONF.getReportPath(), ws_name) From 76538e6583c8644c69c9ecbc1d7def917767708f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 9 Jan 2018 14:18:45 -0300 Subject: [PATCH 0753/1506] Remove TODO and fix solution when there is a conflict --- gui/gtk/dialogs.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/gui/gtk/dialogs.py b/gui/gtk/dialogs.py index a59c60e697d..553b60a7ab5 100644 --- a/gui/gtk/dialogs.py +++ b/gui/gtk/dialogs.py @@ -1232,16 +1232,9 @@ def save(self, button, keeper): elif keeper == "left": n = 1 - # TODO: fix this code to remove interface logic - # # interface needs a special case, 'cause it's the only object - # # which resolveConflict() will expect its solution to have a - # # dicitionary inside the solution dictionary - # if current_conflict_type != "Interface": - # solution = {} - # for row in self.current_conflict_model: - # solution[row[0].lower()] = self.uncook(row[n], row[4]) - # else: - # solution = self.case_for_interfaces(self.current_conflict_model, n) + solution = {} + for row in self.current_conflict_model: + solution[row[0].lower()] = self.uncook(row[n], row[4]) try: guiapi.resolveConflict(self.current_conflict, solution) From 12f0bcad440092acbc58fbcdd4e9057997439174 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 9 Jan 2018 15:10:27 -0300 Subject: [PATCH 0754/1506] Ask user if faraday server credentials are missing --- config/configuration.py | 2 +- faraday.py | 45 +++++++++++++++++++++++------------------ 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/config/configuration.py b/config/configuration.py index f34a3ae51ab..79613cbdde0 100644 --- a/config/configuration.py +++ b/config/configuration.py @@ -405,7 +405,7 @@ def setVersion(self, val): self._version = val def setAPIUrl(self, url): - self._server_uri = url + self._api_url = url def setAPIUsername(self, username): self._username = username diff --git a/faraday.py b/faraday.py index 91feebca763..d9b0c3f16c2 100755 --- a/faraday.py +++ b/faraday.py @@ -6,15 +6,10 @@ ''' -# TODO: -# - Handle requirements dinamically? -# - Additionally parse arguments from file. - - -import argparse import os -import shutil import sys +import shutil +import argparse from config.configuration import getInstanceConfiguration from config.globals import ( @@ -506,20 +501,22 @@ def checkUpdates(): logger.info("No updates available, enjoy Faraday.") -def checkCouchUrl(): +def checkServerUrl(): import requests + server_url = getInstanceConfiguration().getServerURI() + if server_url is None: + doLoginLoop() try: - requests.get(getInstanceConfiguration().getServerURI(), timeout=5) + requests.get(server_url, timeout=5) except requests.exceptions.SSLError: - print """ + print(""" SSL certificate validation failed. You can use the --cert option in Faraday to set the path of the cert - """ + """) sys.exit(-1) - except Exception: - # Non fatal error - pass + except requests.exceptions.MissingSchema as ex: + print("Check ~/.faraday.config/user.xml server url, the following error was found: {0} ".format(ex)) def checkVersion(): @@ -563,20 +560,28 @@ def doLoginLoop(): try: CONF = getInstanceConfiguration() + if CONF.getAPIUrl() is None: + server_url = raw_input( + "Please enter the faraday server url (press enter for http://localhost:5985): ") or "http://localhost:5985" + CONF.setAPIUrl(server_url) for attempt in range(1, 4): - username = raw_input("Username: ") + username = raw_input("Username (press enter for admin): ") or "admin" password = getpass.getpass('Password: ') - session_cookie = login_user(CONF.getServerURI(), username, password) + + session_cookie = login_user(server_url, username, password) if session_cookie: - CONF.setDBUser(username) + + CONF.setAPIUsername(username) + CONF.setAPIPassword(password) CONF.setDBSessionCookies(session_cookie) + CONF.saveConfig() user_info = get_user_info() if user_info is None or 'username' not in user_info or 'roles' not in user_info or 'client' in user_info['roles']: - print "You can't login as a client. You have %s attempt(s) left." % (3 - attempt) + print("You can't login as a client. You have %s attempt(s) left." % (3 - attempt)) continue logger.info('Login successful') @@ -615,7 +620,7 @@ def main(): os.environ[REQUESTS_CA_BUNDLE_VAR] = args.cert_path checkConfiguration(args.gui) setConf() - checkCouchUrl() + checkServerUrl() checkVersion() CONF = getInstanceConfiguration() if args.login: @@ -624,7 +629,7 @@ def main(): if couchURI: CONF.setCouchUri(couchURI) - checkCouchUrl() + checkServerUrl() else: logger.fatal('Please configure couchdb server to authenticate (--login)') sys.exit(-1) From e07dd08125e50f97920d0555aaebf2796ced1361 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 9 Jan 2018 15:10:52 -0300 Subject: [PATCH 0755/1506] Improve backslash remove --- persistence/server/server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/persistence/server/server.py b/persistence/server/server.py index da739ae42ac..dfd19093256 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -85,10 +85,11 @@ def _conf(): def _get_base_server_url(): if FARADAY_UP: - server_url = _conf().getServerURI() + server_url = _conf().getAPIUrl() else: server_url = SERVER_URL - return server_url[:-1] if server_url[-1] == "/" else server_url + + return server_url.rstrip('/') def _create_server_api_url(): From 31b856834ae4e0bb415ce5f040ad87825ea1c3ed Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 9 Jan 2018 15:11:22 -0300 Subject: [PATCH 0756/1506] Add debug log to api requests urls --- persistence/server/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/persistence/server/server.py b/persistence/server/server.py index dfd19093256..689c7b74d2b 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -182,6 +182,7 @@ def _unsafe_io_with_server(server_io_function, server_expected_response, Return the response from the server. """ answer = None + logger.debug('Sending request to api endpoint {0}'.format(server_url)) try: answer = server_io_function(server_url, **payload) if answer.status_code == 409: From 1eaf722bb6c73e21d1930cec74059554a06a1c45 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 9 Jan 2018 15:38:58 -0300 Subject: [PATCH 0757/1506] Fix some issues when using new user.xml settings --- config/configuration.py | 4 ++-- faraday.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/config/configuration.py b/config/configuration.py index 79613cbdde0..48987324aff 100644 --- a/config/configuration.py +++ b/config/configuration.py @@ -408,10 +408,10 @@ def setAPIUrl(self, url): self._api_url = url def setAPIUsername(self, username): - self._username = username + self._api_username = username def setAPIPassword(self, password): - self._password = password + self._api_password = password def setPluginSettings(self, settings): self._plugin_settings = settings diff --git a/faraday.py b/faraday.py index d9b0c3f16c2..5c3225f38df 100755 --- a/faraday.py +++ b/faraday.py @@ -503,8 +503,9 @@ def checkUpdates(): def checkServerUrl(): import requests - server_url = getInstanceConfiguration().getServerURI() - if server_url is None: + CONF = getInstanceConfiguration() + server_url = CONF.getServerURI() + if server_url is None or CONF.getAPIUsername() is None or CONF.getAPIUsername() is None: doLoginLoop() try: requests.get(server_url, timeout=5) @@ -560,7 +561,8 @@ def doLoginLoop(): try: CONF = getInstanceConfiguration() - if CONF.getAPIUrl() is None: + server_url = CONF.getAPIUrl() + if server_url is None: server_url = raw_input( "Please enter the faraday server url (press enter for http://localhost:5985): ") or "http://localhost:5985" CONF.setAPIUrl(server_url) From 0bc33cd4f52d29fdb2e6fc188e2fdc706823cf62 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 9 Jan 2018 15:39:14 -0300 Subject: [PATCH 0758/1506] Save new admin credentials on initdb --- server/commands/initdb.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index 82e6e0728e0..fe4b16f863a 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -9,6 +9,8 @@ from sqlalchemy import create_engine +from config.configuration import getInstanceConfiguration + try: # py2.7 from configparser import ConfigParser, NoSectionError, NoOptionError @@ -115,6 +117,10 @@ def _create_admin_user(self, conn_string): engine = create_engine(conn_string) random_password = self.generate_random_pw(12) engine.execute("INSERT INTO \"user\" (username, name, password, is_ldap, active) VALUES ('admin', 'Administrator', '{0}', false, true);".format(random_password)) + CONF = getInstanceConfiguration() + CONF.setAPIUsername('admin') + CONF.setAPIPassword(random_password) + CONF.saveConfig() print("Admin username created with {red} password{white}: {random_password}".format(random_password=random_password, white=Fore.WHITE, red=Fore.RED)) From 926157d0cfd283f6464e494c876227edde1eca71 Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 9 Jan 2018 17:50:50 -0300 Subject: [PATCH 0759/1506] Add Python dependency --- requirements_server.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_server.txt b/requirements_server.txt index 830b1fb965f..bc68387a4ff 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -13,6 +13,7 @@ requests==2.18.4 restkit==4.2.2 service_identity==17.0.0 SQLAlchemy==1.2.0b2 +sqlalchemy_schemadisplay==1.3 tqdm==4.15.0 twisted==17.5.0 webargs From d6237a3b07b8dbb1deb315ea3ba6d837dcacd736 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 9 Jan 2018 17:56:32 -0300 Subject: [PATCH 0760/1506] Fix unit tests: when the name is none avoid concat on text --- persistence/server/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/persistence/server/utils.py b/persistence/server/utils.py index b8cba8155b9..9a50eb8b7d2 100644 --- a/persistence/server/utils.py +++ b/persistence/server/utils.py @@ -107,8 +107,11 @@ def get_vuln_web_properties(vuln_web): def get_note_properties(note): + text = note.getText() + if note.getName(): + text = '{0}\n{1}'.format(note.getName(), note.getText()) note_dict = { - 'text': '{0}\n{1}'.format(note.getName(), note.getText()), + 'text': text, 'object_id': note.getObjectID(), 'object_type': note.getObjectType() } From 2c92eb6453c3095684642bed5692c62c2fa088de Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 9 Jan 2018 18:06:47 -0300 Subject: [PATCH 0761/1506] Set locahost:5985 as default server url --- server/commands/initdb.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index fe4b16f863a..6bb808912a1 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -118,6 +118,7 @@ def _create_admin_user(self, conn_string): random_password = self.generate_random_pw(12) engine.execute("INSERT INTO \"user\" (username, name, password, is_ldap, active) VALUES ('admin', 'Administrator', '{0}', false, true);".format(random_password)) CONF = getInstanceConfiguration() + CONF.setAPIUrl('http://localhost:5985') CONF.setAPIUsername('admin') CONF.setAPIPassword(random_password) CONF.saveConfig() From de107589fda54fee9d078fc87f97c904ee2a69c5 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 9 Jan 2018 18:09:15 -0300 Subject: [PATCH 0762/1506] Change default username to faraday --- faraday.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/faraday.py b/faraday.py index 5c3225f38df..84ed7e21143 100755 --- a/faraday.py +++ b/faraday.py @@ -569,7 +569,7 @@ def doLoginLoop(): for attempt in range(1, 4): - username = raw_input("Username (press enter for admin): ") or "admin" + username = raw_input("Username (press enter for faraday): ") or "faraday" password = getpass.getpass('Password: ') session_cookie = login_user(server_url, username, password) From 5b82102abd2720d81e7fdad666ea905f2c717daa Mon Sep 17 00:00:00 2001 From: micabot Date: Tue, 9 Jan 2018 18:14:05 -0300 Subject: [PATCH 0763/1506] Fix initb script message to avoid confusions --- server/commands/initdb.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index fe4b16f863a..7ec040e8ab0 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -116,12 +116,17 @@ def run(self): def _create_admin_user(self, conn_string): engine = create_engine(conn_string) random_password = self.generate_random_pw(12) - engine.execute("INSERT INTO \"user\" (username, name, password, is_ldap, active) VALUES ('admin', 'Administrator', '{0}', false, true);".format(random_password)) + engine.execute("INSERT INTO \"user\" (username, name, password, " + "is_ldap, active) VALUES ('faraday', 'Administrator', " + "'{0}', false, true);".format(random_password)) CONF = getInstanceConfiguration() CONF.setAPIUsername('admin') CONF.setAPIPassword(random_password) CONF.saveConfig() - print("Admin username created with {red} password{white}: {random_password}".format(random_password=random_password, white=Fore.WHITE, red=Fore.RED)) + print("Admin user created with {red}username: {white}admin and " + " {red}password{white}: {" + "random_password}".format(random_password=random_password, + white=Fore.WHITE, red=Fore.RED)) def _configure_existing_postgres_user(self): From 9a33722f25dd7579ce049c623378d72bd4b8483e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 9 Jan 2018 18:26:48 -0300 Subject: [PATCH 0764/1506] Fix factorty serialization, the api expects end and start fields. --- test_cases/factories.py | 6 ++++-- test_cases/test_api_non_workspaced_base.py | 7 +++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/test_cases/factories.py b/test_cases/factories.py index 393073deb57..e28f92231ac 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -390,8 +390,10 @@ class Meta: def build_dict(cls, **kwargs): # Ugly hack to JSON-serialize datetimes ret = super(LicenseFactory, cls).build_dict(**kwargs) - ret['start_date'] = ret['start_date'].isoformat() - ret['end_date'] = ret['end_date'].isoformat() + ret['start'] = ret['start_date'].isoformat() + ret['end'] = ret['end_date'].isoformat() + ret.pop('start_date') + ret.pop('end_date') return ret diff --git a/test_cases/test_api_non_workspaced_base.py b/test_cases/test_api_non_workspaced_base.py index fe00954e4a0..7e0f7b16757 100644 --- a/test_cases/test_api_non_workspaced_base.py +++ b/test_cases/test_api_non_workspaced_base.py @@ -4,11 +4,11 @@ import pytest from sqlalchemy.orm.util import was_deleted -from server.models import db API_PREFIX = '/v2/' OBJECT_COUNT = 5 + @pytest.mark.usefixtures('logged_user') class GenericAPITest: @@ -45,8 +45,7 @@ def url(self, obj=None): class ListTestsMixin: - def test_list_retrieves_all_items_from(self, test_client, - session): + def test_list_retrieves_all_items_from(self, test_client): res = test_client.get(self.url()) assert res.status_code == 200 if 'rows' in res.json: @@ -79,7 +78,7 @@ def test_create_succeeds(self, test_client): assert res.status_code == 201 assert self.model.query.count() == OBJECT_COUNT + 1 object_id = res.json['id'] - obj = self.model.query.get(object_id) + self.model.query.get(object_id) def test_create_fails_with_empty_dict(self, test_client): res = test_client.post(self.url(), data={}) From 844e6b19f94af7257822072cef3b6a57e76ee7cc Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 9 Jan 2018 19:07:12 -0300 Subject: [PATCH 0765/1506] Add reuse of db credentials and databases --- server/commands/initdb.py | 50 +++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index f70bec96f29..da2eca6d8e6 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -7,6 +7,7 @@ from tempfile import TemporaryFile from subprocess import Popen, PIPE +import sqlalchemy from sqlalchemy import create_engine from config.configuration import getInstanceConfiguration @@ -116,18 +117,27 @@ def run(self): def _create_admin_user(self, conn_string): engine = create_engine(conn_string) random_password = self.generate_random_pw(12) - engine.execute("INSERT INTO \"user\" (username, name, password, " + already_created = False + try: + engine.execute("INSERT INTO \"user\" (username, name, password, " "is_ldap, active) VALUES ('faraday', 'Administrator', " "'{0}', false, true);".format(random_password)) - CONF = getInstanceConfiguration() - CONF.setAPIUrl('http://localhost:5985') - CONF.setAPIUsername('admin') - CONF.setAPIPassword(random_password) - CONF.saveConfig() - print("Admin user created with {red}username: {white}admin and " - " {red}password{white}: {" - "random_password}".format(random_password=random_password, - white=Fore.WHITE, red=Fore.RED)) + except sqlalchemy.exc.IntegrityError: + # when re using database user could be created previusly + already_created = True + print( + "{yellow}WARNING{white}: Faraday administrator user already exists.".format( + yellow=Fore.YELLOW, white=Fore.WHITE)) + if not already_created: + CONF = getInstanceConfiguration() + CONF.setAPIUrl('http://localhost:5985') + CONF.setAPIUsername('faraday') + CONF.setAPIPassword(random_password) + CONF.saveConfig() + print("Admin user created with {red}username: {white}faraday and " + " {red}password{white}: {" + "random_password}".format(random_password=random_password, + white=Fore.WHITE, red=Fore.RED)) def _configure_existing_postgres_user(self): @@ -166,7 +176,16 @@ def _configure_new_postgres_user(self, psql_log_file): command = postgres_command + ['psql', '-c', 'CREATE ROLE {0} WITH LOGIN PASSWORD \'{1}\';'.format(username, password)] p = Popen(command, stderr=psql_log_file, stdout=psql_log_file) p.wait() - return username, password, p.returncode + psql_log_file.seek(0) + output = psql_log_file.read() + already_exists_error = 'role "{0}" already exists'.format(username) + return_code = p.returncode + if already_exists_error in output: + print("{yellow}WARNING{white}: Role {username} already exists, skipping creation ".format(yellow=Fore.YELLOW, white=Fore.WHITE, username=username)) + password = getpass.getpass("Database password: ") + + return_code = 0 + return username, password, return_code def _create_database(self, database_name, username, psql_log_file): """ @@ -177,7 +196,14 @@ def _create_database(self, database_name, username, psql_log_file): command = postgres_command + ['createdb', '-O', username, database_name] p = Popen(command, stderr=psql_log_file, stdout=psql_log_file, cwd='/tmp') p.wait() - return database_name, p.returncode + return_code = p.returncode + psql_log_file.seek(0) + output = psql_log_file.read() + already_exists_error = 'database creation failed: ERROR: database "{0}" already exists'.format(database_name) + if already_exists_error in output: + print('{yellow}WARNING{white}: Database already exists.'.format(yellow=Fore.YELLOW, white=Fore.WHITE)) + return_code = 0 + return database_name, return_code def _save_config(self, config, username, password, database_name, hostname): """ From 86f5839101c1da2af534ff0d2197d77aa74a9abd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 10 Jan 2018 14:51:59 -0300 Subject: [PATCH 0766/1506] Allow sorting vulns by vuln web fields in the API --- server/api/base.py | 6 ++++-- server/api/modules/vulns.py | 1 + test_cases/test_api_vulnerability.py | 32 +++++++++++++++++++++++++++- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index f2f34144f76..aa14ad9fa29 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -298,6 +298,7 @@ class SortableMixin(object): sort_field_paremeter_name = "sort" sort_direction_paremeter_name = "sort_dir" default_sort_direction = "asc" + sort_model_class = None # Override to use a model with more fields def _get_order_field(self, **kwargs): try: @@ -329,11 +330,12 @@ def _get_order_field(self, **kwargs): # TODO migration: improve this checking or use a whitelist. # Handle PrimaryKeyRelatedField - if order_field not in inspect(self.model_class).attrs: + model_class = self.sort_model_class or self.model_class + if order_field not in inspect(model_class).attrs: # It could be something like fields.Method raise InvalidUsage("Field not in the DB: %s" % order_field) - field = getattr(self.model_class, order_field) + field = getattr(model_class, order_field) sort_dir = flask.request.args.get(self.sort_direction_paremeter_name, self.default_sort_direction) if sort_dir not in ('asc', 'desc'): diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index f8c4a604221..c7d5ab9e575 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -299,6 +299,7 @@ class VulnerabilityView(PaginatedMixin, ReadWriteWorkspacedView): route_base = 'vulns' filterset_class = VulnerabilityFilterSet + sort_model_class = VulnerabilityWeb # It has all the fields unique_fields_by_class = { 'Vulnerability': [('name', 'description', 'host_id', 'service_id')], 'VulnerabilityWeb': [('name', 'description', 'service_id', 'method', diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index c683272fa3f..886c0ec616a 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -519,7 +519,37 @@ def test_filter_by_website(self, test_client, session, assert vuln['website'] == 'faradaysec.com' assert set(vuln['_id'] for vuln in res.json['data']) == expected_ids - def test_create_vuln_with_evidence(self, host_with_hostnames, test_client, session): + @pytest.mark.usefixtures('mock_envelope_list') + def test_sort_by_method(self, session, test_client, second_workspace, + vulnerability_factory, vulnerability_web_factory): + vulnerability_factory.create_batch( + 10, workspace=second_workspace + ) + vulnerability_web_factory.create_batch( + 10, workspace=second_workspace, method=None + ) + + for method in ('afjbeidcgh'): + vulnerability_web_factory.create(workspace=second_workspace, + method=method) + + session.commit() + res = test_client.get(self.url(workspace=second_workspace) + + '?sort=method&sort_dir=asc') + assert res.status_code == 200, res.data + assert len(res.json['data']) == 30 + assert ''.join(v['method'] for v in res.json['data'] + if v['method']) == 'abcdefghij' + + res = test_client.get(self.url(workspace=second_workspace) + + '?sort=method&sort_dir=desc') + assert res.status_code == 200, res.data + assert len(res.json['data']) == 30 + assert ''.join(v['method'] for v in res.json['data'] + if v['method']) == 'abcdefghij'[::-1] + + def test_create_vuln_with_evidence(self, host_with_hostnames, test_client, + session): session.commit() # flush host_with_hostnames attachments = [ open(os.path.join(CURRENT_PATH, 'data', 'faraday.png'), 'r'), From 119e61264088302aeb8571a6414dc94846f2e900 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 10 Jan 2018 15:31:41 -0300 Subject: [PATCH 0767/1506] Disable vuln sorting by impact and by hostnames in the Web UI It doesn't make sense --- server/www/scripts/statusReport/controllers/statusReport.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 9052856f055..261538e6d7e 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -269,7 +269,7 @@ angular.module('faradayApp') headerCellTemplate: header, minWidth: '100', maxWidth: '200', - sort: getColumnSort('hostnames'), + enableSorting: false, visible: $scope.columns["hostnames"] }); $scope.gridOptions.columnDefs.push({ name : 'target', @@ -347,6 +347,7 @@ angular.module('faradayApp') cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/impactcolumn.html', headerCellTemplate: header, sort: getColumnSort('impact'), + enableSorting: false, visible: $scope.columns["impact"] }); $scope.gridOptions.columnDefs.push({ name : 'method', From e6314f7924a5dedbd326ce1834c88f57bfa0f23c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 10 Jan 2018 17:09:22 -0300 Subject: [PATCH 0768/1506] Move vuln target to a model column property This way it can be sorted directly by postgres. Sorting and grouping by target now work in the web ui. --- server/api/modules/vulns.py | 9 ++------- server/models.py | 28 ++++++++++++++++++++++------ test_cases/test_api_vulnerability.py | 2 ++ 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index c7d5ab9e575..21f6461afe9 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -115,7 +115,7 @@ class VulnerabilitySchema(AutoSchema): status = fields.Method(serialize='get_status', deserialize='load_status') # TODO: this breaks enum validation. type = fields.Method(serialize='get_type', deserialize='load_type', required=True) obj_id = fields.String(dump_only=True, attribute='id') - target = fields.Method('get_target') + target = fields.String(dump_only=True, attribute='target_host_ip') metadata = SelfNestedField(CustomMetadataSchema()) date = fields.DateTime(attribute='create_date', dump_only=True) # This is only used for sorting @@ -165,12 +165,6 @@ def get_status(self, obj): def get_issuetracker(self, obj): return {} - def get_target(self, obj): - if obj.service is not None: - return obj.service.host.ip - else: - return obj.host.ip - def load_status(self, value): if value == 'opened': return 'open' @@ -373,6 +367,7 @@ def _get_eagerloaded_query(self, *args, **kwargs): undefer(VulnerabilityGeneric.creator_command_id), undefer(VulnerabilityGeneric.creator_command_tool), + undefer(VulnerabilityGeneric.target_host_ip), joinedload(VulnerabilityGeneric.evidence), joinedload(VulnerabilityGeneric.tags), ] diff --git a/server/models.py b/server/models.py index dbc68c95e0f..174c6c1da1e 100644 --- a/server/models.py +++ b/server/models.py @@ -17,12 +17,10 @@ String, Text, UniqueConstraint, - event, - and_) -from sqlalchemy.ext.hybrid import hybrid_property -from sqlalchemy.orm import backref, relationship, undefer + event) +from sqlalchemy.orm import relationship, undefer from sqlalchemy.sql import select, text, table -from sqlalchemy.sql.expression import asc, join +from sqlalchemy.sql.expression import asc, case, join from sqlalchemy import func from sqlalchemy.orm import ( backref, @@ -45,7 +43,7 @@ RoleMixin, UserMixin, ) -from server.utils.database import get_or_create, BooleanToIntColumn +from server.utils.database import BooleanToIntColumn class SQLAlchemy(OriginalSQLAlchemy): @@ -715,6 +713,24 @@ class VulnerabilityGeneric(VulnerabilityABC): deferred=True ) + _host_vuln_query = ( + select([Host.ip]) + .where(text('vulnerability.host_id = host.id')) + ) + _service_vuln_query = ( + select([text('host_inner.ip')]) + .select_from(text('host as host_inner, service')) + .where(text('vulnerability.service_id = service.id and ' + 'host_inner.id = service.host_id')) + ) + target_host_ip = column_property( + case([ + (text('host_id IS NOT null'), _host_vuln_query.as_scalar()), + (text('service_id IS NOT null'), _service_vuln_query.as_scalar()) + ]), + deferred=True + ) + __mapper_args__ = { 'polymorphic_on': type } diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 886c0ec616a..b98042d3445 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -787,6 +787,8 @@ def test_count_severity_map(self, test_client, second_workspace, session): def test_target(self, test_client, session, second_workspace, host_factory, service_factory, vulnerability_factory, vulnerability_web_factory): + host_factory.create_batch(5, workspace=second_workspace) + service_factory.create_batch(5, workspace=second_workspace) host = host_factory.create(workspace=second_workspace) service = service_factory.create(host=host, workspace=second_workspace) From a32c03a0c788ed726a9ca1a2d61073b240919cb9 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 10 Jan 2018 17:19:16 -0300 Subject: [PATCH 0769/1506] Use master node for faraday unit tests --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ec581fbacd5..9a318a66531 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,5 @@ #!groovy -node { +node (label: "master"){ def ENV_PATH = "$HOME/venv/faraday" echo "${ENV_PATH}" From f0d60c59a8836b67dc037a9565c20116c90fedbc Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 10 Jan 2018 18:26:51 -0300 Subject: [PATCH 0770/1506] Confirmed now in CSV export of web ui --- server/www/scripts/csv/providers/csv.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/server/www/scripts/csv/providers/csv.js b/server/www/scripts/csv/providers/csv.js index 4b91c6aa9d5..b949485c551 100644 --- a/server/www/scripts/csv/providers/csv.js +++ b/server/www/scripts/csv/providers/csv.js @@ -16,7 +16,8 @@ angular.module('faradayApp') var title = "SR-" + ws } - + // Add confirmed to CSV fields + aProperties.push("confirmed"); for(key in properties) { if(properties.hasOwnProperty(key)) { if(properties[key] === true) { @@ -28,11 +29,13 @@ angular.module('faradayApp') aProperties.forEach(function(prop) { object = {}; if(typeof(v[prop]) === "object") v[prop] = parseObject(v[prop]); - if(typeof(v[prop]) != "undefined" && v[prop] != null) { + + if(typeof(v[prop]) != "undefined" && v[prop] != null && prop != "confirmed") object[prop] = cleanCSV(v[prop]); - } else { + else object[prop] = ""; - } + + if(prop === "confirmed") object[prop] = v[prop].toString(); if(prop === "date") object[prop] = parseDate(v["metadata"]["create_time"] * 1000); if(prop === "creator") object[prop] = excelEscape(v["metadata"]["creator"]); if(prop === "web") { From 581149b53e178545f57359f5d5f78e4a27f6c314 Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 11 Jan 2018 16:22:11 -0300 Subject: [PATCH 0771/1506] Add password validation for initdb If postgres is configured to not trust system users, initdb was exploding in some very specific cases. --- server/commands/initdb.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index da2eca6d8e6..613086d1f58 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -3,6 +3,7 @@ import os import sys +import psycopg2 from random import SystemRandom from tempfile import TemporaryFile from subprocess import Popen, PIPE @@ -182,8 +183,34 @@ def _configure_new_postgres_user(self, psql_log_file): return_code = p.returncode if already_exists_error in output: print("{yellow}WARNING{white}: Role {username} already exists, skipping creation ".format(yellow=Fore.YELLOW, white=Fore.WHITE, username=username)) - password = getpass.getpass("Database password: ") + invalid_pwd = True + while invalid_pwd: + password = getpass.getpass("Database password (ctrl-c to " + "cancel): ") + + # check credentials + # this case only applies to instances without 'trust' config + # todo: check postgres config + try: + connection = psycopg2.connect(dbname='postgres', + user=username, + password=password) + cur = connection.cursor() + cur.execute('SELECT * FROM pg_catalog.pg_tables;') + cur.fetchall() + connection.commit() + connection.close() + invalid_pwd = False + except psycopg2.Error as e: + if 'authentication failed' in e.message: + print('{red}ERROR{white}: User {username} already ' + 'exists and provided password ' + 'is incorrect'.format(white=Fore.WHITE, + red=Fore.RED, + username=username)) + else: + raise return_code = 0 return username, password, return_code @@ -232,6 +259,8 @@ def _create_tables(self, conn_string): if 'could not connect to server' in ex.message: print('ERROR: {red}PostgreSQL service{white} is not running. Please verify that it is running in port 5432 before executing setup script.'.format(red=Fore.RED, white=Fore.WHITE)) sys.exit(1) + elif 'password authentication failed' in ex.message: + print('ERROR: ') else: raise except ImportError as ex: From 7ad7746fdf7d058138306c7feb98cce70d6bd63a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 11 Jan 2018 16:11:38 -0300 Subject: [PATCH 0772/1506] Allow filtering by target and type in the vulns api --- server/api/modules/vulns.py | 27 +++++++++--- test_cases/test_api_vulnerability.py | 61 +++++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 7 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 21f6461afe9..af7031435c7 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -6,7 +6,7 @@ import logging from base64 import b64encode, b64decode -from filteralchemy import FilterSet, operators +from filteralchemy import Filter, FilterSet, operators from flask import request, current_app from flask import Blueprint from marshmallow import Schema, fields, post_load, ValidationError @@ -249,18 +249,32 @@ class Meta: _strict_filtering = {'default_operator': operators.Equal} +class TargetFilter(Filter): + def filter(self, query, model, attr, value): + return query.filter(model.target_host_ip == value) + + +class TypeFilter(Filter): + def filter(self, query, model, attr, value): + type_map = { + 'Vulnerability': 'vulnerability', + 'VulnerabilityWeb': 'vulnerability_web', + } + assert value in type_map + return query.filter(model.__table__.c.type == type_map[value]) + + class VulnerabilityFilterSet(FilterSet): class Meta(FilterSetMeta): model = VulnerabilityWeb # It has all the fields # TODO migration: Check if we should add fields creator, owner, - # command, impact, type, service, issuetracker, tags, date, target, - # host, easeofresolution, evidence, policy violations, hostnames, - # target + # command, impact, service, issuetracker, tags, date, + # host, easeofresolution, evidence, policy violations, hostnames fields = ( "status", "website", "parameter_name", "query_string", "path", "data", "severity", "confirmed", "name", "request", "response", "parameters", "resolution", "method", "ease_of_resolution", - "description", "command_id") + "description", "command_id", "target") strict_fields = ( "severity", "confirmed", "method" @@ -270,6 +284,9 @@ class Meta(FilterSetMeta): column_overrides = { field: _strict_filtering for field in strict_fields} operators = (operators.ILike, operators.Equal) + target = TargetFilter(fields.Str()) + type = TypeFilter(fields.Str(validate=[OneOf(['Vulnerability', + 'VulnerabilityWeb'])])) def filter(self): """Generate a filtered query from request parameters. diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index b98042d3445..3411e57e231 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -5,12 +5,17 @@ import pytest import time -from server.api.modules.vulns import VulnerabilityView +from server.api.modules.vulns import ( + TypeFilter, + VulnerabilityFilterSet, + VulnerabilityView +) from test_cases import factories from test_api_workspaced_base import ( ReadOnlyAPITests ) from server.models import ( + VulnerabilityGeneric, Vulnerability, VulnerabilityWeb, Reference, PolicyViolation, CommandObject) @@ -519,6 +524,36 @@ def test_filter_by_website(self, test_client, session, assert vuln['website'] == 'faradaysec.com' assert set(vuln['_id'] for vuln in res.json['data']) == expected_ids + @pytest.mark.usefixtures('mock_envelope_list') + def test_filter_by_target(self, test_client, session, host_factory, + service_factory, vulnerability_factory, + vulnerability_web_factory): + host = host_factory.create(workspace=self.workspace, + ip='9.9.9.9') + expected_ids = set() + + host_vulns = vulnerability_factory.create_batch( + 10, workspace=self.workspace, host=host, service=None) + session.flush() + expected_ids.update(v.id for v in host_vulns) + + for service in service_factory.create_batch(10, + workspace=self.workspace, + host=host): + service_vuln = vulnerability_factory.create( + workspace=self.workspace, service=service, host=None) + web_vuln = vulnerability_web_factory.create( + workspace=self.workspace, service=service) + session.flush() + expected_ids.add(service_vuln.id) + expected_ids.add(web_vuln.id) + + res = test_client.get(self.url() + '?target=9.9.9.9') + assert res.status_code == 200 + for vuln in res.json['data']: + assert vuln['target'] == '9.9.9.9' + assert set(vuln['_id'] for vuln in res.json['data']) == expected_ids + @pytest.mark.usefixtures('mock_envelope_list') def test_sort_by_method(self, session, test_client, second_workspace, vulnerability_factory, vulnerability_web_factory): @@ -1067,5 +1102,27 @@ def test_create_and_update_webvuln(self, host_with_hostnames, test_client, sessi severity='high', ) res = test_client.put('/v2/ws/{0}/vulns/{1}/?command_id={2}'.format(ws_name, res.json['_id'], command.id), - data=raw_data) + data=raw_data) assert res.status_code == 200 + + +def test_type_filter(workspace, session, + vulnerability_factory, + vulnerability_web_factory): + filter_ = VulnerabilityFilterSet().filters['type'] + std_vulns = vulnerability_factory.create_batch(10, workspace=workspace) + web_vulns = vulnerability_web_factory.create_batch(10, workspace=workspace) + + std_filter = filter_.filter(VulnerabilityGeneric.query, + VulnerabilityGeneric, + 'type', + 'Vulnerability' + ) + assert {v.id for v in std_filter} == {v.id for v in std_vulns} + + web_filter = filter_.filter(VulnerabilityGeneric.query, + VulnerabilityGeneric, + 'type', + 'VulnerabilityWeb' + ) + assert {v.id for v in web_filter} == {v.id for v in web_vulns} From 3b67f94d29bec391e752de702e1e880d559c5027 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 11 Jan 2018 17:11:39 -0300 Subject: [PATCH 0773/1506] Add reset password --- server/app.py | 4 ++ server/www/index.html | 5 +- server/www/scripts/auth/controllers/login.js | 11 ++- .../scripts/auth/controllers/resetPassword.js | 51 ++++++++++++++ .../www/scripts/auth/directives/compareTo.js | 21 ++++++ .../scripts/auth/partials/changePassword.html | 69 +++++++++++++++++++ server/www/scripts/auth/services/account.js | 22 ++++++ .../scripts/commons/controllers/homeCtrl.js | 1 - .../www/scripts/commons/providers/server.js | 5 ++ 9 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 server/www/scripts/auth/controllers/resetPassword.js create mode 100644 server/www/scripts/auth/directives/compareTo.js create mode 100644 server/www/scripts/auth/partials/changePassword.html create mode 100644 server/www/scripts/auth/services/account.js diff --git a/server/app.py b/server/app.py index 7963c4248de..89ad2d656b8 100644 --- a/server/app.py +++ b/server/app.py @@ -126,6 +126,10 @@ def create_app(db_connection_string=None, testing=None): app.config['SECURITY_USER_IDENTITY_ATTRIBUTES'] = ['username'] app.config['SECURITY_POST_LOGIN_VIEW'] = '/_api/session' app.config['SECURITY_POST_LOGOUT_VIEW'] = '/_api/login' + app.config['SECURITY_POST_CHANGE_VIEW'] = '/_api/change' + app.config['SECURITY_CHANGEABLE'] = True + app.config['SECURITY_SEND_PASSWORD_CHANGE_EMAIL'] = False + app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False app.config['SQLALCHEMY_RECORD_QUERIES'] = True # app.config['SQLALCHEMY_ECHO'] = True diff --git a/server/www/index.html b/server/www/index.html index 9e04367f2b0..2fcfdecfc82 100644 --- a/server/www/index.html +++ b/server/www/index.html @@ -77,7 +77,7 @@
  • Users
  • Licenses
  • -
  • Change password
    • +
  • Change password
  • Logout
  • Help
    • @@ -145,6 +145,9 @@ + + + diff --git a/server/www/scripts/auth/controllers/login.js b/server/www/scripts/auth/controllers/login.js index 3c70bdd409d..77e7a8f0e9e 100644 --- a/server/www/scripts/auth/controllers/login.js +++ b/server/www/scripts/auth/controllers/login.js @@ -44,7 +44,7 @@ angular.module('faradayApp') }]); angular.module('faradayApp') - .controller('loginBarCtrl', ['$scope', '$location', '$cookies','loginSrv', function($scope, $location, $cookies,loginSrv) { + .controller('loginBarCtrl', ['$scope', '$location', '$cookies','loginSrv', '$uibModal', function($scope, $location, $cookies,loginSrv,$uibModal) { $scope.user = null; $scope.auth = loginSrv.isAuth(); @@ -69,6 +69,15 @@ angular.module('faradayApp') $cookies.currentUrl = ""; }); }; + + $scope.changePasswordModal = function(){ + $scope.modal = $uibModal.open({ + templateUrl: 'scripts/auth/partials/changePassword.html', + controller: 'resetPassword', + size: 'lg' + }); + } + }]); angular.module('faradayApp') diff --git a/server/www/scripts/auth/controllers/resetPassword.js b/server/www/scripts/auth/controllers/resetPassword.js new file mode 100644 index 00000000000..97f164c36d9 --- /dev/null +++ b/server/www/scripts/auth/controllers/resetPassword.js @@ -0,0 +1,51 @@ +// Faraday Penetration Test IDE +// Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +// See the file 'doc/LICENSE' for the license information + +angular.module('faradayApp') + .controller('resetPassword', ['$modalInstance', '$scope', 'AccountSrv', + function($modalInstance, $scope, AccountSrv) { + + // Change Password Modal controller + + $scope.data; + + init = function () { + $scope.data = { + "current": "", + "newPassword": "", + "newPasswordRepeat": "" + }; + }; + + $scope.open = function($event, isStart) { + $event.preventDefault(); + $event.stopPropagation(); + }; + + $scope.ok = function(){ + AccountSrv.changePassword($scope.data).then( + function(response, statusText, xhrObj){ + $scope.modal.close() + }, function(xhrObj, textStatus, err) { + if (xhrObj.data.response.errors.password != undefined) { + $scope.form.current.$setValidity("current", false); + } + if (xhrObj.data.response.errors.new_password_confirm != undefined) { + $scope.form.password.$setValidity("password", true); + $scope.form.passRepeat.$setValidity("passRepeat", false); + } + }); + }; + + $scope.resetError = function() { + $scope.form.current.$setValidity('current', true); + + } + + $scope.cancel = function() { + $modalInstance.dismiss('cancel'); + }; + + init(); + }]); \ No newline at end of file diff --git a/server/www/scripts/auth/directives/compareTo.js b/server/www/scripts/auth/directives/compareTo.js new file mode 100644 index 00000000000..b3c45ccc491 --- /dev/null +++ b/server/www/scripts/auth/directives/compareTo.js @@ -0,0 +1,21 @@ +// Faraday Penetration Test IDE +// Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +// See the file 'doc/LICENSE' for the license information +// http://odetocode.com/blogs/scott/archive/2014/10/13/confirm-password-validation-in-angularjs.aspx +angular.module('faradayApp') + .directive('compareTo', [function() { + return { + require: "ngModel", + scope: { + otherModelValue: "=compareTo" + }, + link: function(scope, element, attributes, ngModel) { + ngModel.$validators.compareTo = function(modelValue) { + return modelValue == scope.otherModelValue; + }; + scope.$watch("otherModelValue", function() { + ngModel.$validate(); + }); + } + }; + }]); diff --git a/server/www/scripts/auth/partials/changePassword.html b/server/www/scripts/auth/partials/changePassword.html new file mode 100644 index 00000000000..d8b8e8f4906 --- /dev/null +++ b/server/www/scripts/auth/partials/changePassword.html @@ -0,0 +1,69 @@ + + + +
    +
    +
    + + +
    +

    Change password {{user.name}}

    +
    +
    +
    +
    + +
    +
    Please type current password
    +
    +
    + + +
    + +
    + +
    New password (must be at least 8 characters long)
    +
    +
    + + +
    +
    + + +
    +
    +
    + +
    +
    + +
    +
    +
    +
    +
    + + +
    +
    +
    \ No newline at end of file diff --git a/server/www/scripts/auth/services/account.js b/server/www/scripts/auth/services/account.js new file mode 100644 index 00000000000..e777a2d7f8f --- /dev/null +++ b/server/www/scripts/auth/services/account.js @@ -0,0 +1,22 @@ +// Faraday Penetration Test IDE +// Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +// See the file 'doc/LICENSE' for the license information + +angular.module('faradayApp'). + factory('AccountSrv', ["BASEURL", 'ServerAPI', function(BASEURL, ServerAPI){ + return { + changePassword: function(data) { + api_data = { + "new_password": data.newPassword, + "new_password_confirm": data.newPasswordRepeat, + "password": data.current + } + return ServerAPI.changePassword(api_data); + } + } +}]); + +angular.module('faradayApp'). + config(['$httpProvider', function($httpProvider) { + $httpProvider.interceptors.push('AuthInterceptor'); + }]); diff --git a/server/www/scripts/commons/controllers/homeCtrl.js b/server/www/scripts/commons/controllers/homeCtrl.js index 2accb641923..3079651a98e 100644 --- a/server/www/scripts/commons/controllers/homeCtrl.js +++ b/server/www/scripts/commons/controllers/homeCtrl.js @@ -10,6 +10,5 @@ angular.module('faradayApp') if(!auth) { $location.path('/login'); } - return deferred.reject(response); }); }]); \ No newline at end of file diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index 70fb560163d..ce8e2e52ad7 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -497,5 +497,10 @@ angular.module("faradayApp") return _delete(dbUrl, false); } + ServerAPI.changePassword = function(data) { + var url = BASEURL + "_api/change"; + return serverComm('POST', url, data); + } + return ServerAPI; }]); From cd81db6b2bf9ea42475a7a9b0970b15230a8801d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 11 Jan 2018 17:29:37 -0300 Subject: [PATCH 0774/1506] Set type field of host and service read only in the API. It was incorrectly writable and this broke the plugin system --- server/api/modules/hosts.py | 2 +- server/api/modules/services.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 16aa6b12ff4..3c8ee122609 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -51,7 +51,7 @@ class HostSchema(AutoSchema): default=[]), fields.List(fields.String)) metadata = SelfNestedField(MetadataSchema()) - type = fields.Function(lambda obj: 'Host') + type = fields.Function(lambda obj: 'Host', dump_only=True) class Meta: model = Host diff --git a/server/api/modules/services.py b/server/api/modules/services.py index 6ad4edb2a3f..6b54f3a5e4d 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -40,7 +40,7 @@ class ServiceSchema(AutoSchema): vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) credentials = fields.Integer(attribute='credentials_count', dump_only=True) metadata = SelfNestedField(MetadataSchema()) - type = fields.Function(lambda obj: 'Host') + type = fields.Function(lambda obj: 'Host', dump_only=True) def load_ports(self, value): # TODO migration: handle empty list and not numeric value From fee5a5373e8e42e102d77fc96f64a4d4440a9095 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 11 Jan 2018 17:30:31 -0300 Subject: [PATCH 0775/1506] Fix workspace provider typo It broke the commands widgets --- server/www/scripts/dashboard/providers/dashboard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/dashboard/providers/dashboard.js b/server/www/scripts/dashboard/providers/dashboard.js index 05cfb4380c3..228498b9deb 100644 --- a/server/www/scripts/dashboard/providers/dashboard.js +++ b/server/www/scripts/dashboard/providers/dashboard.js @@ -251,7 +251,7 @@ angular.module('faradayApp') _cmd.user = _cmd.user || "unknown"; _cmd.hostname = _cmd.hostname || "unknown"; _cmd.ip = _cmd.ip || "0.0.0.0"; - if(_cmd.duration == "In progres") { + if(_cmd.duration == "In progress") { _cmd.duration = "In progress"; } else if (_cmd.duration == "Not started") { _cmd.duration = "Not started"; From fd6f93b4989bbc9f523438410737a52fef3cbaec Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 11 Jan 2018 17:50:08 -0300 Subject: [PATCH 0776/1506] Rename class --- server/api/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index f2f34144f76..179ffc032fb 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -444,7 +444,7 @@ def _perform_create(self, data, **kwargs): return obj -class CommandMixing(): +class CommandMixin(): """ Created the command obj to log model activity after a command execution via the api (ex. from plugins) @@ -485,7 +485,7 @@ def _set_command_id(self, obj, created): db.session.add(command_object) -class CreateWorkspacedMixin(CreateMixin, CommandMixing): +class CreateWorkspacedMixin(CreateMixin, CommandMixin): """Add POST // route""" def _perform_create(self, data, workspace_name): @@ -530,7 +530,7 @@ def _perform_update(self, object_id, obj): db.session.commit() -class UpdateWorkspacedMixin(UpdateMixin, CommandMixing): +class UpdateWorkspacedMixin(UpdateMixin, CommandMixin): """Add PUT // route""" def _perform_update(self, object_id, obj, workspace_name): From 74da53be1b482a9a505f97c197354dbedffb5d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 11 Jan 2018 18:11:47 -0300 Subject: [PATCH 0777/1506] Allow filtering by creator command tool in the vulns API --- server/api/modules/vulns.py | 10 ++++++++-- test_cases/test_api_vulnerability.py | 29 ++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index af7031435c7..8370f97f421 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -264,17 +264,22 @@ def filter(self, query, model, attr, value): return query.filter(model.__table__.c.type == type_map[value]) +class CreatorFilter(Filter): + def filter(self, query, model, attr, value): + return query.filter(model.creator_command_tool == value) + + class VulnerabilityFilterSet(FilterSet): class Meta(FilterSetMeta): model = VulnerabilityWeb # It has all the fields - # TODO migration: Check if we should add fields creator, owner, + # TODO migration: Check if we should add fields owner, # command, impact, service, issuetracker, tags, date, # host, easeofresolution, evidence, policy violations, hostnames fields = ( "status", "website", "parameter_name", "query_string", "path", "data", "severity", "confirmed", "name", "request", "response", "parameters", "resolution", "method", "ease_of_resolution", - "description", "command_id", "target") + "description", "command_id", "target", "creator") strict_fields = ( "severity", "confirmed", "method" @@ -287,6 +292,7 @@ class Meta(FilterSetMeta): target = TargetFilter(fields.Str()) type = TypeFilter(fields.Str(validate=[OneOf(['Vulnerability', 'VulnerabilityWeb'])])) + creator = CreatorFilter(fields.Str()) def filter(self): """Generate a filtered query from request parameters. diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 3411e57e231..78eb05b1d3d 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1112,6 +1112,7 @@ def test_type_filter(workspace, session, filter_ = VulnerabilityFilterSet().filters['type'] std_vulns = vulnerability_factory.create_batch(10, workspace=workspace) web_vulns = vulnerability_web_factory.create_batch(10, workspace=workspace) + session.commit() std_filter = filter_.filter(VulnerabilityGeneric.query, VulnerabilityGeneric, @@ -1126,3 +1127,31 @@ def test_type_filter(workspace, session, 'VulnerabilityWeb' ) assert {v.id for v in web_filter} == {v.id for v in web_vulns} + + +def test_creator_filter(workspace, session, + empty_command_factory, command_object_factory, + vulnerability_factory, vulnerability_web_factory): + filter_ = VulnerabilityFilterSet().filters['creator'] + std_vulns = vulnerability_factory.create_batch(10, + workspace=workspace)[:5] + web_vulns = vulnerability_web_factory.create_batch(10, + workspace=workspace)[:5] + command = empty_command_factory.create(workspace=workspace, + tool="metasploit") + session.commit() + + vulns = std_vulns + web_vulns + session.flush() + for vuln in vulns: + command_object_factory.create(command=command, + object_type='vulnerability', + object_id=vuln.id, + workspace=workspace) + session.commit() + + filtered = filter_.filter(VulnerabilityGeneric.query, + VulnerabilityGeneric, + 'creator', + 'metasploit') + assert {v.id for v in filtered} == {v.id for v in vulns} From 79b17cb130aa3b1ac62a938383779740412faf2a Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 12 Jan 2018 10:56:35 -0300 Subject: [PATCH 0778/1506] Avoid current_psql_output undefined error. moving initialization --- server/commands/initdb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index 613086d1f58..57be66fd560 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -67,6 +67,7 @@ def run(self): # current_psql_output is for checking psql command already known errors for each execution. psql_log_filename = os.path.join(faraday_path_conf, 'logs', 'psql_log.log') configure_existing_user = None + current_psql_output = TemporaryFile() with open(psql_log_filename, 'a+') as psql_log_file: while configure_existing_user is None: configure_existing_user = raw_input('Do you {blue} already have {white} a postgresql username and password? (yes/no): '.format(blue=Fore.BLUE, white=Fore.WHITE)) @@ -84,7 +85,7 @@ def run(self): if hostname not in ['localhost', '127.0.0.1']: print('ERROR: can only create postgresql user on localhost') sys.exit(1) - current_psql_output = TemporaryFile() + username, password, process_status = self._configure_new_postgres_user(current_psql_output) current_psql_output.seek(0) psql_output = current_psql_output.read() From 5023e3220b33c107bbfd3e146b2ff2b14bd13111 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 12 Jan 2018 11:33:15 -0300 Subject: [PATCH 0779/1506] Close modal if change password was successful --- server/www/scripts/auth/controllers/resetPassword.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/auth/controllers/resetPassword.js b/server/www/scripts/auth/controllers/resetPassword.js index 97f164c36d9..aa03633a28f 100644 --- a/server/www/scripts/auth/controllers/resetPassword.js +++ b/server/www/scripts/auth/controllers/resetPassword.js @@ -26,7 +26,7 @@ angular.module('faradayApp') $scope.ok = function(){ AccountSrv.changePassword($scope.data).then( function(response, statusText, xhrObj){ - $scope.modal.close() + $modalInstance.close() }, function(xhrObj, textStatus, err) { if (xhrObj.data.response.errors.password != undefined) { $scope.form.current.$setValidity("current", false); From 9e20c75e31633024132a9513113e155515dc4112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 12 Jan 2018 12:03:30 -0300 Subject: [PATCH 0780/1506] Improve hosts view header Move change workspace and refresh bar and search text input to make it similar to the status report. --- server/www/scripts/hosts/partials/list.html | 252 ++++++++++---------- 1 file changed, 128 insertions(+), 124 deletions(-) diff --git a/server/www/scripts/hosts/partials/list.html b/server/www/scripts/hosts/partials/list.html index bfadda405a6..bf7438b4552 100644 --- a/server/www/scripts/hosts/partials/list.html +++ b/server/www/scripts/hosts/partials/list.html @@ -5,135 +5,139 @@
    -

    - Hosts for {{workspace}} ({{totalHosts}}) -
    - - - -
    - - - - - New - -

    -
    -
    -
    -
    -
    - - - - - - -
    -
    - Show hostnames -
    -
    -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - Name - - Hostnames - - Open Services - - Vulns - - Credentials - - OS - - Owned - - Last modified -
    - {{host.name}} - - - - {{hostname}}
    -     -     		- {{host.credentials}} - - - - - - - - owned - not yet - - -
    -
    -
    - -
    -
    - + +
    + + + + + New + +
    +
    +
    + +
    +
    + + + + + + +
    +
    + Show hostnames +
    - -
    -
    + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Name + + Hostnames + + Open Services + + Vulns + + Credentials + + OS + + Owned + + Last modified +
    + {{host.name}} + + + + {{hostname}}
    +     +     		+ {{host.credentials}} + + + + + + + + owned + not yet + + +
    +
    +
    + +
    +
    + +
    + + +
    +
    +
    +
    From 65c41f38ccaead787cf1bce377063ac2b3ab5a36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 12 Jan 2018 12:46:06 -0300 Subject: [PATCH 0781/1506] Allow enabling and disabling table fields in the host list --- server/www/scripts/hosts/controllers/hosts.js | 22 +++++++- server/www/scripts/hosts/partials/list.html | 54 ++++++++++++------- 2 files changed, 56 insertions(+), 20 deletions(-) diff --git a/server/www/scripts/hosts/controllers/hosts.js b/server/www/scripts/hosts/controllers/hosts.js index 8e543cd5c8e..ec3b6758958 100644 --- a/server/www/scripts/hosts/controllers/hosts.js +++ b/server/www/scripts/hosts/controllers/hosts.js @@ -9,10 +9,19 @@ angular.module('faradayApp') var init = function() { $scope.selectall_hosts = false; - $scope.showHostnames = true; // hosts list $scope.hosts = []; $scope.totalHosts = 0; + $scope.columns = { + "name": true, + "hostnames": false, + "services": true, + "vulns": true, + "credentials": true, + "os": true, + "owned": true, + "last_modified": true, + }; // current workspace $scope.workspace = $routeParams.wsId; @@ -265,6 +274,17 @@ angular.module('faradayApp') }); }; + $scope.hasDisabledFields = function(){ + return Object.values($scope.columns).some(function(show){ + return !show + }); + }; + + $scope.toggleShow = function(column) { + $scope.columns[column] = !$scope.columns[column]; + return false; + }; + // toggles sort field and order $scope.toggleSort = function(field) { if ($scope.sortField != field) { diff --git a/server/www/scripts/hosts/partials/list.html b/server/www/scripts/hosts/partials/list.html index bf7438b4552..ff4c0b25738 100644 --- a/server/www/scripts/hosts/partials/list.html +++ b/server/www/scripts/hosts/partials/list.html @@ -47,39 +47,55 @@

    -
    - Show hostnames -
    +
    +

    Add columns

    + +
    - - - - - - - - @@ -90,32 +106,32 @@

    selection-model-selected-class="multi-selected" selection-model-on-change="selectedHosts()">

    - - - - - + + - - - From 8b5f80bcf9c5d2e1606cfd0adb44812eaa219cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 12 Jan 2018 13:02:03 -0300 Subject: [PATCH 0782/1506] Add creation, mac, and create time fields in the hosts view --- server/www/scripts/hosts/controllers/hosts.js | 3 +++ server/www/scripts/hosts/partials/list.html | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/server/www/scripts/hosts/controllers/hosts.js b/server/www/scripts/hosts/controllers/hosts.js index ec3b6758958..6b7ce6b3ebb 100644 --- a/server/www/scripts/hosts/controllers/hosts.js +++ b/server/www/scripts/hosts/controllers/hosts.js @@ -14,12 +14,15 @@ angular.module('faradayApp') $scope.totalHosts = 0; $scope.columns = { "name": true, + "description": false, "hostnames": false, + "mac": false, "services": true, "vulns": true, "credentials": true, "os": true, "owned": true, + "create_time": true, "last_modified": true, }; // current workspace diff --git a/server/www/scripts/hosts/partials/list.html b/server/www/scripts/hosts/partials/list.html index ff4c0b25738..c850d6222e9 100644 --- a/server/www/scripts/hosts/partials/list.html +++ b/server/www/scripts/hosts/partials/list.html @@ -69,10 +69,18 @@

    Add columns

    Name +     + + + + From 8b30f6a74c7bd0a996984a8ee9f9e400e5038620 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 12 Jan 2018 13:10:42 -0300 Subject: [PATCH 0783/1506] Remember the fields chosen in the table list --- server/www/scripts/hosts/controllers/hosts.js | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/server/www/scripts/hosts/controllers/hosts.js b/server/www/scripts/hosts/controllers/hosts.js index 6b7ce6b3ebb..af53a83b71a 100644 --- a/server/www/scripts/hosts/controllers/hosts.js +++ b/server/www/scripts/hosts/controllers/hosts.js @@ -12,19 +12,23 @@ angular.module('faradayApp') // hosts list $scope.hosts = []; $scope.totalHosts = 0; - $scope.columns = { - "name": true, - "description": false, - "hostnames": false, - "mac": false, - "services": true, - "vulns": true, - "credentials": true, - "os": true, - "owned": true, - "create_time": true, - "last_modified": true, - }; + if($cookies.get('HColumns')) { + $scope.columns = JSON.parse($cookies.get('HColumns')) + }else{ + $scope.columns = { + "name": true, + "description": false, + "hostnames": false, + "mac": false, + "services": true, + "vulns": true, + "credentials": true, + "os": true, + "owned": true, + "create_time": true, + "last_modified": true, + }; + } // current workspace $scope.workspace = $routeParams.wsId; @@ -285,7 +289,7 @@ angular.module('faradayApp') $scope.toggleShow = function(column) { $scope.columns[column] = !$scope.columns[column]; - return false; + $cookies.put('HColumns', JSON.stringify($scope.columns)); }; // toggles sort field and order From c6865eade1e07f3a153ffb67aca4ca6b3580fac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 12 Jan 2018 13:12:51 -0300 Subject: [PATCH 0784/1506] Move OS column to the left in hosts view --- server/www/scripts/hosts/partials/list.html | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/server/www/scripts/hosts/partials/list.html b/server/www/scripts/hosts/partials/list.html index c850d6222e9..584bf5bfe44 100644 --- a/server/www/scripts/hosts/partials/list.html +++ b/server/www/scripts/hosts/partials/list.html @@ -73,6 +73,10 @@

    Add columns

    Description +     - - + - - - - - - + + - + + - + @@ -159,6 +156,9 @@

    Add columns

    ownednot yet + From 4f1385f48d1f23a1b0efc264a79cfff0eadcbfa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 19 Jan 2018 17:42:05 -0300 Subject: [PATCH 0830/1506] Allow grouping vulns by creator in the status report --- server/api/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/api/base.py b/server/api/base.py index b7758f95202..ee00357c9c7 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -318,6 +318,7 @@ def _get_order_field(self, **kwargs): else: for (key, value) in metadata_field.target_schema.fields.items(): schema.fields['metadata.' + key] = value + schema.fields[key] = value try: field_instance = schema.fields[order_field] From 6e57cf9481f9750026f148b377abfb9f0d468754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 19 Jan 2018 17:58:48 -0300 Subject: [PATCH 0831/1506] Improve user with no password warning in couchdb importer Warn that the user was created and its password is "changeme" --- server/importer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index 2ee7dec09dc..2d73d2e20bc 100644 --- a/server/importer.py +++ b/server/importer.py @@ -938,7 +938,8 @@ def get_hash_from_document(self, doc): scheme = doc.get('password_scheme', 'unset') if scheme != 'pbkdf2': # Flask Security will encrypt the password next time the user logs in. - logger.warning('Found user {0} without password.'.format(doc.get('name'))) + logger.warning('Found user {0} without password. Setting its ' + 'password to "changeme"'.format(doc.get('name'))) return 'changeme' return self.modular_crypt_pbkdf2_sha1(doc['derived_key'], doc['salt'], doc['iterations']) From f6b49310c4305f83e40507769e9a6d9ab36d2a30 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 19 Jan 2018 18:00:23 -0300 Subject: [PATCH 0832/1506] Fixes in Arachni plugin by crodriguez --- plugins/repo/arachni/plugin.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/plugins/repo/arachni/plugin.py b/plugins/repo/arachni/plugin.py index f1e75f000bc..398c1ec0e1e 100755 --- a/plugins/repo/arachni/plugin.py +++ b/plugins/repo/arachni/plugin.py @@ -443,21 +443,22 @@ def parseOutputString(self, output, debug=False): # Create issues. for issue in parser.issues: - description = ( - 'Description:\n' + - issue.description + - '\n\nSolution:\n' + - issue.remedy_guidance) + description = issue.description.replace(' ', ' ').replace('\n', ' ').replace('. ', '.\n\n') + resol = issue.remedy_guidance.replace(' ', ' ').replace('\n', ' ').replace('. ', '.\n\n') references = issue.references if issue.cwe != 'None': references.append('CWE-' + issue.cwe) - issue_id = self.createAndAddVulnWebToService( + if resol == 'None': + resol = '' + + self.createAndAddVulnWebToService( host_id, service_id, name=issue.name, desc=description, + resolution=resol, ref=references, severity=issue.severity, website=self.hostname, From beefeba3893cdc7231b651545e5bbf5f4f01656a Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 19 Jan 2018 18:54:28 -0300 Subject: [PATCH 0833/1506] Update page size in service view to 25 --- server/www/scripts/hosts/controllers/host.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/hosts/controllers/host.js b/server/www/scripts/hosts/controllers/host.js index ee5f4ad968a..3df50126baa 100644 --- a/server/www/scripts/hosts/controllers/host.js +++ b/server/www/scripts/hosts/controllers/host.js @@ -70,7 +70,7 @@ angular.module('faradayApp') loadHosts(); loadServices(hostId); - $scope.pageSize = 10; + $scope.pageSize = 25; $scope.currentPage = 1; $scope.newCurrentPage = 1; From f4f85659ba0228f42c761305bbb4b88e6c9e085d Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 19 Jan 2018 19:31:35 -0300 Subject: [PATCH 0834/1506] Generate new secret key --- server/app.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/server/app.py b/server/app.py index 89ad2d656b8..535f470aea5 100644 --- a/server/app.py +++ b/server/app.py @@ -4,8 +4,11 @@ import logging import os +import string from os.path import join, expanduser +from random import SystemRandom +from server.config import LOCAL_CONFIG_FILE from server.models import User try: @@ -117,10 +120,24 @@ def log_queries_count(response): # map(str,Counter(q.statement for q in queries).most_common())) return response +def save_new_secret_key(app): + config = ConfigParser() + config.read(LOCAL_CONFIG_FILE) + rng = SystemRandom() + secret_key = "".join([rng.choice(string.ascii_letters + string.digits) for _ in xrange(25)]) + app.config['SECRET_KEY'] = secret_key + config.set('faraday_server', 'secret_key', secret_key) + with open(LOCAL_CONFIG_FILE, 'w') as configfile: + config.write(configfile) + def create_app(db_connection_string=None, testing=None): app = Flask(__name__) - app.config['SECRET_KEY'] = 'supersecret' + try: + app.config['SECRET_KEY'] = server.config.faraday_server.secret_key + except Exception: + save_new_secret_key(app) + app.config['SECURITY_PASSWORD_SINGLE_HASH'] = True app.config['WTF_CSRF_ENABLED'] = False app.config['SECURITY_USER_IDENTITY_ATTRIBUTES'] = ['username'] From 72b82965c57a036c25e715b75b6af61374d8dd75 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 19 Jan 2018 19:52:41 -0300 Subject: [PATCH 0835/1506] Fix bug hackaton: search ip in gtk don't work --- gui/gtk/mainwidgets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui/gtk/mainwidgets.py b/gui/gtk/mainwidgets.py index 7703543282c..58711580c9f 100644 --- a/gui/gtk/mainwidgets.py +++ b/gui/gtk/mainwidgets.py @@ -469,7 +469,7 @@ def on_click_move_page(self, button, change_page_number_func, *args, **kwargs): self.page = change_page_number_func(self.page) hosts = self.get_hosts_function(page=str(self.page), page_size=20, - name=self.search_entry.get_text(), + search=self.search_entry.get_text(), sort='vulns', sort_dir='desc') self.reset_model(hosts) From 9817f5b763c12fb9e807255d77e4c1e5733033fd Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 19 Jan 2018 20:38:58 -0300 Subject: [PATCH 0836/1506] Fix bug hackaton: select host in HostSidebar and show it --- gui/gtk/application.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gui/gtk/application.py b/gui/gtk/application.py index c1e4dd9bf6f..7ca43482f1d 100644 --- a/gui/gtk/application.py +++ b/gui/gtk/application.py @@ -399,15 +399,14 @@ def show_host_info(self, host_id): looking for the host.""" current_ws_name = self.get_active_workspace().name - # for host in self.model_controller.getAllHosts(): - host = self.serverIO.get_hosts(couchid=host_id) + host = self.serverIO.get_host(host_id) if not host: self.show_normal_error("The host you clicked isn't accessible. " "This is most probably due to an internal " "error.") return False - info_window = HostInfoDialog(self.window, current_ws_name, host[0]) + info_window = HostInfoDialog(self.window, current_ws_name, host) info_window.show_all() return True From 1ca1866779f21ec956b027acd0fbb25496192a09 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 22 Jan 2018 15:18:43 -0300 Subject: [PATCH 0837/1506] Fix an error when a vuln template was create twice * When a vuln template already exists with the same name api returned error 500. * Added unit test to reproduce the error. * Show 409 error on webui when the template already exists. * Implemented method for generic view and use the method from workspaced view (removed TODO) --- server/api/base.py | 70 ++++++++++--------- server/api/modules/vulnerability_template.py | 1 + .../statusReport/controllers/statusReport.js | 4 +- .../www/scripts/vulndb/providers/vulnModel.js | 8 ++- test_cases/test_api_vulnerability_template.py | 24 +++++++ 5 files changed, 71 insertions(+), 36 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index b7758f95202..966d9f78246 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -150,9 +150,42 @@ def _parse_data(self, schema, request, *args, **kwargs): return FlaskParser().parse(schema, request, locations=('json',), *args, **kwargs) - def _validate_uniqueness(self, obj, object_id=None): - # TODO: Implement this - return True + def _validate_uniqueness(self, obj, object_id=None, unique_fields=None): + primary_key_field = inspect(self.model_class).primary_key[0] + + if getattr(obj, 'workspace', None): + query = self._get_base_query(obj.workspace.name) + else: + query = self._get_base_query() + + if object_id is not None: + # The object already exists in DB, we want to fetch an object + # different to this one but with the same unique field + query = query.filter(primary_key_field != object_id) + if unique_fields is None: + # It is usually None, but in some case the child class may want to + # override it based on some condition dependent on the request + unique_fields = self.unique_fields + for field_names in unique_fields: + for field_name in field_names: + # Use type(obj) instead of self.model_class to be + # compatible with polymorphic obejcts (e.g. Vulnerability) + field = getattr(type(obj), field_name) + value = getattr(obj, field_name) + query = query.filter( + field == value) + + existing_obj = query.one_or_none() + conflict_data = self._get_schema_class()().dump(existing_obj).data + if existing_obj: + db.session.rollback() + abort(409, ValidationError( + { + 'message': 'Existing value for unique columns: %s' % ( + field_names,), + 'object': conflict_data, + } + )) @classmethod def register(cls, app, *args, **kwargs): @@ -230,38 +263,9 @@ def _set_schema_context(self, context, **kwargs): return context def _validate_uniqueness(self, obj, object_id=None, unique_fields=None): - # TODO: Use implementation of GenericView assert obj.workspace is not None, "Object must have a " \ "workspace attribute set to call _validate_uniqueness" - primary_key_field = inspect(self.model_class).primary_key[0] - query = self._get_base_query(obj.workspace.name) - if object_id is not None: - # The object already exists in DB, we want to fetch an object - # different to this one but with the same unique field - query = query.filter(primary_key_field != object_id) - if unique_fields is None: - # It is usually None, but in some case the child class may want to - # override it based on some condition dependent on the request - unique_fields = self.unique_fields - for field_names in unique_fields: - for field_name in field_names: - # Use type(obj) instead of self.model_class to be - # compatible with polymorphic obejcts (e.g. Vulnerability) - field = getattr(type(obj), field_name) - value = getattr(obj, field_name) - query = query.filter( - field == value) - - existing_obj = query.one_or_none() - conflict_data = self._get_schema_class()().dump(existing_obj).data - if existing_obj: - db.session.rollback() - abort(409, ValidationError( - { - 'message': 'Existing value for unique columns: %s' % (field_names, ), - 'object': conflict_data, - } - )) + return super(GenericWorkspacedView, self)._validate_uniqueness(obj, object_id, unique_fields) class ListMixin(object): diff --git a/server/api/modules/vulnerability_template.py b/server/api/modules/vulnerability_template.py index 2eb289cba49..e4750721d53 100644 --- a/server/api/modules/vulnerability_template.py +++ b/server/api/modules/vulnerability_template.py @@ -71,6 +71,7 @@ class Meta(FilterSetMeta): class VulnerabilityTemplateView(PaginatedMixin, FilterAlchemyMixin, ReadWriteView): + unique_fields = [('name',)] route_base = 'vulnerability_template' model_class = VulnerabilityTemplate schema_class = VulnerabilityTemplateSchema diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 353a0e9865c..3b781a0fb21 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -455,8 +455,8 @@ angular.module('faradayApp') }); $q.all(promises).then(function(success) { showMessage("Created " + selected.length + " templates successfully.", true); - }, function(failed) { - showMessage("Something failed when creating some of the templates."); + }, function(failedMessage) { + showMessage(failedMessage); }); } catch(err) { showMessage("Something failed when creating some of the templates."); diff --git a/server/www/scripts/vulndb/providers/vulnModel.js b/server/www/scripts/vulndb/providers/vulnModel.js index d4ff3383f57..f334cd3d154 100644 --- a/server/www/scripts/vulndb/providers/vulnModel.js +++ b/server/www/scripts/vulndb/providers/vulnModel.js @@ -97,7 +97,13 @@ angular.module('faradayApp'). msg += res.data.messages[item][0]; } } - deferred.reject("Unable to save the Vuln Model. " + msg); + var message; + if (res.status == 409) { + message = "Vulnerability template already exists. " + res.data.message + } else { + message = "Unable to save the Vuln Model. " + msg; + } + deferred.reject(message); } catch(err) { deferred.reject(err); } diff --git a/test_cases/test_api_vulnerability_template.py b/test_cases/test_api_vulnerability_template.py index 099aed85d70..16d7d6a27ba 100644 --- a/test_cases/test_api_vulnerability_template.py +++ b/test_cases/test_api_vulnerability_template.py @@ -134,3 +134,27 @@ def test_delete_vuln_template(self, session, test_client): assert res.status_code == 204 assert vuln_count_previous - 1 == session.query(VulnerabilityTemplate).count() + + def test_create_same_vuln_template_multipe_times_sohuld_raise_409(self, test_client): + """ + Current test case was found un hackaton. When creating vuln + template it returned error 500 instead of 409. + + """ + + raw_data = { + "id":123010, + "cwe": "", + "description": "test2", + "desc":"test2", + "exploitation":"critical", + "name":"test2", + "references":[], + "refs":[], + "resolution":"", + "type":"vulnerability_template" + } + res = test_client.post(self.url(), data=raw_data) + assert res.status_code == 201 + res = test_client.post(self.url(), data=raw_data) + assert res.status_code == 409 \ No newline at end of file From 42f4c24c120920f829cfd39e30595d775c6ccbd3 Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 22 Jan 2018 15:43:12 -0300 Subject: [PATCH 0838/1506] Add ignore of testing dirs --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 9de77059c56..ccf92bafb76 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,8 @@ pip-log.txt .coverage .tox nosetests.xml +test_cases/.cache/ +test_cases/htmlcov/ # Translations *.mo From 37299b5415361bc857db3215e45769f75f3acb82 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 23 Jan 2018 18:11:54 -0300 Subject: [PATCH 0839/1506] Fix bug when calling super --- server/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/web.py b/server/web.py index 5e0cac598c6..5d54e42cdae 100644 --- a/server/web.py +++ b/server/web.py @@ -58,7 +58,7 @@ def render(self, request): class FaradayRedirectResource(Redirect, object): def render(self, request): request.responseHeaders.removeHeader('Server') - return super(FaradayWSGIResource, self).render(request) + return super(FaradayRedirectResource, self).render(request) class WebServer(object): From d40ace22a8956c1e0906049cdb40c99051177a9d Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 23 Jan 2018 18:12:17 -0300 Subject: [PATCH 0840/1506] API now return 401 --- persistence/server/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence/server/server.py b/persistence/server/server.py index c9f94c73d4c..14a51638b05 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -1491,7 +1491,7 @@ def login_user(uri, uname, upass): auth = {"email": uname, "password": upass} try: resp = requests.post(uri + "/_api/login", json=auth) - if resp.status_code == 400: + if resp.status_code == 401: return None else: return resp.cookies From f43ec996f145e79bf3c913ca771785c1494440d9 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 23 Jan 2018 18:12:29 -0300 Subject: [PATCH 0841/1506] Add deleted code by mistake --- plugins/manager.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/plugins/manager.py b/plugins/manager.py index 7d34b538467..968cfde9ff0 100644 --- a/plugins/manager.py +++ b/plugins/manager.py @@ -100,9 +100,19 @@ def _loadPlugins(self, plugin_repo_path): module_path = os.path.join(plugin_repo_path, name) sys.path.append(module_path) module_filename = os.path.join(module_path, "plugin.py") + if not os.path.exists(module_filename): + module_filename = os.path.join(module_path, + "plugin.pyc") + + file_ext = os.path.splitext(module_filename)[1] + if file_ext.lower() == '.py': + self._plugin_modules[name] = imp.load_source(name, + module_filename) + + elif file_ext.lower() == '.pyc': + self._plugin_modules[name] = imp.load_compiled(name, + module_filename) getLogger(self).debug('Loading plugin {0}'.format(name)) - self._plugin_modules[name] = imp.load_source( - name, module_filename) except Exception as e: msg = "An error ocurred while loading plugin %s.\n%s" % ( module_filename, traceback.format_exc()) From a8c7bd07ae9a7fcdb50a8b4629b900aba5651cf3 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 23 Jan 2018 18:13:30 -0300 Subject: [PATCH 0842/1506] Improve vuln edit when api returns 409 * If the vuln already exists when editing, a message is shown to the user. * When an error occurs on the API show to the user the message. * Avoid closing the modal when a conflicts was raised. * Moved code from status report controller to the modal controller of status report. --- .../www/scripts/commons/providers/commons.js | 18 +++++++++ .../statusReport/controllers/modalEdit.js | 17 +++++--- .../statusReport/controllers/statusReport.js | 39 +++---------------- server/www/scripts/vulns/providers/vuln.js | 8 ++-- 4 files changed, 40 insertions(+), 42 deletions(-) diff --git a/server/www/scripts/commons/providers/commons.js b/server/www/scripts/commons/providers/commons.js index 536d9b890de..6221a259be3 100644 --- a/server/www/scripts/commons/providers/commons.js +++ b/server/www/scripts/commons/providers/commons.js @@ -257,6 +257,24 @@ angular.module('faradayApp') return url; }; + commonsFact.showMessage = function(msg, success) { + if (! success) { var success = false } + if (success) { + var templateUrl = 'scripts/commons/partials/modalOK.html'; + } else { + var templateUrl = 'scripts/commons/partials/modalKO.html'; + } + var modal = $uibModal.open({ + templateUrl: templateUrl, + controller: 'commonsModalKoCtrl', + resolve: { + msg: function() { + return msg; + }, + } + }); + } + commonsFact.errorDialog = function(message) { $uibModal.open(config = { templateUrl: 'scripts/commons/partials/modalKO.html', diff --git a/server/www/scripts/statusReport/controllers/modalEdit.js b/server/www/scripts/statusReport/controllers/modalEdit.js index 5fd1a335700..2f03073ed07 100644 --- a/server/www/scripts/statusReport/controllers/modalEdit.js +++ b/server/www/scripts/statusReport/controllers/modalEdit.js @@ -3,8 +3,8 @@ // See the file 'doc/LICENSE' for the license information angular.module('faradayApp') - .controller('modalEditCtrl', ['$modalInstance', 'EASEOFRESOLUTION', 'STATUSES', 'commonsFact', 'severities', 'vuln', 'vulnModelsManager', - function($modalInstance, EASEOFRESOLUTION, STATUSES, commons, severities, vuln, vulnModelsManager) { + .controller('modalEditCtrl', ['$modalInstance', 'EASEOFRESOLUTION', 'STATUSES', 'commonsFact', 'severities', 'vuln', 'vulnModelsManager', 'vulnsManager', + function($modalInstance, EASEOFRESOLUTION, STATUSES, commonsFact, severities, vuln, vulnModelsManager, vulnsManager) { var vm = this; @@ -75,7 +75,7 @@ angular.module('faradayApp') // TODO: EVIDENCE SHOUD BE LOADED ALREADY? if(vm.vuln._attachments !== undefined) { vm.data._attachments = vm.vuln._attachments; - vm.icons = commons.loadIcons(vm.data._attachments); + vm.icons = commonsFact.loadIcons(vm.data._attachments); } }; @@ -93,7 +93,7 @@ angular.module('faradayApp') vm.file_name_error = true; } }); - vm.icons = commons.loadIcons(vm._attachments); + vm.icons = commonsFact.loadIcons(vm._attachments); } vm.removeEvidence = function(name) { @@ -123,7 +123,14 @@ angular.module('faradayApp') policyviolations.push(policyviolation.value); }); vm.data.policyviolations = policyviolations; - $modalInstance.close(vm.data); + + vulnsManager.updateVuln(vm.vuln, vm.data).then(function(){ + $modalInstance.close(vm.data); + }, function(data){ + commonsFact.showMessage("Error updating vuln " + vm.vuln.name + " (" + vm.vuln._id + "): " + data.message); + }); + + }; vm.cancel = function() { diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index f562295fc09..c917aebbb3c 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -564,24 +564,6 @@ angular.module('faradayApp') loadVulns(); }; - var showMessage = function(msg, success) { - if (! success) { var success = false } - if (success) { - var templateUrl = 'scripts/commons/partials/modalOK.html'; - } else { - var templateUrl = 'scripts/commons/partials/modalKO.html'; - } - var modal = $uibModal.open({ - templateUrl: templateUrl, - controller: 'commonsModalKoCtrl', - resolve: { - msg: function() { - return msg; - } - } - }); - } - // deletes the vulns in the array $scope.remove = function(aVulns) { aVulns.forEach(function(vuln) { @@ -678,15 +660,13 @@ angular.module('faradayApp') } } }); - modal.result.then(function(data) { - vulnsManager.updateVuln(vulns[0], data).then(function(){ - loadVulns(); - }, function(errorMsg){ - showMessage("Error updating vuln " + vulns[0].name + " (" + vulns[0]._id + "): " + errorMsg); - }); + + modal.result.then(function() { + loadVulns(); }); + } else { - showMessage('A vulnierabilty must be selected in order to edit'); + showMessage('A vulnerability must be selected in order to edit'); } }; @@ -911,14 +891,7 @@ angular.module('faradayApp') } showMessage(msg); }); - /* - // this shouldnt be necessary, we should use Angular formatting options directly in the partial - //formating the date - var d = new Date(0); - d.setUTCSeconds(vuln.date); - d = d.getDate() + "/" + (d.getMonth()+1) + "/" + d.getFullYear(); - vuln.date = d; - */ + }; var loadVulns = function() { diff --git a/server/www/scripts/vulns/providers/vuln.js b/server/www/scripts/vulns/providers/vuln.js index 449dbcec275..01d715380d9 100644 --- a/server/www/scripts/vulns/providers/vuln.js +++ b/server/www/scripts/vulns/providers/vuln.js @@ -128,8 +128,8 @@ angular.module('faradayApp') .then(function(response) { self.set(self.ws, vuln); deferred.resolve(); - }, function() { - deferred.reject(); + }, function(response) { + deferred.reject(response.data); }); }); } else { @@ -138,8 +138,8 @@ angular.module('faradayApp') self.set(self.ws, vuln); self._rev = response.rev; deferred.resolve(); - }, function() { - deferred.reject(); + }, function(response) { + deferred.reject(response.data); }); } From 39007a7322676297fd56c03585745f3e1637e938 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 24 Jan 2018 14:34:10 -0300 Subject: [PATCH 0843/1506] Add test to reproduce error 500 --- server/api/modules/vulns.py | 3 ++- test_cases/test_api_vulnerability.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 1f937456ef1..92d8e6031b8 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -177,6 +177,8 @@ def load_type(self, value): return 'vulnerability_web' def load_parent(self, value): + if type(value) != int: + raise ValidationError("Invalid parent type") return value @post_load @@ -203,7 +205,6 @@ def post_load_parent(self, data): parent_class = Service parent_field = 'service_id' if not parent_class: - print('parent_type', parent_type) raise ValidationError('Unknown parent type') try: diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index b2ac187f354..f1caf585ab7 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1133,6 +1133,24 @@ def test_create_vuln_from_command(self, test_client, session): assert cmd_obj.object_type == 'vulnerability' assert cmd_obj.object_id == res.json['_id'] + def test_with_invalid_id_returns_400(self, session, test_client): + """ + Bug found on hackaton. + """ + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='Vulnerability', + parent_id='', + parent_type='Host', + refs=[], + policyviolations=[], + description='helloworld', + severity='low', + ) + res = test_client.post(self.url(), data=raw_data) + assert res.status_code == 400 + def test_type_filter(workspace, session, vulnerability_factory, From 657865bc7430a9c7eba2e67a6a51a90eacdabd1e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 24 Jan 2018 15:14:55 -0300 Subject: [PATCH 0844/1506] Improvements and fixes on new vuln * move code to newModal and show the proper error message when the vuln already exists. * Some code simplificacion when using parents. --- .../statusReport/controllers/modalNew.js | 22 +++++++++---------- .../statusReport/controllers/statusReport.js | 15 +------------ server/www/scripts/vulns/providers/vuln.js | 2 +- server/www/scripts/vulns/providers/vulns.js | 4 +--- 4 files changed, 13 insertions(+), 30 deletions(-) diff --git a/server/www/scripts/statusReport/controllers/modalNew.js b/server/www/scripts/statusReport/controllers/modalNew.js index 61743e5257e..236bfe01198 100644 --- a/server/www/scripts/statusReport/controllers/modalNew.js +++ b/server/www/scripts/statusReport/controllers/modalNew.js @@ -4,8 +4,8 @@ angular.module('faradayApp') .controller('modalNewVulnCtrl', - ['$modalInstance', '$filter', '$upload', 'EASEOFRESOLUTION', 'commonsFact', 'severities', 'workspace', 'targetFact', 'vulnModelsManager', - function($modalInstance, $filter, $upload, EASEOFRESOLUTION, commons, severities, workspace, targetFact, vulnModelsManager) { + ['$modalInstance', '$filter', '$upload', 'EASEOFRESOLUTION', 'commonsFact', 'severities', 'workspace', 'targetFact', 'vulnModelsManager', 'vulnsManager', + function($modalInstance, $filter, $upload, EASEOFRESOLUTION, commonsFact, severities, workspace, targetFact, vulnModelsManager, vulnsManager) { var vm = this; @@ -105,7 +105,7 @@ angular.module('faradayApp') vm.file_name_error = true; } }); - vm.icons = commons.loadIcons(vm.data._attachments); + vm.icons = commonsFact.loadIcons(vm.data._attachments); }; vm.removeEvidence = function(name) { @@ -135,17 +135,15 @@ angular.module('faradayApp') }); vm.data.policyviolations = policyviolations; - var parents = vm.data.parents; - vm.data.parents = []; - parents.forEach(function(parent) { - var parent_type = "Service"; - if (Host.prototype.isPrototypeOf(parents[0])) { - parent_type = "Host"; + vulnsManager.createVuln(vm.workspace, vm.data).then(function(){ + $modalInstance.close(vm.data); + }, function(response){ + if (response.status == 409) { + commonsFact.showMessage("Error while creating a new Vulnerability " + vm.data.name + " Conflicting Vulnarability with id: " + response.data.object._id + ". " + response.data.message); + } else { + commonsFact.showMessage("Error from backend: " + response.status); } - vm.data.parents.push({parent_id: parent._id, type:parent_type}); }); - - $modalInstance.close(vm.data); }; vm.cancel = function() { diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index c917aebbb3c..53b0d6669c6 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -881,19 +881,6 @@ angular.module('faradayApp') }); }; - $scope.insert = function(vuln) { - vulnsManager.createVuln($scope.workspace, vuln).then(function() { - loadVulns(); - }, function(data) { - var msg = "The vulnerability couldn't be created"; - if(data.status == 409) { - msg += " because a vulnerability with the same parameters already exists in this Workspace"; - } - showMessage(msg); - }); - - }; - var loadVulns = function() { delete searchFilter.confirmed; if ($scope.confirmed) @@ -943,7 +930,7 @@ angular.module('faradayApp') }); modal.result.then(function(data) { - $scope.insert(data); + loadVulns(); }); }; diff --git a/server/www/scripts/vulns/providers/vuln.js b/server/www/scripts/vulns/providers/vuln.js index 01d715380d9..73c604c89f9 100644 --- a/server/www/scripts/vulns/providers/vuln.js +++ b/server/www/scripts/vulns/providers/vuln.js @@ -139,7 +139,7 @@ angular.module('faradayApp') self._rev = response.rev; deferred.resolve(); }, function(response) { - deferred.reject(response.data); + deferred.reject(response.data.message); }); } diff --git a/server/www/scripts/vulns/providers/vulns.js b/server/www/scripts/vulns/providers/vulns.js index 6fd28a5c49a..05d5c106734 100644 --- a/server/www/scripts/vulns/providers/vulns.js +++ b/server/www/scripts/vulns/providers/vulns.js @@ -12,11 +12,9 @@ angular.module('faradayApp') var parents = data.parents, promises = []; - delete data.parents; - parents.forEach(function(parent) { // we iterate parents when creating multiple vulns from new vuln modal. - data.parent = parent.parent_id; + data.parent = parent.id; data.parent_type = parent.type if(data.type == "Vulnerability") { From fb8d67f95cc56b7a3647f28c82b5b4584248a2cd Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 24 Jan 2018 16:15:55 -0300 Subject: [PATCH 0845/1506] Fix bind address for websockets --- server/web.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/web.py b/server/web.py index 5d54e42cdae..8966272536c 100644 --- a/server/web.py +++ b/server/web.py @@ -126,11 +126,12 @@ def run(self): self.__listen_func( self.__listen_port, site, interface=self.__bind_address) - listenWS(self.__build_websockets_resource()) + listenWS(self.__build_websockets_resource(), interface=self.__bind_address) reactor.run() except error.CannotListenError as e: logger.error(str(e)) sys.exit(1) except Exception as e: + logger.debug(e) logger.error('Something went wrong when trying to setup the Web UI') sys.exit(1) From 6cf75ca79389892b4ad188b2aa58e8eb0d949ac1 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 24 Jan 2018 16:17:56 -0300 Subject: [PATCH 0846/1506] Add websocket port configuration. Fix a bug when using secure websockets --- faraday-server.py | 4 ++++ server/default.ini | 1 + server/web.py | 11 +++++++++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/faraday-server.py b/faraday-server.py index 0e685b7eb2f..74cf000096c 100755 --- a/faraday-server.py +++ b/faraday-server.py @@ -89,6 +89,7 @@ def main(): parser.add_argument('--nodeps', action='store_true', help='Skip dependency check') parser.add_argument('--no-setup', action='store_true', help=argparse.SUPPRESS) parser.add_argument('--port', help='Overides server.ini port configuration') + parser.add_argument('--websocket_port', help='Overides server.ini websocket port configuration') parser.add_argument('--bind_address', help='Overides server.ini bind_address configuration') f = open(server.config.VERSION_FILE) @@ -121,6 +122,9 @@ def main(): if args.bind_address: server.config.faraday_server.bind_address = args.bind_address + if args.websocket_port: + server.config.faraday_server.websocket_port = args.websocket_port + if args.start: # Starts a new process on background with --ignore-setup # and without --start nor --stop diff --git a/server/default.ini b/server/default.ini index c06472a4b89..525e2e54dc5 100644 --- a/server/default.ini +++ b/server/default.ini @@ -1,6 +1,7 @@ [faraday_server] port=5985 bind_address=localhost +websocket_port=9000 [ssl] port=6985 diff --git a/server/web.py b/server/web.py index 8966272536c..55a2f073bb2 100644 --- a/server/web.py +++ b/server/web.py @@ -107,8 +107,15 @@ def __build_api_resource(self): return FaradayWSGIResource(reactor, reactor.getThreadPool(), app) def __build_websockets_resource(self): - logger.info(u"Websocket listening at wss://{0}:9000".format(self.__bind_address)) - factory = WorkspaceServerFactory(u"ws://{0}:9000".format(self.__bind_address)) + websocket_port = int(server.config.faraday_server.websocket_port) + url = '{0}:{1}'.format(self.__bind_address, websocket_port) + if self.__ssl_enabled: + url = 'wss://' + url + else: + url = 'ws://' + url + logger.info(u"Websocket listening at {url}".format(url=url)) + + factory = WorkspaceServerFactory(url=url) factory.protocol = BroadcastServerProtocol return factory From 66b42029a1303f508c9985207aa8b3ddd1d50d32 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 24 Jan 2018 16:29:26 -0300 Subject: [PATCH 0847/1506] Fix a bug: Avoid showing select workspace when user is not logged in (modal was shown in login screen). --- server/www/scripts/navigation/controllers/navigationCtrl.js | 6 ++++-- server/www/scripts/workspaces/providers/workspaces.js | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/server/www/scripts/navigation/controllers/navigationCtrl.js b/server/www/scripts/navigation/controllers/navigationCtrl.js index bd20b0f62a8..da945b23f79 100644 --- a/server/www/scripts/navigation/controllers/navigationCtrl.js +++ b/server/www/scripts/navigation/controllers/navigationCtrl.js @@ -50,8 +50,10 @@ angular.module('faradayApp') $scope.wss = wss; }); - workspacesFact.exists($routeParams.wsId).then(function(resp){ - if(resp !== true) { + workspacesFact.exists($routeParams.wsId).then(function(response){ + // ok! workspace was found. + }, function(response){ + if(response.status === 404) { $scope.modalWsNoExist(); } }); diff --git a/server/www/scripts/workspaces/providers/workspaces.js b/server/www/scripts/workspaces/providers/workspaces.js index 958401f8e6b..0fa4e60b2e4 100644 --- a/server/www/scripts/workspaces/providers/workspaces.js +++ b/server/www/scripts/workspaces/providers/workspaces.js @@ -57,9 +57,9 @@ angular.module('faradayApp') var deferred = $q.defer(); ServerAPI.getWorkspace(workspace_name).then( function(response) { - deferred.resolve(true); - }, function(error) { - deferred.resolve(false); + deferred.resolve(response); + }, function(response) { + deferred.reject(response); }); return deferred.promise; }; From 4aee4c6a56f411108d2541b0bca4e93cb94c2bb9 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 24 Jan 2018 18:08:31 -0300 Subject: [PATCH 0848/1506] Fix GTK pagination --- gui/gtk/application.py | 3 ++- gui/gtk/mainwidgets.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/gui/gtk/application.py b/gui/gtk/application.py index 7ca43482f1d..2b0ed76f889 100644 --- a/gui/gtk/application.py +++ b/gui/gtk/application.py @@ -555,8 +555,9 @@ def workspace_changed_event(): self.serverIO.active_workspace = event.workspace.name host_count, service_count, vuln_count = self.update_counts() total_host_amount = self.serverIO.get_hosts_number() - first_host_page = self.serverIO.get_hosts(page='0', page_size='20', + first_host_page = self.serverIO.get_hosts(page='1', page_size='20', sort='vulns', sort_dir='desc') + total_host_amount = self.serverIO.get_workspace_numbers()[0] GObject.idle_add(self.statusbar.set_workspace_label, event.workspace.name) GObject.idle_add(self.hosts_sidebar.reset_model_after_workspace_changed, diff --git a/gui/gtk/mainwidgets.py b/gui/gtk/mainwidgets.py index 58711580c9f..92394c1b698 100644 --- a/gui/gtk/mainwidgets.py +++ b/gui/gtk/mainwidgets.py @@ -138,7 +138,7 @@ def __init__(self, open_dialog_callback, get_several_hosts_function, self.progress_label = Gtk.Label("") self.host_amount_total = 0 self.host_amount_in_model = 0 - self.page = 0 + self.page = 1 self.host_id_to_iter = {} self.linux_icon = icons + "tux.png" self.windows_icon = icons + "windows.png" @@ -194,7 +194,7 @@ def reset_model(self, hosts): def reset_model_after_workspace_changed(self, hosts, total_host_amount): """Reset the model and also sets the page to 0 and the new total host amount will be the length of host.""" - self.page = 0 + self.page = 1 self.host_amount_total = total_host_amount self.reset_model(hosts) self.update_progress_label() @@ -426,11 +426,11 @@ def set_move_buttons_sensitivity(self): """Update the sensitity of the prev and next buttons according to the page we're on and the total number of pages. """ - self.prev_button.set_sensitive(self.page > 0) # its a boolean! + self.prev_button.set_sensitive(self.page >= 2) # its a boolean! # we add one to self.page 'cause they start at zero, but number of pages is # always at least one :) - self.next_button.set_sensitive(self.number_of_pages > self.page + 1) + self.next_button.set_sensitive(self.number_of_pages > self.page) def get_box(self): """Return the sidebar_box, which contains all the elements of the @@ -477,7 +477,7 @@ def on_click_move_page(self, button, change_page_number_func, *args, **kwargs): def update_progress_label(self): """Updates the progress label with values from self.page and self.number_of_pages.""" - self.progress_label.set_label("{0} / {1}".format(self.page + 1, self.number_of_pages)) + self.progress_label.set_label("{0} / {1}".format(self.page , self.number_of_pages)) def create_search_entry(self): """Returns a simple search entry""" From a72c3bd8f99455f0e05fc4e07dd2489c84c6b826 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 24 Jan 2018 18:14:54 -0300 Subject: [PATCH 0849/1506] Fix GTK bug with pagination and 0 hosts --- gui/gtk/mainwidgets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gui/gtk/mainwidgets.py b/gui/gtk/mainwidgets.py index 92394c1b698..8a2ad4487f5 100644 --- a/gui/gtk/mainwidgets.py +++ b/gui/gtk/mainwidgets.py @@ -147,6 +147,8 @@ def __init__(self, open_dialog_callback, get_several_hosts_function, @property def number_of_pages(self): + if self.host_amount_total == 0: + return 1 return int(math.ceil(float(self.host_amount_total) / 20)) @scrollable(width=160) From f591baadb8ded1b6b4bc371b07dbbcadbcf44696 Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 24 Jan 2018 19:56:36 -0300 Subject: [PATCH 0850/1506] Add HTTP headers to prevent clickjacking --- server/web.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/web.py b/server/web.py index 55a2f073bb2..a4f3c19fae7 100644 --- a/server/web.py +++ b/server/web.py @@ -44,10 +44,17 @@ def render(self, request): class FileWithoutDirectoryListing(File, CleanHttpHeadersResource): - def directoryListing(self): return ForbiddenResource() + def render(self, request): + ret = super(FileWithoutDirectoryListing, self).render(request) + if self.type == 'text/html': + request.responseHeaders.addRawHeader('Content-Security-Policy', + 'frame-ancestors \'none\'') + request.responseHeaders.addRawHeader('X-Frame-Options', 'DENY') + return ret + class FaradayWSGIResource(WSGIResource, object): def render(self, request): From 7cfdcf25fe757136a3d6aa0f0621243d0a1ec208 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 25 Jan 2018 14:52:11 -0300 Subject: [PATCH 0851/1506] Fix creator for Web UI. add unit tests --- server/api/modules/vulns.py | 13 ++++- test_cases/test_api_vulnerability.py | 80 +++++++++++++++++++--------- 2 files changed, 66 insertions(+), 27 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 92d8e6031b8..4bfba05fd6d 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -26,6 +26,7 @@ from server.fields import FaradayUploadedFile from server.models import ( db, + Command, CommandObject, File, Host, @@ -81,7 +82,17 @@ class CustomMetadataSchema(MetadataSchema): Implements command_id and creator logic """ command_id = fields.Integer(dump_only=True, attribute='creator_command_id') - creator = fields.String(dump_only=True, attribute='creator_command_tool') + creator = fields.Method('get_creator', dump_only=True) + + def get_creator(self, obj): + + if obj: + command = db.session.query(Command, CommandObject.command_id).join(CommandObject).filter_by( + object_type='vulnerability', + object_id=obj.id).first() + if command: + return command[0].tool + return 'Web UI' class VulnerabilitySchema(AutoSchema): diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index f1caf585ab7..e4cb2631aa3 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -152,35 +152,45 @@ def _create_post_data_vulnerability(self, name, vuln_type, parent_id, parent_type, refs, policyviolations, status='opened', attachments=None, impact=None, description='desc1234', - confirmed=True, data='data1234', easeofresolution=Vulnerability.EASE_OF_RESOLUTIONS[0], - owned=False, resolution='res1234', severity='critical'): + confirmed=True, data='data1234', + easeofresolution=Vulnerability.EASE_OF_RESOLUTIONS[0], + owned=False, resolution='res1234', severity='critical', + update_controller_action='UI Web' + ): if not impact: impact = {'accountability': False, 'availability': False, 'confidentiality': False, 'integrity': False} - data = {'metadata': {'update_time': 1508254070.211, 'update_user': '', 'update_action': 0, 'creator': 'UI Web', - 'create_time': 1508254070.211, 'update_controller_action': 'UI Web New', 'owner': ''}, - 'obj_id': '5a60af7f01dde6d3acfa8e9d3bef265c361a49d2', - 'owner': '', - 'parent': parent_id, - 'parent_type': parent_type, - 'type': vuln_type, - 'ws': 'airbnb', - 'confirmed': confirmed, - 'data': data, - 'desc': description, - 'easeofresolution': easeofresolution, - 'impact': impact, - 'name': name, - 'owned': owned, - 'policyviolations': policyviolations, - 'refs': refs, - 'resolution': resolution, - 'severity': severity, - 'status': status, - '_attachments': {}, - 'description': '', - 'protocol': '', - 'version': ''} + data = { + 'metadata': { + 'update_time': 1508254070.211, + 'update_user': '', + 'update_action': 0, + 'creator': 'UI Web', + 'create_time': 1508254070.211, + 'update_controller_action': update_controller_action, + 'owner': ''}, + 'obj_id': '5a60af7f01dde6d3acfa8e9d3bef265c361a49d2', + 'owner': '', + 'parent': parent_id, + 'parent_type': parent_type, + 'type': vuln_type, + 'ws': 'airbnb', + 'confirmed': confirmed, + 'data': data, + 'desc': description, + 'easeofresolution': easeofresolution, + 'impact': impact, + 'name': name, + 'owned': owned, + 'policyviolations': policyviolations, + 'refs': refs, + 'resolution': resolution, + 'severity': severity, + 'status': status, + '_attachments': {}, + 'description': '', + 'protocol': '', + 'version': ''} if vuln_type == 'VulnerabilityWeb': data.update({ @@ -1132,6 +1142,7 @@ def test_create_vuln_from_command(self, test_client, session): cmd_obj = command.command_objects[0] assert cmd_obj.object_type == 'vulnerability' assert cmd_obj.object_id == res.json['_id'] + assert res.json['metadata']['creator'] == command.tool def test_with_invalid_id_returns_400(self, session, test_client): """ @@ -1151,6 +1162,23 @@ def test_with_invalid_id_returns_400(self, session, test_client): res = test_client.post(self.url(), data=raw_data) assert res.status_code == 400 + def test_vuln_created_without_command_has_webui_in_metadata(self, test_client, session): + host = HostFactory.create(workspace=self.workspace) + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='Vulnerability', + parent_id=host.id, + parent_type='Host', + refs=[], + policyviolations=[], + description='helloworld', + severity='low', + ) + res = test_client.post(self.url(), data=raw_data) + assert res.status_code == 201 + assert res.json['metadata']['creator'] == 'Web UI' + def test_type_filter(workspace, session, vulnerability_factory, From 9f26c5848511935fd62c2d3b1a8ab89e447963fc Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 25 Jan 2018 17:14:16 -0300 Subject: [PATCH 0852/1506] Fix fplugin bug --- bin/fplugin | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/bin/fplugin b/bin/fplugin index b84c1d189b5..7c37661b25d 100755 --- a/bin/fplugin +++ b/bin/fplugin @@ -7,15 +7,16 @@ Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) See the file 'doc/LICENSE' for the license information """ -import argparse -import imp -import inspect import os -import shlex import sys +import imp +import shlex +import atexit import signal +import inspect +import argparse import readline -import atexit +from Queue import Queue parent_path = os.path.abspath(os.path.join(__file__, '../..')) sys.path.insert(0, parent_path) @@ -53,7 +54,8 @@ def dispatch(args, unknown, help): # We need the ModelController to register all available models mappers_manager = MapperManager() - model_controller = ModelController(mappers_manager) + pending_actions = Queue() + model_controller = ModelController(mappers_manager, pending_actions) if not args.command: print help From e5d0537f7f907c0a5e4dc0ca513bc3617dd29728 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 26 Jan 2018 17:23:12 -0300 Subject: [PATCH 0853/1506] Fix attachments * Fix filename for attachments * Add Download attachment view --- server/api/modules/vulns.py | 30 +++++++++++++--- server/app.py | 2 ++ .../ui-grid/columns/evidencecolumn.html | 2 +- test_cases/test_api_vulnerability.py | 34 +++++++++++++++++++ 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 92d8e6031b8..cff0f5c500f 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -2,16 +2,17 @@ # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information import os -import time +import io import logging from base64 import b64encode, b64decode +import flask from filteralchemy import Filter, FilterSet, operators -from flask import request, current_app +from flask import request from flask import Blueprint +from flask_classful import route from marshmallow import Schema, fields, post_load, ValidationError from marshmallow.validate import OneOf -from sqlalchemy import and_ from sqlalchemy.orm import joinedload, selectin_polymorphic, undefer from sqlalchemy.orm.exc import NoResultFound @@ -137,13 +138,13 @@ def get_type(self, obj): return obj.__class__.__name__ def get_attachments(self, obj): - res = [] + res = {} for file_obj in obj.evidence: ret, errors = EvidenceSchema().dump(file_obj) if errors: raise ValidationError(errors, data=ret) - res.append(ret) + res[file_obj.filename] = ret return res @@ -453,5 +454,24 @@ def convert_group(group): res['groups'] = [convert_group(group) for group in res['groups']] return res + @route('//attachment//') + def attachment(self, workspace_name, vuln_id, attachment_filename): + vuln_workspace_check = db.session.query(VulnerabilityGeneric, Workspace.id).join( + Workspace).filter(VulnerabilityGeneric.id == vuln_id, + Workspace.name == workspace_name).first() + if vuln_workspace_check: + file_obj = db.session.query(File).filter_by(object_type='vulnerability', + object_id=vuln_id, + filename=attachment_filename).first() + if file_obj: + depot = DepotManager.get() + depot_file = depot.get(file_obj.content.get('file_id')) + return flask.send_file( + io.BytesIO(depot_file.read()), + attachment_filename=file_obj.filename, + as_attachment=True, + mimetype=depot_file.content_type + ) + VulnerabilityView.register(vulns_api) diff --git a/server/app.py b/server/app.py index 535f470aea5..283bc28ec40 100644 --- a/server/app.py +++ b/server/app.py @@ -120,6 +120,7 @@ def log_queries_count(response): # map(str,Counter(q.statement for q in queries).most_common())) return response + def save_new_secret_key(app): config = ConfigParser() config.read(LOCAL_CONFIG_FILE) @@ -130,6 +131,7 @@ def save_new_secret_key(app): with open(LOCAL_CONFIG_FILE, 'w') as configfile: config.write(configfile) + def create_app(db_connection_string=None, testing=None): app = Flask(__name__) diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/evidencecolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/evidencecolumn.html index 7d6259c502f..4eae69b43a7 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/evidencecolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/evidencecolumn.html @@ -1,4 +1,4 @@
    -

    {{key | decodeURIComponent}}

    +

    {{key | decodeURIComponent}}

    {{COL_FIELD CUSTOM_FILTERS}}
    \ No newline at end of file diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index f1caf585ab7..9f540fd44ef 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1,4 +1,7 @@ #-*- coding: utf8 -*- +from StringIO import StringIO +from tempfile import NamedTemporaryFile + import os from base64 import b64encode try: @@ -235,6 +238,37 @@ def test_create_vuln(self, host_with_hostnames, test_client, session): assert res.json['description'] == 'helloworld' assert res.json['severity'] == 'low' + def test_create_vuln_with_attachments(self, host_with_hostnames, test_client, session): + session.commit() # flush host_with_hostnames + attachment = NamedTemporaryFile() + file_content = 'test file' + attachment.write(file_content) + attachment.seek(0) + raw_data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='Vulnerability', + parent_id=host_with_hostnames.id, + parent_type='Host', + refs=[], + policyviolations=[], + description='helloworld', + severity='low', + attachments=[attachment] + ) + + res = test_client.post(self.url(), data=raw_data) + vuln_id = res.json['_id'] + assert res.status_code == 201 + filename = attachment.name.split('/')[-1] + assert filename in res.json['_attachments'] + attachment.close() + # check the attachment can be downloaded + res = test_client.get(self.url() + '{0}/attachment/{1}/'.format(vuln_id, filename)) + assert res.status_code == 200 + assert res.data == file_content + + + def test_create_vuln_props(self, host_with_hostnames, test_client, session): """ This one should check all the vuln props that don't have a specific case From 8dbb588bbc19f2d74a8481c5cd212cf51275ec16 Mon Sep 17 00:00:00 2001 From: Winna_Z Date: Fri, 26 Jan 2018 17:34:07 -0300 Subject: [PATCH 0854/1506] changed the assigned severities --- plugins/repo/qualysguard/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/repo/qualysguard/plugin.py b/plugins/repo/qualysguard/plugin.py index 091e8c389d0..4838135237b 100644 --- a/plugins/repo/qualysguard/plugin.py +++ b/plugins/repo/qualysguard/plugin.py @@ -175,7 +175,7 @@ def __init__(self, issue_node, glossary): self.severity_dict = { '1': 'info', - '2': 'low', + '2': 'info', '3': 'med', '4': 'high', '5': 'critical'} From 8fc2ee03858a2e3c0e22511c169f0832842cac0f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 29 Jan 2018 17:25:12 -0300 Subject: [PATCH 0855/1506] Fix vulnerability attach on edit (put) --- server/api/base.py | 9 ++-- server/api/modules/vulns.py | 19 ++++++++- test_cases/test_api_vulnerability.py | 62 ++++++++++++++++++++++++++-- 3 files changed, 81 insertions(+), 9 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index fc077dc5215..79dd8038094 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -524,7 +524,7 @@ def put(self, object_id, **kwargs): # just in case an schema allows id as writable. data.pop('id', None) self._update_object(obj, data) - self._perform_update(object_id, obj, **kwargs) + self._perform_update(object_id, obj, data, **kwargs) return self._dump(obj, kwargs), 200 @@ -532,17 +532,18 @@ def _update_object(self, obj, data): for (key, value) in data.items(): setattr(obj, key, value) - def _perform_update(self, object_id, obj): + def _perform_update(self, object_id, obj, workspace_name): with db.session.no_autoflush: self._validate_uniqueness(obj, object_id) db.session.add(obj) db.session.commit() + return obj class UpdateWorkspacedMixin(UpdateMixin, CommandMixin): """Add PUT // route""" - def _perform_update(self, object_id, obj, workspace_name): + def _perform_update(self, object_id, obj, data, workspace_name): # # Make sure that if I created new objects, I had properly commited them # assert not db.session.new @@ -551,7 +552,7 @@ def _perform_update(self, object_id, obj, workspace_name): self._set_command_id(obj, False) return super(UpdateWorkspacedMixin, self)._perform_update( - object_id, obj) + object_id, obj, data) class DeleteMixin(object): diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index aef39b898f2..44db6271299 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -357,16 +357,25 @@ def _perform_create(self, data, **kwargs): request) # TODO migration: use default values when popping and validate the # popped object has the expected type. - attachments = data.pop('_attachments', {}) - # This will be set after setting the workspace + attachments = data.pop('_attachments', {}) references = data.pop('references') policyviolations = data.pop('policy_violations') obj = super(VulnerabilityView, self)._perform_create(data, **kwargs) obj.references = references obj.policy_violations = policyviolations + db.session.commit() + self.process_attachments(obj, attachments) + return obj + def process_attachments(self, obj, attachments): + old_attachments = db.session.query(File).filter_by( + object_id=obj.id, + object_type='vulnerability', + ) + for old_attachment in old_attachments: + db.session.delete(old_attachment) for filename, attachment in attachments.items(): faraday_file = FaradayUploadedFile(b64decode(attachment['data'])) get_or_create( @@ -378,6 +387,12 @@ def _perform_create(self, data, **kwargs): filename=os.path.basename(filename), content=faraday_file, ) + + def _perform_update(self, object_id, obj, data, workspace_name): + attachments = data.pop('_attachments', {}) + obj = super(VulnerabilityView, self)._perform_update(object_id, obj, data, workspace_name) + db.session.flush() + self.process_attachments(obj, attachments) db.session.commit() return obj diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 02b8ccf244b..c70d894e1c7 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -4,6 +4,9 @@ import os from base64 import b64encode + +from test_cases.conftest import ignore_nplusone + try: from urllib import urlencode except ImportError: @@ -26,8 +29,9 @@ Vulnerability, VulnerabilityWeb, Reference, PolicyViolation, CommandObject) -from test_cases.factories import ServiceFactory, CommandFactory, CommandObjectFactory, HostFactory, EmptyCommandFactory, \ - UserFactory, VulnerabilityWebFactory +from test_cases.factories import ServiceFactory, CommandFactory, \ + CommandObjectFactory, HostFactory, EmptyCommandFactory, \ + UserFactory, VulnerabilityWebFactory, VulnerabilityFactory CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) @@ -277,6 +281,49 @@ def test_create_vuln_with_attachments(self, host_with_hostnames, test_client, se assert res.status_code == 200 assert res.data == file_content + @ignore_nplusone + def test_update_vuln_add_attachment_on_update(self, test_client, session): + host = HostFactory.create(workspace=self.workspace) + vuln = VulnerabilityFactory.create(workspace=self.workspace, host_id=host.id) + session.commit() # flush host_with_hostnames + attachment = NamedTemporaryFile() + file_content = 'test file' + attachment.write(file_content) + attachment.seek(0) + raw_data = self._create_put_data( + 'Updated with attachment', + 'Updated vuln', + 'open', + host.id, + 'Host', + attachments=[attachment] + ) + res = test_client.put(self.url(obj=vuln, workspace=self.workspace), data=raw_data) + assert res.status_code == 200 + filename = attachment.name.split('/')[-1] + res = test_client.get( + self.url() + '{0}/attachment/{1}/'.format(vuln.id, filename)) + assert res.status_code == 200 + assert res.data == file_content + + new_attachment = NamedTemporaryFile() + new_filename = new_attachment.name.split('/')[-1] + file_content = 'new test file' + new_attachment.write(file_content) + new_attachment.seek(0) + res = test_client.put(self.url(obj=vuln, workspace=self.workspace), + data=raw_data) + assert res.status_code == 200 + + # verify that the old file was deleted and the new one exists + res = test_client.get( + self.url() + '{0}/attachment/{1}/'.format(vuln.id, filename)) + assert res.status_code == 404 + res = test_client.get( + self.url() + '{0}/attachment/{1}/'.format(vuln.id, new_filename)) + assert res.status_code == 200 + assert res.data == file_content + def test_create_vuln_props(self, host_with_hostnames, test_client, session): @@ -391,7 +438,8 @@ def test_create_vuln_with_closed_status(self, host_with_hostnames, test_client, def _create_put_data(self, name, desc, status, parent, parent_type, - impact=None, refs=None, policy_violations=None): + attachments=None, impact=None, refs=None, + policy_violations=None): if not refs: refs = [] if not policy_violations: @@ -434,6 +482,14 @@ def _create_put_data(self, "protocol":"", "version":""} + if attachments: + raw_data['_attachments'] = {} + for attachment in attachments: + raw_data['_attachments'][attachment.name] = { + "content_type": "application/x-shellscript", + "data": b64encode(attachment.read()) + } + return raw_data def test_update_vuln_from_open_to_close(self, test_client, session, host_with_hostnames): From 2ebe1c21f362270462069f021497e2855231e802 Mon Sep 17 00:00:00 2001 From: Winna_Z Date: Wed, 31 Jan 2018 16:30:00 -0300 Subject: [PATCH 0856/1506] typo wordspace --> workspace --- plugins/repo/metasploiton/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/repo/metasploiton/plugin.py b/plugins/repo/metasploiton/plugin.py index e97cdbab6a2..4f92ac2bd6d 100644 --- a/plugins/repo/metasploiton/plugin.py +++ b/plugins/repo/metasploiton/plugin.py @@ -60,7 +60,7 @@ def __init__(self): self.addSetting("Server", str, "localhost") self.addSetting("Port", str, "5432") ### NOTE: do _not_ correct the typo, it's used by the user.xml - self.addSetting("Wordspace", str, "a_workspace") + self.addSetting("Workspace", str, "a_workspace") self.addSetting("Enable", str, "0") self._sdate = "" @@ -98,7 +98,7 @@ def parseOutputString(self, output, debug=False): cur = self._doSql( cur, - "select * from hosts inner join workspaces ON (hosts.workspace_id=workspaces.id) where workspaces.name like '" + self.getSetting("Wordspace") + "';") + "select * from hosts inner join workspaces ON (hosts.workspace_id=workspaces.id) where workspaces.name like '" + self.getSetting("Workspace") + "';") if cur is None: print "Error getting database data\n" return From 4a059c8c18e361a17687d599f09dc3f46ad96b87 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 31 Jan 2018 17:36:02 -0300 Subject: [PATCH 0857/1506] Refactor to fix race condition We detected that when the server was running with multiple threads or processes a race condition could be triggered when we first select object from the database. With these changes we try to insert data first and if the unique constraint failed then we use unique fields + data to select the conflicting object from the database. With this we return the object which will be later used for conflict resolution --- server/api/base.py | 38 +++++++++++---- server/models.py | 12 +---- server/utils/database.py | 72 +++++++++++++++++++++++++++- test_cases/test_api_services.py | 2 +- test_cases/test_api_vulnerability.py | 5 +- test_cases/test_utils_database.py | 36 ++++++++++++++ 6 files changed, 142 insertions(+), 23 deletions(-) create mode 100644 test_cases/test_utils_database.py diff --git a/server/api/base.py b/server/api/base.py index 79dd8038094..1b221bcfb59 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -1,6 +1,7 @@ import json import flask +import sqlalchemy from flask import abort, g from flask_classful import FlaskView from sqlalchemy.orm import joinedload, undefer @@ -15,6 +16,7 @@ from webargs.core import ValidationError from server.models import Workspace, db, Command, CommandObject import server.utils.logger +from server.utils.database import get_unique_fields, get_conflict_object logger = server.utils.logger.get_logger(__name__) @@ -443,11 +445,20 @@ def post(self, **kwargs): def _perform_create(self, data, **kwargs): obj = self.model_class(**data) # assert not db.session.new - with db.session.no_autoflush: - # Required because _validate_uniqueness does a select. Doing this - # outside a no_autoflush block would result in a premature create. - self._validate_uniqueness(obj) + try: db.session.add(obj) + db.session.commit() + except sqlalchemy.exc.IntegrityError as ex: + db.session.rollback() + conflict_obj = get_conflict_object(db.session, obj, data) + if conflict_obj: + abort(409, ValidationError( + { + 'message': 'Existing value', + 'object': self._get_schema_class()().dump( + conflict_obj).data, + } + )) return obj @@ -503,12 +514,21 @@ def _perform_create(self, data, workspace_name): obj = self.model_class(**data) obj.workspace = workspace # assert not db.session.new - with db.session.no_autoflush: - # Required because _validate_uniqueness does a select. Doing this - # outside a no_autoflush block would result in a premature create. - self._validate_uniqueness(obj) + try: db.session.add(obj) - db.session.commit() + db.session.commit() + except sqlalchemy.exc.IntegrityError as ex: + db.session.rollback() + conflict_obj = get_conflict_object(db.session, obj, data) + if conflict_obj: + abort(409, ValidationError( + { + 'message': 'Existing value', + 'object': self._get_schema_class()().dump( + conflict_obj).data, + } + )) + self._set_command_id(obj, True) return obj diff --git a/server/models.py b/server/models.py index aa2fcf85cee..5c10f78de4b 100644 --- a/server/models.py +++ b/server/models.py @@ -43,7 +43,7 @@ RoleMixin, UserMixin, ) -from server.utils.database import BooleanToIntColumn +from server.utils.database import BooleanToIntColumn, get_object_type_for OBJECT_TYPES = [ 'vulnerability', @@ -567,15 +567,7 @@ def __init__(self, object_=None, **kwargs): if object_ is not None: assert 'object_type' not in kwargs assert 'object_id' not in kwargs - object_type = object_.__tablename__ - if object_type is None: - if object_.__class__.__name__ in ['Vulnerability', - 'VulnerabilityWeb', - 'VulnerabilityCode']: - object_type = 'vulnerability' - else: - raise RuntimeError("Unknown table for object: {}".format( - object_)) + object_type = get_object_type_for(object_) # db.session.flush() assert object_.id is not None, "object must have an ID. Try " \ diff --git a/server/utils/database.py b/server/utils/database.py index 47433cb1856..9ec6d4081bd 100644 --- a/server/utils/database.py +++ b/server/utils/database.py @@ -7,6 +7,7 @@ from sqlalchemy.sql.expression import ClauseElement from sqlalchemy.sql import expression from sqlalchemy.ext import compiler +from sqlalchemy.engine.reflection import Inspector class ORDER_DIRECTIONS: @@ -202,4 +203,73 @@ def _integer_to_boolean_postgresql(element, compiler, **kw): @compiler.compiles(BooleanToIntColumn, 'sqlite') def _integer_to_boolean_sqlite(element, compiler, **kw): - return element.expression_str \ No newline at end of file + return element.expression_str + + +def get_object_type_for(instance): + object_type = instance.__tablename__ + if object_type is None: + if instance.__class__.__name__ in ['Vulnerability', + 'VulnerabilityWeb', + 'VulnerabilityCode']: + object_type = 'vulnerability' + else: + raise RuntimeError("Unknown table for object: {}".format( + instance)) + return object_type + + +def get_unique_fields(session, instance): + table_name = get_object_type_for(instance) + if table_name != 'vulnerability': + engine = session.connection().engine + insp = Inspector.from_engine(engine) + unique_constraints = insp.get_unique_constraints(table_name) + else: + unique_constraints = [] + unique_constraints.append({ + 'column_names': [ + 'name', + 'description', + 'host_id', + 'service_id', + 'method', + 'parameter_name', + 'path', + 'website', + 'workspace_id', + ] + }) + if unique_constraints: + for unique_constraint in unique_constraints: + yield unique_constraint['column_names'] + + +def get_conflict_object(session, obj, data): + unique_fields_gen = get_unique_fields(session, obj) + for unique_fields in unique_fields_gen: + relations_fields = filter( + lambda unique_field: unique_field.endswith('_id'), + unique_fields) + unique_fields = filter( + lambda unique_field: not unique_field.endswith('_id'), + unique_fields) + filter_data = {unique_field: data[unique_field] for unique_field in + unique_fields if unique_field in data} + + if 'workspace_id' in relations_fields: + relations_fields.remove('workspace_id') + filter_data['workspace_id'] = obj.workspace.id + + for relations_field in relations_fields: + if relations_field not in data and relations_field.strip('_id') in data: + filter_data[relations_field] = data[ + relations_field.strip('_id')].id + else: + relation_id = data.get(relations_field, None) + if relation_id: + filter_data[relations_field] = relation_id + + conflict_obj = session.query(obj.__class__).filter_by( + **filter_data).first() + return conflict_obj \ No newline at end of file diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index f1199e27648..6174908e5fa 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -231,4 +231,4 @@ def test_create_service_without_ost(self, test_client, host, session): } res = test_client.post(self.url(), data=data) assert res.status_code == 400 - assert res.json['messages']['_schema'] == res.json['messages']['_schema'] \ No newline at end of file + assert res.json['messages']['_schema'] == res.json['messages']['_schema'] diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index c70d894e1c7..a189e0f301b 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1047,13 +1047,14 @@ def test_vulnerability_metadata(self, session, test_client, workspace): object_id=vuln.id, workspace=workspace ) - session.commit() + CommandObjectFactory.create( command=update_command, object_type='vulnerability', object_id=vuln.id, workspace=workspace ) + session.commit() res = test_client.get(self.url()) assert res.status_code == 200 @@ -1063,7 +1064,7 @@ def test_vulnerability_metadata(self, session, test_client, workspace): expected_metadata = { u'command_id': command.id, u'create_time': int(time.mktime(vuln.create_date.timetuple()) * 1000), - u'creator': command.tool, + u'creator': update_command.tool, u'owner': owner.username, u'update_action': 0, u'update_controller_action': u'', diff --git a/test_cases/test_utils_database.py b/test_cases/test_utils_database.py new file mode 100644 index 00000000000..a2b65b162d9 --- /dev/null +++ b/test_cases/test_utils_database.py @@ -0,0 +1,36 @@ +import pytest + +from server.utils.database import get_unique_fields +from server.models import ( + License, + Service, + Host, + Vulnerability, + Workspace, +) + +UNIQUE_FIELDS = { + License: [u'product', u'start_date', u'end_date'], + Service: [u'port', u'protocol', u'host_id', u'workspace_id'], + Host: [u'ip', u'workspace_id'], + Vulnerability: [ + 'name', + 'description', + 'host_id', + 'service_id', + 'method', + 'parameter_name', + 'path', + 'website', + 'workspace_id', + ], + Workspace: ['name'] +} + + +@pytest.mark.parametrize("obj_class, expected_unique_fields", UNIQUE_FIELDS.items()) +def test_unique_fields_workspace(obj_class, expected_unique_fields, session): + workspace = obj_class() + unique_constraints = get_unique_fields(session, workspace) + for unique_constraint in unique_constraints: + assert unique_constraint == expected_unique_fields \ No newline at end of file From 4b674c12cdda8adb187adb19b5726940c4bc3386 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 31 Jan 2018 17:40:50 -0300 Subject: [PATCH 0858/1506] Fix race condition for update --- server/api/base.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 1b221bcfb59..c1532c7003e 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -552,11 +552,21 @@ def _update_object(self, obj, data): for (key, value) in data.items(): setattr(obj, key, value) - def _perform_update(self, object_id, obj, workspace_name): - with db.session.no_autoflush: - self._validate_uniqueness(obj, object_id) - db.session.add(obj) - db.session.commit() + def _perform_update(self, object_id, obj, data, workspace_name=None): + try: + db.session.add(obj) + db.session.commit() + except sqlalchemy.exc.IntegrityError as ex: + db.session.rollback() + conflict_obj = get_conflict_object(db.session, obj, data) + if conflict_obj: + abort(409, ValidationError( + { + 'message': 'Existing value', + 'object': self._get_schema_class()().dump( + conflict_obj).data, + } + )) return obj @@ -572,7 +582,7 @@ def _perform_update(self, object_id, obj, data, workspace_name): self._set_command_id(obj, False) return super(UpdateWorkspacedMixin, self)._perform_update( - object_id, obj, data) + object_id, obj, data, workspace_name) class DeleteMixin(object): From c0dfb999a665eb83c56e27792e68313ca1a203d8 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 31 Jan 2018 17:48:10 -0300 Subject: [PATCH 0859/1506] Clean up --- server/api/base.py | 46 +------------------ server/api/modules/hosts.py | 1 - server/api/modules/licenses.py | 1 - server/api/modules/services.py | 1 - server/api/modules/vulnerability_template.py | 1 - test_cases/test_api_services.py | 2 - test_cases/test_api_vulnerability_template.py | 2 - 7 files changed, 1 insertion(+), 53 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index c1532c7003e..de080430383 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -16,7 +16,7 @@ from webargs.core import ValidationError from server.models import Workspace, db, Command, CommandObject import server.utils.logger -from server.utils.database import get_unique_fields, get_conflict_object +from server.utils.database import get_conflict_object logger = server.utils.logger.get_logger(__name__) @@ -66,7 +66,6 @@ class GenericView(FlaskView): } lookup_field = 'id' lookup_field_type = int - unique_fields = [] # Fields unique # Attributes to improve the performance of list and retrieve views get_joinedloads = [] # List of relationships to eagerload @@ -152,43 +151,6 @@ def _parse_data(self, schema, request, *args, **kwargs): return FlaskParser().parse(schema, request, locations=('json',), *args, **kwargs) - def _validate_uniqueness(self, obj, object_id=None, unique_fields=None): - primary_key_field = inspect(self.model_class).primary_key[0] - - if getattr(obj, 'workspace', None): - query = self._get_base_query(obj.workspace.name) - else: - query = self._get_base_query() - - if object_id is not None: - # The object already exists in DB, we want to fetch an object - # different to this one but with the same unique field - query = query.filter(primary_key_field != object_id) - if unique_fields is None: - # It is usually None, but in some case the child class may want to - # override it based on some condition dependent on the request - unique_fields = self.unique_fields - for field_names in unique_fields: - for field_name in field_names: - # Use type(obj) instead of self.model_class to be - # compatible with polymorphic obejcts (e.g. Vulnerability) - field = getattr(type(obj), field_name) - value = getattr(obj, field_name) - query = query.filter( - field == value) - - existing_obj = query.one_or_none() - conflict_data = self._get_schema_class()().dump(existing_obj).data - if existing_obj: - db.session.rollback() - abort(409, ValidationError( - { - 'message': 'Existing value for unique columns: %s' % ( - field_names,), - 'object': conflict_data, - } - )) - @classmethod def register(cls, app, *args, **kwargs): """Register and add JSON error handler. Use error code @@ -233,7 +195,6 @@ class GenericWorkspacedView(GenericView): # Default attributes route_prefix = '/v2/ws//' base_args = ['workspace_name'] # Required to prevent double usage of - unique_fields = [] # Fields unique together with workspace_id def _get_workspace(self, workspace_name): try: @@ -264,11 +225,6 @@ def _set_schema_context(self, context, **kwargs): context.update(kwargs) return context - def _validate_uniqueness(self, obj, object_id=None, unique_fields=None): - assert obj.workspace is not None, "Object must have a " \ - "workspace attribute set to call _validate_uniqueness" - return super(GenericWorkspacedView, self)._validate_uniqueness(obj, object_id, unique_fields) - class ListMixin(object): """Add GET / route""" diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index dd550b17821..c621ff1a7bf 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -91,7 +91,6 @@ class HostsView(PaginatedMixin, model_class = Host order_field = Host.ip.asc() schema_class = HostSchema - unique_fields = [('ip', )] filterset_class = HostFilterSet get_undefer = [Host.credentials_count, Host.open_service_count, diff --git a/server/api/modules/licenses.py b/server/api/modules/licenses.py index d3bd4f7499c..733e8f204b3 100644 --- a/server/api/modules/licenses.py +++ b/server/api/modules/licenses.py @@ -29,6 +29,5 @@ class LicenseView(ReadWriteView): route_base = 'licenses' model_class = License schema_class = LicenseSchema - unique_fields = [] LicenseView.register(license_api) diff --git a/server/api/modules/services.py b/server/api/modules/services.py index bbe2655c46f..6eb014d5e45 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -86,7 +86,6 @@ class ServiceView(ReadWriteWorkspacedView): schema_class = ServiceSchema count_extra_filters = [Service.status == 'open'] get_undefer = [Service.credentials_count, Service.vulnerability_count] - unique_fields = [('port', 'protocol', 'host')] def _envelope_list(self, objects, pagination_metadata=None): services = [] diff --git a/server/api/modules/vulnerability_template.py b/server/api/modules/vulnerability_template.py index 58d45ebf8db..f5f7583ac66 100644 --- a/server/api/modules/vulnerability_template.py +++ b/server/api/modules/vulnerability_template.py @@ -72,7 +72,6 @@ class Meta(FilterSetMeta): class VulnerabilityTemplateView(PaginatedMixin, FilterAlchemyMixin, ReadWriteView): - unique_fields = [('name',)] route_base = 'vulnerability_template' model_class = VulnerabilityTemplate schema_class = VulnerabilityTemplateSchema diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index 6174908e5fa..5ce8f2d5f32 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -22,8 +22,6 @@ class TestListServiceView(ReadOnlyAPITests): model = Service factory = factories.ServiceFactory api_endpoint = 'services' - #unique_fields = ['ip'] - #update_fields = ['ip', 'description', 'os'] view_class = ServiceView def test_service_list_backwards_compatibility(self, test_client, diff --git a/test_cases/test_api_vulnerability_template.py b/test_cases/test_api_vulnerability_template.py index 16d7d6a27ba..cd73d704e1a 100644 --- a/test_cases/test_api_vulnerability_template.py +++ b/test_cases/test_api_vulnerability_template.py @@ -21,8 +21,6 @@ class TestListVulnerabilityTemplateView(ReadOnlyAPITests): model = VulnerabilityTemplate factory = factories.VulnerabilityTemplateFactory api_endpoint = 'vulnerability_template' - #unique_fields = ['ip'] - #update_fields = ['ip', 'description', 'os'] view_class = VulnerabilityTemplateView def test_backwards_json_compatibility(self, test_client, session): From f551cbd9c742de25241f9ef7852db15dd34487da Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 1 Feb 2018 12:26:06 -0300 Subject: [PATCH 0860/1506] add unit test to verify unique fields when ddl changed --- server/utils/database.py | 3 +++ test_cases/test_utils_database.py | 26 ++++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/server/utils/database.py b/server/utils/database.py index 9ec6d4081bd..11be686e7d7 100644 --- a/server/utils/database.py +++ b/server/utils/database.py @@ -226,6 +226,9 @@ def get_unique_fields(session, instance): insp = Inspector.from_engine(engine) unique_constraints = insp.get_unique_constraints(table_name) else: + # Vulnerability unique index can't be retrieved via reflection. + # If the unique index changes we need to update here. + # A test should fail when the unique index changes unique_constraints = [] unique_constraints.append({ 'column_names': [ diff --git a/test_cases/test_utils_database.py b/test_cases/test_utils_database.py index a2b65b162d9..46ae7e6e685 100644 --- a/test_cases/test_utils_database.py +++ b/test_cases/test_utils_database.py @@ -7,6 +7,7 @@ Host, Vulnerability, Workspace, + vulnerability_uniqueness ) UNIQUE_FIELDS = { @@ -28,9 +29,30 @@ } +def test_vulnerability_ddl_invariant(session): + """ + This test is to make sure that get_unique_fields for Vulnerability is + returning the same columns as in the DDL + :return: + """ + statement = vulnerability_uniqueness.statement + column_part = statement.split('%(fullname)s')[1] + statements_clean = column_part.strip().strip(')').strip('(').split(',') + statements_clean = map(lambda column: column.replace('COALESCE(', '').replace('md5(', '').strip('('), statements_clean) + statements_clean = filter(len, map(lambda column: column.strip("'')").strip('-1').strip('-1));').strip(), statements_clean)) + statements_clean.remove('source_code_id') # we don't support source_code yet + unique_constraints = get_unique_fields(session, Vulnerability()) + for unique_constraint in unique_constraints: + assert len(statements_clean) == len(unique_constraint) + for statement_clean in statements_clean: + statement_clean = statement_clean + if statement_clean not in unique_constraint: + raise Exception('Please check server.data.utils.get_unique_fields. Vulnerability DDL changed?') + + @pytest.mark.parametrize("obj_class, expected_unique_fields", UNIQUE_FIELDS.items()) def test_unique_fields_workspace(obj_class, expected_unique_fields, session): - workspace = obj_class() - unique_constraints = get_unique_fields(session, workspace) + object_ = obj_class() + unique_constraints = get_unique_fields(session, object_) for unique_constraint in unique_constraints: assert unique_constraint == expected_unique_fields \ No newline at end of file From fb203efe9832c1b27a4b699db145bcd8f82b95b7 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 1 Feb 2018 17:35:37 -0300 Subject: [PATCH 0861/1506] Check database connection when starting the server --- faraday-server.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/faraday-server.py b/faraday-server.py index 74cf000096c..066ab67efdc 100755 --- a/faraday-server.py +++ b/faraday-server.py @@ -7,10 +7,14 @@ import argparse import subprocess +import sqlalchemy + import server.config import server.couchdb import server.utils.logger +from server.models import db, Workspace from server.utils import daemonize +from server.web import app from utils import dependencies from utils.user_input import query_yes_no from faraday import FARADAY_BASE @@ -79,7 +83,19 @@ def run_server(args): logger.info('Faraday Server is ready') +def check_postgresql(): + with app.app_context(): + try: + if not db.session.query(Workspace).count(): + logger.warn('No workspaces found. Remeber to execute couchdb importer') + except sqlalchemy.exc.OperationalError: + logger.error( + 'Could not connect to postgresql, please check database is running or configuration settings are correct.') + sys.exit(1) + + def main(): + check_postgresql() os.chdir(FARADAY_BASE) parser = argparse.ArgumentParser() parser.add_argument('--ssl', action='store_true', help='enable HTTPS') From ca422ee803d3f3e75e5d6a1f881c5996fcde08b9 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 1 Feb 2018 19:14:33 -0300 Subject: [PATCH 0862/1506] Avoid changing password when importing couchdb users --- server/commands/initdb.py | 1 + server/importer.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index 57be66fd560..ab39704146f 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -140,6 +140,7 @@ def _create_admin_user(self, conn_string): " {red}password{white}: {" "random_password}".format(random_password=random_password, white=Fore.WHITE, red=Fore.RED)) + print("{yellow} WARNING {white} If you are going to execute couchdb importer you must use the couddb password for faraday user.") def _configure_existing_postgres_user(self): diff --git a/server/importer.py b/server/importer.py index 2d73d2e20bc..094209bfd0b 100644 --- a/server/importer.py +++ b/server/importer.py @@ -971,13 +971,16 @@ def import_admins(self, admins): # Import admin users for (username, password) in admins.items(): logger.info('Creating user {0}'.format(username)) - if not db.session.query(User).filter_by(username=username).first(): + admin = db.session.query(User).filter_by(username=username).first() + if not admin: app.user_datastore.create_user( username=username, email=username + '@test.com', password=self.convert_couchdb_hash(password), is_ldap=False ) + else: + admin.password=self.convert_couchdb_hash(password) def import_users(self, all_users, admins): # Import non admin users @@ -990,13 +993,16 @@ def import_users(self, all_users, admins): # This is an already imported admin user, skip continue logger.info(u'Importing user {0}'.format(user['name'])) - if not db.session.query(User).filter_by(username=user['name']).first(): + old_user = db.session.query(User).filter_by(username=user['name']).first() + if not old_user: app.user_datastore.create_user( username=user['name'], email=user['name'] + '@test.com', password=self.get_hash_from_document(user), is_ldap=False ) + else: + old_user.password = self.get_hash_from_document(user) def run(self): all_users, admins = self.get_users_and_admins() From d4500953bd70d46a277999a22f6cf31e1645a2ed Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 2 Feb 2018 14:27:21 -0300 Subject: [PATCH 0863/1506] Add notes to license api --- server/api/modules/licenses.py | 4 +++- test_cases/test_api_license.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/server/api/modules/licenses.py b/server/api/modules/licenses.py index d3bd4f7499c..9ff0b5fad8a 100644 --- a/server/api/modules/licenses.py +++ b/server/api/modules/licenses.py @@ -22,7 +22,9 @@ class LicenseSchema(AutoSchema): start = fields.DateTime(attribute='start_date') class Meta: model = License - fields = ('_id', 'id', 'product', 'start', 'end', 'lictype') + fields = ('_id', 'id', 'product', + 'start', 'end', 'lictype', + 'notes') class LicenseView(ReadWriteView): diff --git a/test_cases/test_api_license.py b/test_cases/test_api_license.py index 1e111d315ff..15caef520b1 100644 --- a/test_cases/test_api_license.py +++ b/test_cases/test_api_license.py @@ -8,6 +8,8 @@ License, ) from server.api.modules.licenses import LicenseView +from test_cases.factories import LicenseFactory + class LicenseEnvelopedView(LicenseView): """A custom view to test that enveloping on generic views work ok""" @@ -32,3 +34,11 @@ def test_envelope_list(self, test_client, app): assert new_res.status_code == 200 assert new_res.json == {"object_list": original_res.json} + + def test_license_note_was_missing(self, test_client, session): + notes = 'A great note. License' + lic = LicenseFactory.create(notes=notes) + session.commit() + res = test_client.get(self.url(obj=lic)) + assert res.status_code == 200 + assert res.json['notes'] == 'A great note. License' \ No newline at end of file From 55ecfc3f80ce4c29b74e64f75fb119b50819527e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 2 Feb 2018 14:27:33 -0300 Subject: [PATCH 0864/1506] Improve warning message --- faraday-server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/faraday-server.py b/faraday-server.py index 066ab67efdc..ea7051d23f4 100755 --- a/faraday-server.py +++ b/faraday-server.py @@ -90,7 +90,7 @@ def check_postgresql(): logger.warn('No workspaces found. Remeber to execute couchdb importer') except sqlalchemy.exc.OperationalError: logger.error( - 'Could not connect to postgresql, please check database is running or configuration settings are correct.') + 'Could not connect to postgresql, please check if database is running or configuration settings are correct.') sys.exit(1) From cb985042b2ee9128966f1cc31cad8ac0c025e04e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 5 Feb 2018 16:37:56 -0300 Subject: [PATCH 0865/1506] Fix bug remembering status report columns The ID field was always showed when refreshing the page even if the user disabled it. I had to change the visible name from "id" to "_id". This is horrible. --- server/www/scripts/statusReport/controllers/statusReport.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 53b0d6669c6..172215019ce 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -235,7 +235,7 @@ angular.module('faradayApp') ' '; $scope.gridOptions.columnDefs.push({ name : '_id', - displayName : "id", + displayName : "_id", cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/idcolumn.html', headerCellTemplate: header, width: '50', From fb6527d3470f369eef19bf7526720e46da72512f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 5 Feb 2018 18:12:12 -0300 Subject: [PATCH 0866/1506] Fix nplusone issue in vulns api A regression made the creator field to do a large amount of queries Revert "Fix creator for Web UI. add unit tests" This reverts commit 7cfdcf25fe757136a3d6aa0f0621243d0a1ec208. --- server/api/modules/vulns.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 44db6271299..915cbfefad7 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -27,7 +27,6 @@ from server.fields import FaradayUploadedFile from server.models import ( db, - Command, CommandObject, File, Host, @@ -86,14 +85,7 @@ class CustomMetadataSchema(MetadataSchema): creator = fields.Method('get_creator', dump_only=True) def get_creator(self, obj): - - if obj: - command = db.session.query(Command, CommandObject.command_id).join(CommandObject).filter_by( - object_type='vulnerability', - object_id=obj.id).first() - if command: - return command[0].tool - return 'Web UI' + return obj.creator_command_tool or 'Web UI' class VulnerabilitySchema(AutoSchema): From a70a2ac256ec39031e5e21853701d1a93ed333a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 5 Feb 2018 18:25:49 -0300 Subject: [PATCH 0867/1506] Fix factory-boy and pytest-factoryboy dependency conflicts https://github.com/pytest-dev/pytest-factoryboy/issues/47 was fixed in 2.0.1. Now it isn't needed (actually, it break its) to require old versions of factory boy --- requirements_dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 0c1f9128949..0b258cbb70e 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,4 @@ -factory-boy<=2.8.1 +factory-boy pytest pytest-factoryboy responses From 18edb2bad87ec171229729ad49c3e828d1ffd613 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Mon, 5 Feb 2018 19:05:50 -0300 Subject: [PATCH 0868/1506] Fix string initdb --- server/commands/initdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index ab39704146f..c89f1d6174b 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -140,7 +140,7 @@ def _create_admin_user(self, conn_string): " {red}password{white}: {" "random_password}".format(random_password=random_password, white=Fore.WHITE, red=Fore.RED)) - print("{yellow} WARNING {white} If you are going to execute couchdb importer you must use the couddb password for faraday user.") + print("{yellow}WARNING{white}: If you are going to execute couchdb importer you must use the couchdb password for faraday user.".format(white=Fore.WHITE, yellow=Fore.YELLOW)) def _configure_existing_postgres_user(self): From 3cf46b00964b96fce28b5142b588a728158e4593 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 7 Feb 2018 19:06:03 -0300 Subject: [PATCH 0869/1506] Add support for conflicts for service * Add support for 409 when a new service is created. We show the proper pmessage and avoid closing the modal. * Add support for 409 on edit and the service already exists. We avoid closing the modal. * Add support for bulk edit and 409. * Some code was deleted and also the _load was fixed to support the new api --- .../www/scripts/commons/providers/server.js | 2 +- server/www/scripts/hosts/controllers/host.js | 31 ++-------------- .../services/controllers/serviceModalEdit.js | 35 ++++++++++++++----- .../services/controllers/serviceModalNew.js | 22 +++++++----- .../scripts/services/partials/modalEdit.html | 18 +++++----- .../scripts/services/partials/modalNew.html | 14 ++++---- .../www/scripts/services/providers/service.js | 5 +-- .../scripts/services/providers/services.js | 31 ++++++++-------- 8 files changed, 76 insertions(+), 82 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index aa650d7396b..ba1c335423f 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -238,7 +238,7 @@ angular.module("faradayApp") return get(url); } - ServerAPI.getService = function(wsName, data, objId) { + ServerAPI.getService = function(wsName, objId) { var getUrl = createGetUrl(wsName, 'services', objId); return get(getUrl); } diff --git a/server/www/scripts/hosts/controllers/host.js b/server/www/scripts/hosts/controllers/host.js index 3df50126baa..c79a25b357b 100644 --- a/server/www/scripts/hosts/controllers/host.js +++ b/server/www/scripts/hosts/controllers/host.js @@ -25,9 +25,6 @@ angular.module('faradayApp') // services by host var hostId = $routeParams.hidId; dashboardSrv.getServicesByHost($scope.workspace, hostId) - .then(function(services) { - return $q.all(services); - }) .then(function(services) { $scope.services = services; @@ -248,35 +245,13 @@ angular.module('faradayApp') }); modal.result.then(function(data) { - $scope.insert(data); - }); - }; - - $scope.insert = function(service) { - servicesManager.createService(service, $scope.workspace).then(function(service) { - $scope.services.push(service); - }, function(message) { - $uibModal.open(config = { - templateUrl: 'scripts/commons/partials/modalKO.html', - controller: 'commonsModalKoCtrl', - size: 'sm', - resolve: { - msg: function() { - return message; - } - } - }); + $scope.services.push(service); }); }; $scope.update = function(services, data) { - services.forEach(function(service) { - servicesManager.updateService(service, data, $scope.workspace).then(function(s) { - loadServices(); - }, function(message) { - console.log(message); - }); - }); + //hostId + loadServices(); }; $scope.edit = function() { diff --git a/server/www/scripts/services/controllers/serviceModalEdit.js b/server/www/scripts/services/controllers/serviceModalEdit.js index 381d53d2a2f..16e95deddaf 100644 --- a/server/www/scripts/services/controllers/serviceModalEdit.js +++ b/server/www/scripts/services/controllers/serviceModalEdit.js @@ -4,15 +4,15 @@ angular.module('faradayApp') .controller('serviceModalEdit', - ['$scope', '$modalInstance', '$routeParams', 'SERVICE_STATUSES', 'service', 'servicesManager', 'commonsFact', - function($scope, $modalInstance, $routeParams, SERVICE_STATUSES, service, servicesManager, commons) { + ['$q', '$scope', '$modalInstance', '$routeParams', 'SERVICE_STATUSES', 'service', 'servicesManager', 'commonsFact', 'servicesManager', + function($q, $scope, $modalInstance, $routeParams, SERVICE_STATUSES, service, servicesManager, commonsFact, servicesManager) { init = function() { // current Workspace var ws = $routeParams.wsId; if(service.length == 1) { - $scope.service = { + $scope.data = { "name": service[0].name, "description": service[0].description, "owned": service[0].owned, @@ -23,21 +23,40 @@ angular.module('faradayApp') "status": service[0].status, "version": service[0].version, }; - } else { - $scope.services_selected = service; } + $scope.servicesSelected = service; + $scope.statuses = SERVICE_STATUSES; }; $scope.ok = function() { - var date = new Date(), + var date = new Date(), updateAll = []; + var ws = $routeParams.wsId; timestamp = date.getTime()/1000.0; - $modalInstance.close($scope.service); + for (var i = 0; i < $scope.servicesSelected.length; i++) { + console.log($scope.servicesSelected[i]._id); + updateAll.push(servicesManager.getService($scope.servicesSelected[i]._id, ws, false).then(function(serviceObj){ + console.log(serviceObj._id); + $scope.data['_id'] = serviceObj._id; + return servicesManager.updateService(serviceObj, $scope.data, $routeParams.wsId); + })); + } + + $q.all(updateAll).then(function(){ + $modalInstance.close($scope.data); + }, function(response) { + if (response.status == 409) { + commonsFact.showMessage("Error while updating a new Vulnerability " + response.data.name + " Conflicting Vulnarability with id: " + response.data.object._id + ". " + response.data.message); + } else { + commonsFact.showMessage("Error from backend: " + response.status); + } + + }) }; $scope.call = function(service) { - $scope.service = { + $scope.data = { "name": service.name, "description": service.description, "owned": service.owned, diff --git a/server/www/scripts/services/controllers/serviceModalNew.js b/server/www/scripts/services/controllers/serviceModalNew.js index 913d41d40ed..71a77a5934e 100644 --- a/server/www/scripts/services/controllers/serviceModalNew.js +++ b/server/www/scripts/services/controllers/serviceModalNew.js @@ -4,11 +4,11 @@ angular.module('faradayApp') .controller('serviceModalNew', - ['$scope', '$modalInstance', '$routeParams', 'SERVICE_STATUSES', 'host', 'servicesManager', 'hostsManager', - function($scope, $modalInstance, $routeParams, SERVICE_STATUSES, host, servicesManager, hostsManager) { + ['$scope', '$modalInstance', '$routeParams', 'SERVICE_STATUSES', 'host', 'servicesManager', 'hostsManager', 'commonsFact', + function($scope, $modalInstance, $routeParams, SERVICE_STATUSES, host, servicesManager, hostsManager, commonsFact) { init = function() { - $scope.service = { + $scope.data = { "name": "", "description": "", "owned": false, @@ -21,16 +21,15 @@ angular.module('faradayApp') }; // current Workspace var ws = $routeParams.wsId; - $scope.service.parent = host.id; + $scope.data.parent = host.id; $scope.statuses = SERVICE_STATUSES; - }; $scope.ok = function() { var date = new Date(), timestamp = date.getTime()/1000.0; - $scope.service.metadata = { + $scope.data.metadata = { "update_time": timestamp, "update_user": "", "update_action": 0, @@ -39,8 +38,15 @@ angular.module('faradayApp') "update_controller_action": "UI Web New", "owner": "" }; - - $modalInstance.close($scope.service); + servicesManager.createService($scope.data, $routeParams.wsId).then(function() { + $modalInstance.close($scope.data); + }, function(response) { + if (response.status == 409) { + commonsFact.showMessage("Error while creating a new Service " + response.data.name + " Conflicting Vulnarability with id: " + response.data.object._id + ". " + response.data.message); + } else { + commonsFact.showMessage("Error from backend: " + response.status); + } + }); }; $scope.cancel = function() { diff --git a/server/www/scripts/services/partials/modalEdit.html b/server/www/scripts/services/partials/modalEdit.html index 0bfd952155a..6779b0fb157 100644 --- a/server/www/scripts/services/partials/modalEdit.html +++ b/server/www/scripts/services/partials/modalEdit.html @@ -15,20 +15,20 @@

    Edit service

    - +
    - +
    @@ -39,28 +39,28 @@

    Edit service

    Port
    - +
    Protocol
    - +
    Version
    - +
    Status
    -
    -
    +
    Services to update
    @@ -86,7 +86,7 @@
    - + diff --git a/server/www/scripts/services/partials/modalNew.html b/server/www/scripts/services/partials/modalNew.html index 721ce7f0ee7..1999f2ed414 100644 --- a/server/www/scripts/services/partials/modalNew.html +++ b/server/www/scripts/services/partials/modalNew.html @@ -15,20 +15,20 @@

    New service

    - +
    - +
    @@ -39,23 +39,23 @@

    New service

    Port
    - +
    Protocol
    - +
    Version
    - +
    Status
    -
    diff --git a/server/www/scripts/services/providers/service.js b/server/www/scripts/services/providers/service.js index 6537715ada3..51c703129dc 100644 --- a/server/www/scripts/services/providers/service.js +++ b/server/www/scripts/services/providers/service.js @@ -38,10 +38,7 @@ angular.module('faradayApp') self.ports = [self.ports]; } - return (self._save(ws, self, true).then( - function(data) { - self._rev = data.rev; - })); + return self._save(ws, self, true); }, save: function(ws) { var self = this; diff --git a/server/www/scripts/services/providers/services.js b/server/www/scripts/services/providers/services.js index 3a98ae97587..1ff79a1ba99 100644 --- a/server/www/scripts/services/providers/services.js +++ b/server/www/scripts/services/providers/services.js @@ -25,21 +25,18 @@ angular.module('faradayApp') return this._objects[id]; } - servicesManager._load = function(id, ws, deferred) { + servicesManager._load = function(id, ws) { var self = this; + var deferred = $q.defer(); ServerAPI.getService(ws, id). then(function(response) { - if (response.data.services.length == 1) { - var service_data = response.data.services[0].value; - var service_id = service_data._id; - var service = self._get(service_id, service_data); - deferred.resolve(service); - } else { - deferred.reject("More than one object found by ID"); - } + + deferred.resolve(new Service(response.data)); + }, function(error) { deferred.reject(error); - }) + }); + return deferred.promise; }; servicesManager.getService = function(id, ws, force_reload) { @@ -49,7 +46,7 @@ angular.module('faradayApp') if((service) && (!force_reload)) { deferred.resolve(service); } else { - this._load(id, ws, deferred); + return this._load(id, ws); } return deferred.promise; } @@ -95,9 +92,8 @@ angular.module('faradayApp') var service = new Service(serviceData); service.save(ws).then(function(saved_service){ deferred.resolve(saved_service.data); - }, function(){ - // host couldn't be saved - deferred.reject("Error: host couldn't be saved"); + }, function(response){ + deferred.reject(response); }) return deferred.promise; @@ -106,9 +102,10 @@ angular.module('faradayApp') servicesManager.updateService = function(service, data, ws) { var deferred = $q.defer(); var self = this; - self._get(service._id, service).update(data, ws).then(function(){ - service = self._load(service._id, ws, deferred); - deferred.resolve(service); + service.update(data, ws).then(function() { + deferred.resolve(); + }, function(response) { + deferred.reject(response); }); return deferred.promise; } From 8b96293539cae31a514e497de7404db78c478592 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 7 Feb 2018 19:27:38 -0300 Subject: [PATCH 0870/1506] Refresh after a new service was created --- server/www/scripts/hosts/controllers/host.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/hosts/controllers/host.js b/server/www/scripts/hosts/controllers/host.js index c79a25b357b..642d60cab9d 100644 --- a/server/www/scripts/hosts/controllers/host.js +++ b/server/www/scripts/hosts/controllers/host.js @@ -245,7 +245,7 @@ angular.module('faradayApp') }); modal.result.then(function(data) { - $scope.services.push(service); + loadServices(); }); }; From fe5cf1332e47ac72e14c1d9eea6b2e741e366ca4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 8 Feb 2018 15:41:16 -0300 Subject: [PATCH 0871/1506] Refactor service API eagerloading technique Use the helpers in server/api/base.py instead of overwriting _get_base_query --- server/api/modules/services.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/server/api/modules/services.py b/server/api/modules/services.py index bbe2655c46f..537d747e340 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -86,6 +86,7 @@ class ServiceView(ReadWriteWorkspacedView): schema_class = ServiceSchema count_extra_filters = [Service.status == 'open'] get_undefer = [Service.credentials_count, Service.vulnerability_count] + get_joinedloads = [Service.credentials] unique_fields = [('port', 'protocol', 'host')] def _envelope_list(self, objects, pagination_metadata=None): @@ -100,10 +101,5 @@ def _envelope_list(self, objects, pagination_metadata=None): 'services': services, } - def _get_base_query(self, workspace_name): - original = super(ServiceView, self)._get_base_query(workspace_name) - return original.options( - joinedload(Service.credentials) - ) ServiceView.register(services_api) From 79d3dcab0ae7bb82f9d8670c901cc5d42439351e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 8 Feb 2018 16:27:33 -0300 Subject: [PATCH 0872/1506] Fix an issue while generating the csv. The value was a number and replace was not a valid method --- server/www/scripts/csv/providers/csv.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/csv/providers/csv.js b/server/www/scripts/csv/providers/csv.js index 4b91c6aa9d5..a02f544eebf 100644 --- a/server/www/scripts/csv/providers/csv.js +++ b/server/www/scripts/csv/providers/csv.js @@ -28,7 +28,7 @@ angular.module('faradayApp') aProperties.forEach(function(prop) { object = {}; if(typeof(v[prop]) === "object") v[prop] = parseObject(v[prop]); - if(typeof(v[prop]) != "undefined" && v[prop] != null) { + if(typeof(v[prop]) != "undefined" && v[prop] != null && typeof(v[prop]) != "number") { object[prop] = cleanCSV(v[prop]); } else { object[prop] = ""; From ae4667b0a6aeffc61b8dbf95538e97fa8a61b3eb Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 9 Feb 2018 14:39:59 -0300 Subject: [PATCH 0873/1506] Add parent to persitence server model service Fixes filter_services fpluging --- bin/filter_services.py | 4 ++-- persistence/server/models.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/filter_services.py b/bin/filter_services.py index 3ff6bee5d93..5e0b3108d7d 100644 --- a/bin/filter_services.py +++ b/bin/filter_services.py @@ -29,8 +29,8 @@ # FIXME Update when persistence API changes COLUMNS = { - 'host': lambda service, workspace: models.get_host(workspace, service.id.split('.')[0]).name, - 'host_os': lambda service, workspace: models.get_host(workspace, service.id.split('.')[0]).os, + 'host': lambda service, workspace: models.get_host(workspace, service.getParent()).name, + 'host_os': lambda service, workspace: models.get_host(workspace, service.getParent()).os, 'service': lambda service, workspace: service.name, 'ports': lambda service, workspace: str(service.ports[0]), 'protocol': lambda service, workspace: service.protocol, diff --git a/persistence/server/models.py b/persistence/server/models.py index 71e6caa916a..a64902e8578 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -882,6 +882,7 @@ class Service(ModelBase): def __init__(self, service, workspace_name): ModelBase.__init__(self, service, workspace_name) self.protocol = service['protocol'] + self.parent_id = service['parent'] if type(service['ports']) == int: # the new api returns an integer in ports self.ports = [service['ports']] @@ -922,6 +923,9 @@ def updateAttributes(self, name=None, description=None, protocol=None, ports=Non def __str__(self): return "{0} ({1})".format(self.name, self.vuln_amount) + def getParent(self): + return self.parent_id + def getStatus(self): return self.status From 43d445a72e66627550772b10b41d7d5b37458ad9 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 14 Feb 2018 14:46:35 -0300 Subject: [PATCH 0874/1506] Fix create_vuln fplugin --- bin/create_vuln.py | 34 +++++++++++++++++++------ persistence/server/server.py | 2 ++ server/api/modules/vulns.py | 6 ++++- test_cases/test_api_vulnerability.py | 37 ++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 8 deletions(-) diff --git a/bin/create_vuln.py b/bin/create_vuln.py index fd936822ca9..81ff7d69d9e 100644 --- a/bin/create_vuln.py +++ b/bin/create_vuln.py @@ -22,6 +22,10 @@ def main(workspace='', args=None, parser=None): help='Vulnerability severity', choices=['critical', 'high', 'med', 'low', 'info', 'unclassified'], default='unclassified') + parser.add_argument('--parent_type', + help='Vulnerability severity', + choices=['Host', 'Service'], + default='unclassified') parser.add_argument('--resolution', help='Resolution', default='') parser.add_argument('--confirmed', help='Is the vulnerability confirmed', @@ -33,22 +37,38 @@ def main(workspace='', args=None, parser=None): parsed_args = parser.parse_args(args) - obj = factory.createModelObject(models.Vuln.class_signature, parsed_args.name, workspace, + obj = factory.createModelObject(models.Vuln.class_signature, + parsed_args.name, + workspace, ref=parsed_args.reference, severity=parsed_args.severity, resolution=parsed_args.resolution, confirmed=(parsed_args.confirmed == 'true'), desc=parsed_args.description, - parent_id=parsed_args.parent + parent_id=parsed_args.parent, + parent_type=parsed_args.parent_type.capitalize() ) + params = { + 'name': parsed_args.name, + 'description': parsed_args.description, + 'parent_type': parsed_args.parent_type.capitalize(), + 'parent': parsed_args.parent, + } - old = models.get_vuln(workspace, obj.getID()) + old = models.get_vulns( + workspace, + **params + ) - if old is None: + if not old: if not parsed_args.dry_run: - models.create_vuln(workspace, obj) + models.create_vuln(workspace, obj, None) + old = models.get_vulns( + workspace, + **params + ) else: - print "A vulnerability with ID %s already exists!" % obj.getID() + print "A vulnerability with ID %s already exists!" % old[0].getID() return 2, None - return 0, obj.getID() + return 0, old[0].getID() diff --git a/persistence/server/server.py b/persistence/server/server.py index 14a51638b05..1b6e0153c47 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -195,6 +195,8 @@ def _unsafe_io_with_server(server_io_function, server_expected_response, raise requests.exceptions.RequestException(response=answer) except requests.exceptions.RequestException as ex: logger.debug(ex) + if answer and 'messages' in answer.json(): + logger.info('Faraday server error message: {0}'.format(answer.json()['messages'])) raise CantCommunicateWithServerError(server_io_function, server_url, payload, answer) return answer diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 915cbfefad7..cf117998192 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -181,7 +181,11 @@ def load_type(self, value): return 'vulnerability_web' def load_parent(self, value): - if type(value) != int: + try: + # sometimes api requests send str or unicode. + value = int(value) + except ValueError: + raise ValidationError("Invalid parent type") return value diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index c70d894e1c7..b3298d34bec 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1269,6 +1269,43 @@ def test_vuln_created_without_command_has_webui_in_metadata(self, test_client, s assert res.status_code == 201 assert res.json['metadata']['creator'] == 'Web UI' + def test_invalid_host_id_error_message(self, test_client): + """ + This test reporduces a bug when the parent_id is a string it returned + the error message "Invalid Parent Type" + """ + raw_data = { + 'command_id': None, + 'confirmed': False, + 'data': None, + 'desc': 'pepe', + 'description': 'pepe', + 'metadata': { + 'command_id': '', + 'create_time': 1518627247.194113, + 'creator': '', + 'owner': '', + 'update_action': 0, + 'update_controller_action': 'No model controller call', + 'update_time': 1518627247.194114, + 'update_user': ''}, + 'name': 'vuln1', + 'owned': False, + 'owner': '', + 'parent': '358302', + 'parent_type': 'Host', + 'policyviolations': [], + 'refs': [], + 'resolution': '', + 'severity': 'critical', + 'status': 'opened', + 'type': 'Vulnerability' + } + + res = test_client.post(self.url(), data=raw_data) + assert res.status_code == 400 + assert res.json == {u'messages': {u'_schema': [u'Parent id not found: 358302']}} + def test_type_filter(workspace, session, vulnerability_factory, From 42bce0492f51f3524af14433700d3aac5201e935 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 14 Feb 2018 15:15:38 -0300 Subject: [PATCH 0875/1506] Set command_id default vlaue to None. --- bin/create_vuln.py | 2 +- persistence/server/models.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/create_vuln.py b/bin/create_vuln.py index 81ff7d69d9e..9b04128947d 100644 --- a/bin/create_vuln.py +++ b/bin/create_vuln.py @@ -62,7 +62,7 @@ def main(workspace='', args=None, parser=None): if not old: if not parsed_args.dry_run: - models.create_vuln(workspace, obj, None) + models.create_vuln(workspace, obj) old = models.get_vulns( workspace, **params diff --git a/persistence/server/models.py b/persistence/server/models.py index a64902e8578..92fdf83f2e5 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -373,7 +373,7 @@ def update_service(workspace_name, service, command_id): @_ignore_in_changes -def create_vuln(workspace_name, vuln, command_id): +def create_vuln(workspace_name, vuln, command_id=None): """Take a workspace_name and an vulnerability object and save it to the sever. The rev parameter must be provided if you are updating the object. Return the server's json response as a dictionary. @@ -383,7 +383,7 @@ def create_vuln(workspace_name, vuln, command_id): @_ignore_in_changes -def update_vuln(workspace_name, vuln, command_id): +def update_vuln(workspace_name, vuln, command_id=None): """Take a workspace_name and a Vuln object and update it in the sever. Return the server's json response as a dictionary. From 523583ea514bafbe50d4304209291aa483bc3e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 14 Feb 2018 16:27:27 -0300 Subject: [PATCH 0876/1506] Fix bug when creating services with empty IP It shouldn't be allowed, but it is for now. Avoid raising an AttributeError produced by an inline or expression that doesn't distinguish between None and "" --- server/events.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/events.py b/server/events.py index 5760beaa1f6..558ab700f2e 100644 --- a/server/events.py +++ b/server/events.py @@ -13,7 +13,10 @@ def new_object_event(mapper, connection, instance): # Since we don't have jet a model for workspace we # retrieve the name from the connection string - name = getattr(instance, 'ip', None) or instance.name + try: + name = instance.ip + except AttributeError: + name = instance.name msg = { 'id': instance.id, 'action': 'CREATE', @@ -80,4 +83,4 @@ def after_insert_check_child_has_same_workspace(mapper, connection, inserted_ins # Update object bindings event.listen(Host, 'after_update', update_object_event) -event.listen(Service, 'after_update', update_object_event) \ No newline at end of file +event.listen(Service, 'after_update', update_object_event) From 6da7d60c8c3b3e97c3ea852fea854ccc7903db9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 14 Feb 2018 16:49:46 -0300 Subject: [PATCH 0877/1506] Fix typo in service API type was "Host" instead of "Service" --- server/api/modules/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/api/modules/services.py b/server/api/modules/services.py index 537d747e340..397b85b0a8e 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -39,7 +39,7 @@ class ServiceSchema(AutoSchema): vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) credentials = fields.Integer(attribute='credentials_count', dump_only=True) metadata = SelfNestedField(MetadataSchema()) - type = fields.Function(lambda obj: 'Host', dump_only=True) + type = fields.Function(lambda obj: 'Service', dump_only=True) def load_ports(self, value): # TODO migration: handle empty list and not numeric value From 9fe0662d7f74126027cdd58351506e7ecc9507ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 14 Feb 2018 17:26:51 -0300 Subject: [PATCH 0878/1506] Fix bug in severity filter It wasn't translating 'med' to 'medium' or 'info' to 'informational' so some filters failed used from the web ui --- server/api/modules/vulns.py | 1 + test_cases/test_api_vulnerability.py | 29 +++++++++++++++++----------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index cf117998192..117855c4dee 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -303,6 +303,7 @@ class Meta(FilterSetMeta): type = TypeFilter(fields.Str(validate=[OneOf(['Vulnerability', 'VulnerabilityWeb'])])) creator = CreatorFilter(fields.Str()) + severity = Filter(SeverityField()) def filter(self): """Generate a filtered query from request parameters. diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index b3298d34bec..7ef2c81de66 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -539,34 +539,41 @@ def test_create_vuln_web(self, host_with_hostnames, test_client, session): assert res.json['path'] == '/pepep' @pytest.mark.usefixtures('mock_envelope_list') + @pytest.mark.parametrize('medium_name', ['medium', 'med']) def test_filter_by_severity(self, test_client, session, second_workspace, vulnerability_factory, vulnerability_web_factory, + medium_name, ): expected_ids = set() - high_vulns = vulnerability_factory.create_batch( + vulnerability_factory.create_batch( 5, workspace=second_workspace, severity='high') - high_vulns_web = vulnerability_web_factory.create_batch( + vulnerability_web_factory.create_batch( 5, workspace=second_workspace, severity='high') - session.commit() - - expected_ids.update(vuln.id for vuln in high_vulns) - expected_ids.update(vuln.id for vuln in high_vulns_web) - vulnerability_factory.create_batch( + medium_vulns = vulnerability_factory.create_batch( 5, workspace=second_workspace, severity='medium') - vulnerability_web_factory.create_batch( + medium_vulns_web = vulnerability_web_factory.create_batch( 5, workspace=second_workspace, severity='medium') + session.commit() + expected_ids.update(vuln.id for vuln in medium_vulns) + expected_ids.update(vuln.id for vuln in medium_vulns_web) + res = test_client.get(self.url( - workspace=second_workspace) + '?severity=high') + workspace=second_workspace) + '?severity=%s' % medium_name) assert res.status_code == 200 for vuln in res.json['data']: - assert vuln['severity'] == 'high' + assert vuln['severity'] == 'med' assert set(vuln['_id'] for vuln in res.json['data']) == expected_ids + def test_filter_by_invalid_severity_fails(self, test_client): + res = test_client.get(self.url() + '?severity=131231') + assert res.status_code == 400 + assert 'Invalid severity type' in res.data + @pytest.mark.usefixtures('mock_envelope_list') def test_filter_by_invalid_severity(self, test_client): res = test_client.get(self.url() + '?severity=invalid') @@ -1355,4 +1362,4 @@ def test_creator_filter(workspace, session, VulnerabilityGeneric, 'creator', 'metasploit') - assert {v.id for v in filtered} == {v.id for v in vulns} \ No newline at end of file + assert {v.id for v in filtered} == {v.id for v in vulns} From f5f63be71f036a61060b1d6afbf565e1a73ca72a Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 14 Feb 2018 17:48:30 -0300 Subject: [PATCH 0879/1506] Fix broken tests --- persistence/server/models.py | 8 ++++---- persistence/server/server.py | 14 ++++++++++---- test_cases/test_models.py | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index 92fdf83f2e5..1981b6e9087 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -261,9 +261,9 @@ def get_credentials(workspace_name, **params): return _get_faraday_ready_credentials(workspace_name, credentials_dictionary) -def get_credential(workspace_name, credential_id): +def get_credential(workspace_name, credential_id, **params): """Return the Credential of id credential_id. None if not found.""" - return force_unique(get_credentials(workspace_name, id=credential_id)) + return force_unique(get_credentials(workspace_name, id=credential_id, **params)) def get_notes(workspace_name, **params): @@ -431,7 +431,7 @@ def update_note(workspace_name, note, command_id): @_ignore_in_changes -def create_credential(workspace_name, credential, command_id): +def create_credential(workspace_name, credential, command_id=None): """Take a workspace_name and an credential object and save it to the sever. Return the server's json response as a dictionary. """ @@ -882,7 +882,7 @@ class Service(ModelBase): def __init__(self, service, workspace_name): ModelBase.__init__(self, service, workspace_name) self.protocol = service['protocol'] - self.parent_id = service['parent'] + self.parent_id = service.get('parent') or service.get('host_id') or service.get('service_id') if type(service['ports']) == int: # the new api returns an integer in ports self.ports = [service['ports']] diff --git a/persistence/server/server.py b/persistence/server/server.py index 1b6e0153c47..6d6dba17307 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -22,6 +22,7 @@ be used with care, specially regarding the ID of objects, which must be always unique. """ +import urllib import os import json @@ -96,7 +97,7 @@ def _create_server_api_url(): """Return the server's api url.""" return "{0}/_api/v2".format(_get_base_server_url()) -def _create_server_get_url(workspace_name, object_name=None, object_id=None): +def _create_server_get_url(workspace_name, object_name=None, object_id=None, **params): """Creates a url to get from the server. Takes the workspace name as a string, an object_name paramter which is the object you want to query as a string ('hosts', 'interfaces', etc) . @@ -110,6 +111,8 @@ def _create_server_get_url(workspace_name, object_name=None, object_id=None): get_url = '{0}/ws/{1}{2}'.format(_create_server_api_url(), workspace_name, object_name) + if params: + get_url = '{0}?{1}'.format(get_url, urllib.urlencode(params)) return get_url @@ -195,8 +198,11 @@ def _unsafe_io_with_server(server_io_function, server_expected_response, raise requests.exceptions.RequestException(response=answer) except requests.exceptions.RequestException as ex: logger.debug(ex) - if answer and 'messages' in answer.json(): - logger.info('Faraday server error message: {0}'.format(answer.json()['messages'])) + try: + if answer and 'messages' in answer.json(): + logger.info('Faraday server error message: {0}'.format(answer.json()['messages'])) + except ValueError: + logger.debug('Could not decode json from server') raise CantCommunicateWithServerError(server_io_function, server_url, payload, answer) return answer @@ -292,7 +298,7 @@ def _get_raw_notes(workspace_name, **params): def _get_raw_credentials(workspace_name, **params): """Take a workspace name and an arbitrary number of params and return a dictionary with the credentials table.""" - request_url = _create_server_get_url(workspace_name, 'credential', params.get('id', None)) + request_url = _create_server_get_url(workspace_name, 'credential', params.pop('id', None), **params) return _get(request_url, **params) diff --git a/test_cases/test_models.py b/test_cases/test_models.py index 7595bb1f5e4..4ca847a9bfe 100644 --- a/test_cases/test_models.py +++ b/test_cases/test_models.py @@ -8,7 +8,7 @@ INTERFACE_JSON_STRING = '{"_id":1,"id":"08d3b6545ec70897daf05cd471f4166a8e605c00.02946afc59c50a4d76c1adbb082c2d5439baf50a","key":"08d3b6545ec70897daf05cd471f4166a8e605c00.02946afc59c50a4d76c1adbb082c2d5439baf50a","value":{"_id":"08d3b6545ec70897daf05cd471f4166a8e605c00.02946afc59c50a4d76c1adbb082c2d5439baf50a","_rev":"1-c279e0906d2b1f02b832a99d5f58f99c","description":"","host_id":1,"hostnames":["qa3app09"],"ipv4":{"DNS":[],"address":"10.31.112.29","gateway":"0.0.0.0","mask":"0.0.0.0"},"ipv6":{"DNS":[],"address":"0000:0000:0000:0000:0000:0000:0000:0000","gateway":"0000:0000:0000:0000:0000:0000:0000:0000","prefix":"00"},"mac":"00:50:56:81:01:e3","metadata":{"create_time":1475852074.456803,"creator":"","owner":"","update_action":0,"update_controller_action":"ModelControler._processAction ModelControler.newInterface","update_time":1475852074.456803,"update_user":""},"name":"10.31.112.29","network_segment":"","owned":false,"owner":"","ports":{"closed":null,"filtered":null,"opened":null}}}' -SERVICE_JSON_STRING = '{"_id":1,"id":"08d3b6545ec70897daf05cd471f4166a8e605c00.02946afc59c50a4d76c1adbb082c2d5439baf50a.029384202ef91fff5892042392875595fb0b41ed","key":"08d3b6545ec70897daf05cd471f4166a8e605c00.02946afc59c50a4d76c1adbb082c2d5439baf50a.029384202ef91fff5892042392875595fb0b41ed","value":{"_id":"08d3b6545ec70897daf05cd471f4166a8e605c00.02946afc59c50a4d76c1adbb082c2d5439baf50a.029384202ef91fff5892042392875595fb0b41ed","_rev":"1-73ef6b9e6488fd05823b89e36bbbb626","description":"","metadata":{"create_time":1475852074.457551,"creator":"","owner":"","update_action":0,"update_controller_action":"ModelControler._processAction ModelControler.newService","update_time":1475852074.457551,"update_user":""},"name":"msrdp","owned":false,"owner":"","ports":[3389],"protocol":"tcp","status":"open","version":"unknown"},"vulns":8}' +SERVICE_JSON_STRING = '{"_id":1,"id":"08d3b6545ec70897daf05cd471f4166a8e605c00.02946afc59c50a4d76c1adbb082c2d5439baf50a.029384202ef91fff5892042392875595fb0b41ed","key":"08d3b6545ec70897daf05cd471f4166a8e605c00.02946afc59c50a4d76c1adbb082c2d5439baf50a.029384202ef91fff5892042392875595fb0b41ed","value":{"_id":"08d3b6545ec70897daf05cd471f4166a8e605c00.02946afc59c50a4d76c1adbb082c2d5439baf50a.029384202ef91fff5892042392875595fb0b41ed","_rev":"1-73ef6b9e6488fd05823b89e36bbbb626","description":"","metadata":{"create_time":1475852074.457551,"creator":"","owner":"","update_action":0,"update_controller_action":"ModelControler._processAction ModelControler.newService","update_time":1475852074.457551,"update_user":""},"name":"msrdp","owned":false,"owner":"","ports":[3389],"protocol":"tcp","status":"open","version":"unknown", "host_id": 1},"vulns":8}' VULN_JSON_STRING = '{"_id":8,"id":"08d3b6545ec70897daf05cd471f4166a8e605c00.2a21f3916b8c9a40e70b2fc6b7ea8f7a3a498558","key":"08d3b6545ec70897daf05cd471f4166a8e605c00.2a21f3916b8c9a40e70b2fc6b7ea8f7a3a498558","value":{"_attachments":{},"_id":"08d3b6545ec70897daf05cd471f4166a8e605c00.2a21f3916b8c9a40e70b2fc6b7ea8f7a3a498558","_rev":"1-28cb6b1372f4712dbbf7b8e1e23699e4","confirmed":false,"data":"","desc":"Each ethernet MAC address starts with a 24-bit Organizationally Unique Identifier.\\nThese OUI are registered by IEEE.\\nOutput: The following card manufacturers were identified :\\n\\n00:50:56:81:01:e3 : VMware, Inc.","description":"Each ethernet MAC address starts with a 24-bit Organizationally Unique Identifier.\\nThese OUI are registered by IEEE.\\nOutput: The following card manufacturers were identified :\\n\\n00:50:56:81:01:e3 : VMware, Inc.","easeofresolution":null,"hostnames":["qa3app09"],"impact":{"accountability":null,"availability":null,"confidentiality":null,"integrity":null},"issuetracker":{},"metadata":{"create_time":1475852074.459108,"creator":"","owner":"","update_action":0,"update_controller_action":"ModelControler._processAction ModelControler.newVuln","update_time":1475852074.459108,"update_user":""},"method":null,"name":"Ethernet Card Manufacturer Detection","obj_id":"2a21f3916b8c9a40e70b2fc6b7ea8f7a3a498558","owned":"false","owner":"","params":"","parent":"08d3b6545ec70897daf05cd471f4166a8e605c00","path":null,"pname":null,"query":null,"refs":[],"request":null,"resolution":"n/a","response":null,"service":"","severity":"info","status":"","tags":[],"target":"10.31.112.29","type":"Vulnerability","website":null}}' From cf9e8dae5ed400032bdbf58804affcaefd8a3255 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 14 Feb 2018 18:08:15 -0300 Subject: [PATCH 0880/1506] Make SQL queries counter work properly on test cases Now it doesn't raise an error, but produces a warning. --- test_cases/conftest.py | 4 ++++ test_cases/test_api_hosts.py | 13 +++++++++---- test_cases/test_api_vulnerability.py | 1 + 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 09209d281fc..2638340ddb1 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -46,6 +46,10 @@ def open(self, *args, **kwargs): ('Content-Type', 'application/json'), ] + # Reset queries to make the log_queries_count + from flask import _app_ctx_stack + _app_ctx_stack.top.sqlalchemy_queries = [] + ret = super(CustomClient, self).open(*args, **kwargs) if ret.headers.get('content-type') == 'application/json': try: diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index f8004470d55..72b7454475a 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -210,8 +210,8 @@ def test_get_host_services(self, test_client, session, assert other_host.id not in ids_returned assert ids_expected == ids_returned - def test_retrieve_shows_service_count(self, test_client, host_services, - service_factory): + def test_retrieve_shows_service_count(self, test_client, session, + host_services, service_factory): for (host, services) in host_services.items(): # Adding closed and filtered services shouldn't impact on the # service count since it should only count opened services @@ -219,11 +219,12 @@ def test_retrieve_shows_service_count(self, test_client, host_services, workspace=host.workspace) service_factory.create_batch(2, status='filtered', host=host, workspace=host.workspace) + session.commit() res = test_client.get(self.url(host)) assert res.json['services'] == len(services) - def test_index_shows_service_count(self, test_client, host_services, - service_factory): + def test_index_shows_service_count(self, test_client, session, + host_services, service_factory): ids_map = {host.id: services for (host, services) in host_services.items()} @@ -235,6 +236,7 @@ def test_index_shows_service_count(self, test_client, host_services, service_factory.create_batch(2, status='filtered', host=host, workspace=host.workspace) + session.commit() res = test_client.get(self.url()) assert len(res.json['rows']) >= len(ids_map) # Some hosts can have no services for host in res.json['rows']: @@ -252,6 +254,7 @@ def test_filter_by_os_exact(self, test_client, session, workspace, # This shouldn't be shown, they are from other workspace host_factory.create_batch(5, workspace=second_workspace, os='Unix') + session.commit() url = self.url() + '?os=Unix' res = test_client.get(url) assert res.status_code == 200 @@ -273,6 +276,7 @@ def test_filter_by_os_like_ilike(self, test_client, session, workspace, # This shouldn't be shown anywhere, they are from other workspace host_factory.create_batch(5, workspace=second_workspace, os='Unix') + session.commit() res = test_client.get(self.url() + '?os__like=Unix %') assert res.status_code == 200 self.compare_results(hosts, res) @@ -290,6 +294,7 @@ def test_filter_by_service(self, test_client, session, workspace, # Hosts that shouldn't be shown host_factory.create_batch(5, workspace=workspace) + session.commit() res = test_client.get(self.url() + '?service=IRC') assert res.status_code == 200 shown_hosts_ids = set(obj['id'] for obj in res.json['rows']) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 7ef2c81de66..223c3ed7136 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -947,6 +947,7 @@ def test_target(self, test_client, session, second_workspace, workspace=second_workspace), ] + session.commit() res = test_client.get(self.url(workspace=second_workspace)) assert res.status_code == 200 for v in res.json['data']: From ecbdb6f91d52c26405120ba6fbc1eda75d9d7655 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 14 Feb 2018 18:31:04 -0300 Subject: [PATCH 0881/1506] Fix fplugin create credential --- bin/create_cred.py | 31 +++++++++++++++++++++++++------ persistence/server/models.py | 2 +- server/api/modules/credentials.py | 4 +++- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/bin/create_cred.py b/bin/create_cred.py index 8bb329d68ec..f8ae34fff92 100644 --- a/bin/create_cred.py +++ b/bin/create_cred.py @@ -19,23 +19,42 @@ def main(workspace='', args=None, parser=None): parser.add_argument('username', help='Username') parser.add_argument('password', help='Password') - parser.add_argument('--dry-run', action='store_true', help='Do not touch the database. Only print the object ID') + parser.add_argument('--parent_type', + help='Vulnerability severity', + choices=['Host', 'Service'], + default='unclassified') + parser.add_argument('--dry-run', action='store_true', help='Do not touch the database. Only print the object ID') parsed_args = parser.parse_args(args) - obj = factory.createModelObject(models.Credential.class_signature, parsed_args.name, workspace, + params = { + 'username': parsed_args.username, + } + + if parsed_args.parent_type == 'Host': + params.update({'host_id': parsed_args.parent}) + elif parsed_args.parent_type == 'Service': + params.update({'service_id': parsed_args.parent}) + else: + raise UserWarning('Credential only allow Host or Service as parent_type') + + obj = factory.createModelObject(models.Credential.class_signature, + parsed_args.name, + workspace, username=parsed_args.username, password=parsed_args.password, - parent_id=parsed_args.parent + parent_type=parsed_args.parent_type, + parent=parsed_args.parent ) - old = models.get_credential(workspace, obj.getID()) + old = models.get_credential(workspace, **params) if old is None: if not parsed_args.dry_run: models.create_credential(workspace, obj) + old = models.get_credential(workspace, **params) else: - print "A credential with ID %s already exists!" % obj.getID() + print "A credential with ID %s already exists!" % old.getID() return 2, None - return 0, obj.getID() + return 0, old.getID() diff --git a/persistence/server/models.py b/persistence/server/models.py index 1981b6e9087..c7074f77682 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -261,7 +261,7 @@ def get_credentials(workspace_name, **params): return _get_faraday_ready_credentials(workspace_name, credentials_dictionary) -def get_credential(workspace_name, credential_id, **params): +def get_credential(workspace_name, credential_id=None, **params): """Return the Credential of id credential_id. None if not found.""" return force_unique(get_credentials(workspace_name, id=credential_id, **params)) diff --git a/server/api/modules/credentials.py b/server/api/modules/credentials.py index f32693f1e83..c5ba7eb30d9 100644 --- a/server/api/modules/credentials.py +++ b/server/api/modules/credentials.py @@ -20,7 +20,7 @@ class CredentialSchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') _rev = fields.String(default='', dump_only=True) - owned = fields.Boolean(default=False) + owned = fields.Boolean(default=False, dump_only=True) owner = fields.String(dump_only=True, attribute='creator.username', default='') username = fields.String(default='') @@ -94,6 +94,8 @@ class CredentialFilterSet(FilterSet): class Meta(FilterSetMeta): model = Credential fields = ( + 'name', + 'username', 'host_id', 'service_id' ) From 1df8bcd50b8a65e4475f0975f1bbda70619ede1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 15 Feb 2018 13:58:08 -0300 Subject: [PATCH 0882/1506] Disable sorting vulns by reference and policy violations It doesn't make sense. Disable in the web ui --- server/www/scripts/statusReport/controllers/statusReport.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 172215019ce..5c1a8637118 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -343,7 +343,8 @@ angular.module('faradayApp') cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/refscolumn.html', headerCellTemplate: header, sort: getColumnSort('refs'), - visible: $scope.columns["refs"] + visible: $scope.columns["refs"], + enableSorting: false, }); $scope.gridOptions.columnDefs.push({ name : '_attachments', displayName: "evidence", @@ -410,7 +411,8 @@ angular.module('faradayApp') headerCellTemplate: header, width: '100', sort: getColumnSort('policyviolations'), - visible: $scope.columns["policyviolations"] + visible: $scope.columns["policyviolations"], + enableSorting: false, }); }; From d66bb369f7ae2f6a592e54e0519ecfff96ce2f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 15 Feb 2018 14:10:08 -0300 Subject: [PATCH 0883/1506] Fix error 500 when sorting vulns by service --- server/api/base.py | 8 +++++++- test_cases/test_api_vulnerability.py | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/server/api/base.py b/server/api/base.py index 79dd8038094..7959d3b9026 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -346,7 +346,13 @@ def _get_order_field(self, **kwargs): if sort_dir not in ('asc', 'desc'): raise InvalidUsage("Invalid value for sorting direction: %s" % sort_dir) - return getattr(field, sort_dir)() + try: + return getattr(field, sort_dir)() + except NotImplementedError: + # There are some fields that can't be used for sorting + raise InvalidUsage("field {} doesn't support sorting".format( + order_field + )) class PaginatedMixin(object): diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 223c3ed7136..a1369f95def 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -694,6 +694,11 @@ def test_sort_by_method(self, session, test_client, second_workspace, assert ''.join(v['method'] for v in res.json['data'] if v['method']) == 'abcdefghij'[::-1] + def test_cant_sort_by_service(self, test_client): + res = test_client.get(self.url() + '?sort=service&sort_dir=asc') + assert res.status_code == 400 + assert "service doesn't support sorting" in res.data + def test_create_vuln_with_evidence(self, host_with_hostnames, test_client, session): session.commit() # flush host_with_hostnames From d09c593471de416613f3326f0cee54545f0d4bfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 15 Feb 2018 14:29:04 -0300 Subject: [PATCH 0884/1506] Fix filter by status And disable the option of leaving it blank when updating vuln in the web --- server/api/modules/vulns.py | 2 +- server/www/scripts/statusReport/partials/modalEdit.html | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 117855c4dee..dd37c098d4b 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -291,7 +291,7 @@ class Meta(FilterSetMeta): "description", "command_id", "target", "creator") strict_fields = ( - "severity", "confirmed", "method" + "severity", "confirmed", "method", "status" ) default_operator = operators.ILike diff --git a/server/www/scripts/statusReport/partials/modalEdit.html b/server/www/scripts/statusReport/partials/modalEdit.html index acab6c1fc77..8384bcdc124 100644 --- a/server/www/scripts/statusReport/partials/modalEdit.html +++ b/server/www/scripts/statusReport/partials/modalEdit.html @@ -37,7 +37,6 @@
    Ease of Resolution
    Status
    From b1d3b794ef3e2cd0bf27565326e23162edd4c492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 15 Feb 2018 15:01:25 -0300 Subject: [PATCH 0885/1506] Fix filtering by ease of resolution Now the API accepts both "easeofresolution" and "ease_of_resolution" fields for compatiblity with the web --- server/api/modules/vulns.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index dd37c098d4b..a3c7aafb90c 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -288,10 +288,12 @@ class Meta(FilterSetMeta): "status", "website", "parameter_name", "query_string", "path", "data", "severity", "confirmed", "name", "request", "response", "parameters", "resolution", "method", "ease_of_resolution", - "description", "command_id", "target", "creator") + "description", "command_id", "target", "creator", + "easeofresolution") strict_fields = ( - "severity", "confirmed", "method", "status" + "severity", "confirmed", "method", "status", "easeofresolution", + "ease_of_resolution", ) default_operator = operators.ILike @@ -304,6 +306,10 @@ class Meta(FilterSetMeta): 'VulnerabilityWeb'])])) creator = CreatorFilter(fields.Str()) severity = Filter(SeverityField()) + easeofresolution = Filter(fields.String( + attribute='ease_of_resolution', + validate=OneOf(Vulnerability.EASE_OF_RESOLUTIONS), + allow_none=True)) def filter(self): """Generate a filtered query from request parameters. From 23416de36b2d39f8990569379c904c5d84509036 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 15 Feb 2018 16:12:34 -0300 Subject: [PATCH 0886/1506] Disable grouping by hostnames It doesn't make sense if we do the same as in previous versions. Grouping by target would be better. Maybe it could be added in future versions where a vuln can be shown in multiple groups --- server/www/scripts/statusReport/partials/statusReport.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/statusReport/partials/statusReport.html b/server/www/scripts/statusReport/partials/statusReport.html index 08725979377..8948feced5f 100644 --- a/server/www/scripts/statusReport/partials/statusReport.html +++ b/server/www/scripts/statusReport/partials/statusReport.html @@ -87,7 +87,7 @@

    Status report for {{ workspace

    From ad72def6ce0a3217155ced80631640a4082486bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 16 Feb 2018 14:17:31 -0300 Subject: [PATCH 0887/1506] Fix query string and parameter name compatibility Between the API and the Web UI --- server/api/modules/vulns.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index a3c7aafb90c..6fe5770330e 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -232,12 +232,12 @@ class VulnerabilityWebSchema(VulnerabilitySchema): method = fields.String(default='') params = fields.String(attribute='parameters', default='') - pname = fields.String(dump_only=True, attribute='parameter_name', default='') + pname = fields.String(attribute='parameter_name', default='') path = fields.String(default='') response = fields.String(default='') request = fields.String(default='') website = fields.String(default='') - query = fields.String(dump_only=True, attribute='query_string', default='') + query = fields.String(attribute='query_string', default='') class Meta: model = VulnerabilityWeb @@ -285,11 +285,11 @@ class Meta(FilterSetMeta): # command, impact, service, issuetracker, tags, date, # host, easeofresolution, evidence, policy violations, hostnames fields = ( - "status", "website", "parameter_name", "query_string", "path", + "status", "website", "pname", "query", "path", "data", "severity", "confirmed", "name", "request", "response", - "parameters", "resolution", "method", "ease_of_resolution", - "description", "command_id", "target", "creator", - "easeofresolution") + "parameters", "params", "resolution", "ease_of_resolution", + "description", "command_id", "target", "creator", "method", + "easeofresolution", "query_string", "parameter_name") strict_fields = ( "severity", "confirmed", "method", "status", "easeofresolution", @@ -310,6 +310,9 @@ class Meta(FilterSetMeta): attribute='ease_of_resolution', validate=OneOf(Vulnerability.EASE_OF_RESOLUTIONS), allow_none=True)) + pname = Filter(fields.String(attribute='parameter_name')) + query = Filter(fields.String(attribute='query_string')) + params = Filter(fields.String(attribute='parameters')) def filter(self): """Generate a filtered query from request parameters. From b073ce2df5958c6c5a6c11be0cbcb8cc069504ea Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 16 Feb 2018 14:26:32 -0300 Subject: [PATCH 0888/1506] Update create host to the new api --- bin/create_host.py | 54 ++++++++++++++++++++++ bin/create_host_and_interface.py | 79 -------------------------------- persistence/server/models.py | 14 ++++-- persistence/server/server.py | 2 +- server/api/modules/hosts.py | 2 +- 5 files changed, 67 insertions(+), 84 deletions(-) create mode 100644 bin/create_host.py delete mode 100644 bin/create_host_and_interface.py diff --git a/bin/create_host.py b/bin/create_host.py new file mode 100644 index 00000000000..92091751b05 --- /dev/null +++ b/bin/create_host.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python2.7 +# -*- coding: utf-8 -*- +""" +Faraday Penetration Test IDE +Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information +""" + +from model.common import factory +from persistence.server import models + +__description__ = 'Creates a new host in current workspace' +__prettyname__ = 'Create Host' + + +def main(workspace='', args=None, parser=None): + parser.add_argument('ip', help='Host IP') + parser.add_argument('os', help='OS') + + parser.add_argument('mac', help='Interface MAC Address') + + parser.add_argument('--gateway', help='IPV4 or IPV6 Gateway', default='0.0.0.0') + + parser.add_argument('--netsegment', help='Network Segment', default='') + + + parser.add_argument('--dry-run', action='store_true', help='Do not touch the database. Only print the object ID') + + parsed_args = parser.parse_args(args) + + params = { + 'ip': parsed_args.ip, + } + + obj_host = factory.createModelObject(models.Host.class_signature, + parsed_args.ip, + workspace, + os=parsed_args.os, + mac=parsed_args.mac, + network_segment=parsed_args.netsegment, + parent_id=None) + + + old_host = models.get_host(workspace, **params) + + if old_host is None: + if not parsed_args.dry_run: + models.create_host(workspace, obj_host) + old_host = models.get_host(workspace, **params) + else: + print "A host with ID %s already exists!" % old_host.getID() + return 2, None + + return 0, old_host.getID() diff --git a/bin/create_host_and_interface.py b/bin/create_host_and_interface.py deleted file mode 100644 index 795553ff61d..00000000000 --- a/bin/create_host_and_interface.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python2.7 -# -*- coding: utf-8 -*- -""" -Faraday Penetration Test IDE -Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) -See the file 'doc/LICENSE' for the license information -""" - -from model.common import factory -from persistence.server import models - -__description__ = 'Creates a new host and interface in current workspace' -__prettyname__ = 'Create Host and Interface' - - -def main(workspace='', args=None, parser=None): - parser.add_argument('host_name', help='Host name') - parser.add_argument('os', help='OS') - - parser.add_argument('interface_name', help='Interface Name') - parser.add_argument('mac', help='Interface MAC Address') - - parser.add_argument('--ipv4address', help='IPV4 Address', default='0.0.0.0') - parser.add_argument('--ipv4gateway', help='IPV4 Gateway', default='0.0.0.0') - parser.add_argument('--ipv4mask', help='IPV4 Mask', default='0.0.0.0') - parser.add_argument('--ipv4dns', help='IPV4 DNS, as a comma separated list', default='[]') - - parser.add_argument('--ipv6address', help='IPV6 Address', default='0000:0000:0000:0000:0000:0000:0000:0000') - parser.add_argument('--ipv6prefix', help='IPV6 Prefix', default='00') - parser.add_argument('--ipv6gateway', help='IPV4 Gateway', default='0000:0000:0000:0000:0000:0000:0000:0000') - parser.add_argument('--ipv6dns', help='IPV6 DNS, as a comma separated list', default='') - - parser.add_argument('--netsegment', help='Network Segment', default='') - parser.add_argument('--hostres', help='Hostname Resolution', default='') - - - parser.add_argument('--dry-run', action='store_true', help='Do not touch the database. Only print the object ID') - - parsed_args = parser.parse_args(args) - - obj_host = factory.createModelObject(models.Host.class_signature, parsed_args.host_name, - workspace, os=parsed_args.os, parent_id=None) - - - old_host = models.get_host(workspace, obj_host.getID()) - - if old_host is None: - if not parsed_args.dry_run: - models.create_host(workspace, obj_host) - else: - print "A host with ID %s already exists!" % obj_host.getID() - return 2, None - - - - obj_interface = factory.createModelObject(models.Interface.class_signature, parsed_args.interface_name, workspace, - mac=parsed_args.mac, - ipv4_address=parsed_args.ipv4address, - ipv4_mask=parsed_args.ipv4mask, - ipv4_gateway=parsed_args.ipv4gateway, - ipv4_dns=parsed_args.ipv4dns, - ipv6_address=parsed_args.ipv6address, - ipv6_prefix=parsed_args.ipv6prefix, - ipv6_gateway=parsed_args.ipv6gateway, - ipv6_dns=parsed_args.ipv6dns, - network_segment=parsed_args.netsegment, - hostname_resolution=parsed_args.hostres, - parent_id= obj_host.getID() ) - - old_interface = models.get_interface(workspace, obj_interface.getID()) - - if old_interface is None: - if not parsed_args.dry_run: - models.create_interface(workspace, obj_interface) - else: - print "An interface with ID %s already exists!" % obj_interface.getID() - return 2, None - - return 0, obj_interface.getID() diff --git a/persistence/server/models.py b/persistence/server/models.py index c7074f77682..8448f1d4970 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -33,6 +33,8 @@ MERGE_STRATEGY = None # you may change it the string 'NEW' to prefer new objects # you may ask why this can be None type or 'New' as a string # the answer is: Faraday. +logger = logging.getLogger(__name__) + def _conf(): if FARADAY_UP: @@ -191,9 +193,15 @@ def get_hosts(workspace_name, **params): return _get_faraday_ready_hosts(workspace_name, host_dictionaries) -def get_host(workspace_name, host_id): +def get_host(workspace_name, host_id=None, **params): """Return the host by host_id. None if it can't be found.""" - return get_hosts(workspace_name, id=host_id).pop() + hosts = get_hosts(workspace_name, object_id=host_id, **params) + if len(hosts) == 0: + return + else: + if len(hosts) > 1: + logger.warn('More than one hosts found. returning only one of them.') + return hosts.pop() def get_all_vulns(workspace_name, **params): @@ -334,7 +342,7 @@ def get_deleted_object_name_and_type(workspace_name, object_id): @_ignore_in_changes -def create_host(workspace_name, host, command_id): +def create_host(workspace_name, host, command_id=None): """Take a workspace_name and a host object and save it to the sever. Return the server's json response as a dictionary. diff --git a/persistence/server/server.py b/persistence/server/server.py index 6d6dba17307..5832ce86134 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -270,7 +270,7 @@ def _delete(delete_url, database=False): def _get_raw_hosts(workspace_name, **params): """Take a workspace_name and an arbitrary number of params and return a dictionary with the hosts table.""" - request_url = _create_server_get_url(workspace_name, 'hosts', params.get('id', None)) + request_url = _create_server_get_url(workspace_name, 'hosts', **params) return _get(request_url, **params) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index dd550b17821..e13c1221242 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -79,7 +79,7 @@ def filter(self, query, model, attr, value): class HostFilterSet(FilterSet): class Meta(FilterSetMeta): model = Host - fields = ('os', 'service') + fields = ('ip', 'os', 'service') operators = (operators.Equal, operators.Like, operators.ILike) service = ServiceFilter(fields.Str()) From efbdbdf2c512c5b7f08d67965d2240dc643e127b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 16 Feb 2018 14:34:02 -0300 Subject: [PATCH 0889/1506] Update create_note to use new api --- bin/create_note.py | 23 ++++++++++++----------- persistence/server/models.py | 2 +- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/bin/create_note.py b/bin/create_note.py index c6e55dbe508..737c9b1855b 100644 --- a/bin/create_note.py +++ b/bin/create_note.py @@ -5,6 +5,7 @@ Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) See the file 'doc/LICENSE' for the license information """ +import logging from model.common import factory from persistence.server import models @@ -12,9 +13,13 @@ __description__ = 'Creates a new note' __prettyname__ = 'Create Note' +logger = logging.getLogger(__name__) + def main(workspace='', args=None, parser=None): + logger.warn('Create note will create a comment. fplugin name will be changed to create_comment') parser.add_argument('parent', help='Parent ID') + parser.add_argument('parent_type', help='Parent Type') parser.add_argument('name', help='Note name') parser.add_argument('text', help='Note content') @@ -22,18 +27,14 @@ def main(workspace='', args=None, parser=None): parsed_args = parser.parse_args(args) - obj = factory.createModelObject(models.Note.class_signature, parsed_args.name, workspace, + obj = factory.createModelObject(models.Note.class_signature, + parsed_args.name, + workspace, text=parsed_args.text, - parent_id=parsed_args.parent + object_id=parsed_args.parent, + object_type=parsed_args.parent_type.lower() ) - old = models.get_note(workspace, obj.getID()) - - if old is None: - if not parsed_args.dry_run: - models.create_note(workspace, obj) - else: - print "A note with ID %s already exists!" % obj.getID() - return 2, None + models.create_note(workspace, obj) - return 0, obj.getID() + return 0, 1 diff --git a/persistence/server/models.py b/persistence/server/models.py index 8448f1d4970..001524791ee 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -421,7 +421,7 @@ def update_vuln_web(workspace_name, vuln_web, command_id): @_ignore_in_changes -def create_note(workspace_name, note, command_id): +def create_note(workspace_name, note, command_id=None): """Take a workspace_name and an note object and save it to the sever. Return the server's json response as a dictionary. """ From 52ff97b55309b6b09d5fa70196024cd3a0dc02f0 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 16 Feb 2018 14:53:44 -0300 Subject: [PATCH 0890/1506] Update create_service fplugin to use the new api --- bin/create_service.py | 54 +++++++++++++++++++++------------- persistence/server/models.py | 6 ++-- persistence/server/server.py | 2 +- server/api/modules/services.py | 18 ++++++++---- 4 files changed, 50 insertions(+), 30 deletions(-) diff --git a/bin/create_service.py b/bin/create_service.py index 53903922571..b78bba80641 100644 --- a/bin/create_service.py +++ b/bin/create_service.py @@ -14,7 +14,7 @@ def main(workspace='', args=None, parser=None): - parser.add_argument('interface_id', help='Interface ID') + parser.add_argument('host_id', help='Service Parent Host ID') parser.add_argument('name', help='Service Name') parser.add_argument('ports', help='Service ports, as a comma separated list') parser.add_argument('--protocol', help='Service protocol', default='tcp') @@ -27,23 +27,35 @@ def main(workspace='', args=None, parser=None): parsed_args = parser.parse_args(args) ports = filter(None, parsed_args.ports.split(',')) - - obj = factory.createModelObject(models.Service.class_signature, parsed_args.name, workspace, - protocol=parsed_args.protocol, - ports=ports, - status=parsed_args.status, - version=parsed_args.version, - description=parsed_args.description, - parent_id=parsed_args.interface_id - ) - - old = models.get_service(workspace, obj.getID()) - - if old is None: - if not parsed_args.dry_run: - models.create_service(workspace, obj) - else: - print "A service with ID %s already exists!" % obj.getID() - return 2, None - - return 0, obj.getID() + res_ids = [] #new service or old services ids affected by the command + for port in ports: + params = { + 'name': parsed_args.name, + 'port': port, + 'protocol': parsed_args.protocol, + 'host_id': parsed_args.host_id + } + + obj = factory.createModelObject(models.Service.class_signature, + parsed_args.name, + workspace, + protocol=parsed_args.protocol, + ports=ports, + status=parsed_args.status, + version=parsed_args.version, + description=parsed_args.description, + parent_id=parsed_args.host_id + ) + + old = models.get_service(workspace, **params) + + if old is None: + if not parsed_args.dry_run: + models.create_service(workspace, obj) + old = models.get_service(workspace, **params) + else: + print "A service with ID %s already exists!" % old.getID() + + res_ids.append(old.getID()) + + return 0, res_ids diff --git a/persistence/server/models.py b/persistence/server/models.py index 001524791ee..93ce4559a05 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -254,9 +254,9 @@ def get_services(workspace_name, **params): return _get_faraday_ready_services(workspace_name, services_dictionary) -def get_service(workspace_name, service_id): +def get_service(workspace_name, service_id=None, **params): """Return the Service of id service_id. None if not found.""" - return force_unique(get_services(workspace_name, id=service_id)) + return force_unique(get_services(workspace_name, object_id=service_id, **params)) def get_credentials(workspace_name, **params): @@ -362,7 +362,7 @@ def update_host(workspace_name, host, command_id): @_ignore_in_changes -def create_service(workspace_name, service, command_id): +def create_service(workspace_name, service, command_id=None): """Take a workspace_name and a service object and save it to the sever. Return the server's json response as a dictionary. """ diff --git a/persistence/server/server.py b/persistence/server/server.py index 5832ce86134..c0c0dced974 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -284,7 +284,7 @@ def _get_raw_vulns(workspace_name, **params): def _get_raw_services(workspace_name, **params): """Take a workspace_name and an arbitrary number of params and return a dictionary with the services table.""" - request_url = _create_server_get_url(workspace_name, 'services', params.get('id', None)) + request_url = _create_server_get_url(workspace_name, 'services', **params) return _get(request_url, **params) diff --git a/server/api/modules/services.py b/server/api/modules/services.py index 397b85b0a8e..0fc9e276c8f 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -1,15 +1,14 @@ # Faraday Penetration Test IDE # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information -import time -import flask from flask import Blueprint +from filteralchemy import FilterSet, operators from marshmallow import fields, post_load, ValidationError -from sqlalchemy.orm import joinedload from sqlalchemy.orm.exc import NoResultFound -from server.api.base import AutoSchema, ReadWriteWorkspacedView +from server.api.base import AutoSchema, ReadWriteWorkspacedView, FilterSetMeta, \ + FilterAlchemyMixin from server.models import Host, Service, Workspace from server.schemas import ( MetadataSchema, @@ -80,7 +79,15 @@ class Meta: 'metadata', 'summary', 'host_id') -class ServiceView(ReadWriteWorkspacedView): +class ServiceFilterSet(FilterSet): + class Meta(FilterSetMeta): + model = Service + fields = ('host_id', 'protocol', 'name', 'port') + default_operator = operators.Equal + operators = (operators.Equal,) + + +class ServiceView(FilterAlchemyMixin, ReadWriteWorkspacedView): route_base = 'services' model_class = Service schema_class = ServiceSchema @@ -88,6 +95,7 @@ class ServiceView(ReadWriteWorkspacedView): get_undefer = [Service.credentials_count, Service.vulnerability_count] get_joinedloads = [Service.credentials] unique_fields = [('port', 'protocol', 'host')] + filterset_class = ServiceFilterSet def _envelope_list(self, objects, pagination_metadata=None): services = [] From 7ef7bb8dfcdd9ee557858de319237abc82534e99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 16 Feb 2018 15:06:16 -0300 Subject: [PATCH 0891/1506] Fix and remove some TODOs --- server/api/modules/services.py | 6 +++++- server/api/modules/vulns.py | 9 +++++++-- server/models.py | 4 +--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/server/api/modules/services.py b/server/api/modules/services.py index 397b85b0a8e..7da18a304c1 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -42,7 +42,11 @@ class ServiceSchema(AutoSchema): type = fields.Function(lambda obj: 'Service', dump_only=True) def load_ports(self, value): - # TODO migration: handle empty list and not numeric value + if not isinstance(value, list): + raise ValidationError('ports must be a list') + if len(value) != 1: + raise ValidationError('ports must be a list with exactly one' + 'element') return str(value.pop()) @post_load diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 6fe5770330e..01738d42e34 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -116,8 +116,13 @@ class VulnerabilitySchema(AutoSchema): ]), dump_only=True) host = fields.Integer(dump_only=True, attribute='host_id') severity = SeverityField(required=True) - status = fields.Method(serialize='get_status', deserialize='load_status') # TODO: this breaks enum validation. - type = fields.Method(serialize='get_type', deserialize='load_type', required=True) + status = fields.Method( + serialize='get_status', + validate=OneOf(Vulnerability.STATUSES + ['opened']), + deserialize='load_status') + type = fields.Method(serialize='get_type', + deserialize='load_type', + required=True) obj_id = fields.String(dump_only=True, attribute='id') target = fields.String(dump_only=True, attribute='target_host_ip') metadata = SelfNestedField(CustomMetadataSchema()) diff --git a/server/models.py b/server/models.py index aa2fcf85cee..1c8b7909dcf 100644 --- a/server/models.py +++ b/server/models.py @@ -393,8 +393,7 @@ class VulnerabilityABC(Metadata): name = Column(Text, nullable=False) resolution = Column(Text, nullable=True) severity = Column(Enum(*SEVERITIES, name='vulnerability_severity'), nullable=False) - risk = Column(Float(3,1), nullable=True) - # TODO add evidence + risk = Column(Float(3, 1), nullable=True) impact_accountability = Column(Boolean, default=False) impact_availability = Column(Boolean, default=False) @@ -638,7 +637,6 @@ class Command(Metadata): workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship('Workspace', foreign_keys=[workspace_id]) - # TODO: add Tool relationship and report_attachment sum_created_vulnerabilities = _make_created_objects_sum('vulnerability') From 769738e5e8f26baf58d78d550dd9a73e630b665c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 16 Feb 2018 15:30:32 -0300 Subject: [PATCH 0892/1506] Update create_vulnweb fplugin to use new api --- bin/create_vulnweb.py | 20 ++++++++++++++++---- persistence/server/models.py | 6 +++--- persistence/server/server.py | 3 ++- server/api/modules/vulns.py | 4 ++-- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/bin/create_vulnweb.py b/bin/create_vulnweb.py index 60264b38f8b..6db7e33ef8a 100644 --- a/bin/create_vulnweb.py +++ b/bin/create_vulnweb.py @@ -44,6 +44,16 @@ def main(workspace='', args=None, parser=None): parsed_args = parser.parse_args(args) + params = { + 'name': parsed_args.name, + 'description': parsed_args.description, + 'service_id': parsed_args.service, + 'method': parsed_args.method, + 'parameter_name': parsed_args.params, + 'path': parsed_args.path, + 'website': parsed_args.website, + } + obj = factory.createModelObject(models.VulnWeb.class_signature, parsed_args.name, workspace, desc=parsed_args.description, ref=parsed_args.reference, @@ -61,16 +71,18 @@ def main(workspace='', args=None, parser=None): category=parsed_args.category, confirmed=(parsed_args.confirmed == 'true'), - parent_id=parsed_args.service + parent_id=parsed_args.service, + parent_type='Service' ) - old = models.get_web_vuln(workspace, obj.getID()) + old = models.get_web_vuln(workspace, **params) if old is None: if not parsed_args.dry_run: models.create_vuln_web(workspace, obj) + old = models.get_web_vuln(workspace, **params) else: - print "A web vulnerability with ID %s already exists!" % obj.getID() + print "A web vulnerability with ID %s already exists!" % old.getID() return 2, None - return 0, obj.getID() + return 0, old.getID() diff --git a/persistence/server/models.py b/persistence/server/models.py index 93ce4559a05..e2d6444668f 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -239,9 +239,9 @@ def get_web_vulns(workspace_name, **params): return _get_faraday_ready_vulns(workspace_name, vulns_web_dictionaries, vulns_type='vulns_web') -def get_web_vuln(workspace_name, vuln_id): +def get_web_vuln(workspace_name, vuln_id=None, **params): """Return the WebVuln of id vuln_id. None if not found.""" - return force_unique(get_web_vulns(workspace_name, id=vuln_id)) + return force_unique(get_web_vulns(workspace_name, object_id=vuln_id, **params)) def get_services(workspace_name, **params): @@ -401,7 +401,7 @@ def update_vuln(workspace_name, vuln, command_id=None): @_ignore_in_changes -def create_vuln_web(workspace_name, vuln_web, command_id): +def create_vuln_web(workspace_name, vuln_web, command_id=None): """Take a workspace_name and an vulnerabilityWeb object and save it to the sever. Return the server's json response as a dictionary. diff --git a/persistence/server/server.py b/persistence/server/server.py index c0c0dced974..fbf64603f43 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -277,7 +277,8 @@ def _get_raw_hosts(workspace_name, **params): def _get_raw_vulns(workspace_name, **params): """Take a workspace_name and an arbitrary number of params and return a dictionary with the vulns table.""" - request_url = _create_server_get_url(workspace_name, 'vulns', params.get('id', None)) + params = {key: value for key, value in params.items() if value} + request_url = _create_server_get_url(workspace_name, 'vulns', **params) return _get(request_url, **params) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index a3c7aafb90c..4fcdf04b561 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -288,12 +288,12 @@ class Meta(FilterSetMeta): "status", "website", "parameter_name", "query_string", "path", "data", "severity", "confirmed", "name", "request", "response", "parameters", "resolution", "method", "ease_of_resolution", - "description", "command_id", "target", "creator", + "description", "command_id", "target", "creator", "service_id", "easeofresolution") strict_fields = ( "severity", "confirmed", "method", "status", "easeofresolution", - "ease_of_resolution", + "ease_of_resolution", "service_id", ) default_operator = operators.ILike From 71b0e9b503ae047ca8e128a388c5a1bf57842691 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 16 Feb 2018 15:42:43 -0300 Subject: [PATCH 0893/1506] Update code to allow deleted requests --- persistence/server/server.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/persistence/server/server.py b/persistence/server/server.py index fbf64603f43..abd3e012299 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -138,8 +138,8 @@ def _create_server_put_url(workspace_name, obj_type, obj_id, command_id): return put_url -def _create_server_delete_url(workspace_name, object_id): - return _create_server_post_url(workspace_name, object_id) +def _create_server_delete_url(workspace_name, obj_type, object_id, command_id=None): + return _create_server_put_url(workspace_name, obj_type, object_id, command_id) # XXX: COUCH IT! def _create_couch_get_url(workspace_name, object_id): @@ -330,12 +330,12 @@ def _save_db_to_server(db_name, **params): post_url = _create_server_db_url(db_name) return _post(post_url, expected_response=201, **params) -# XXX: SEMI COUCH IT! -def _delete_from_couch(workspace_name, faraday_object_id): - delete_url = _create_server_delete_url(workspace_name, faraday_object_id) + +def _delete_from_server(workspace_name, faraday_object_type, faraday_object_id): + delete_url = _create_server_delete_url(workspace_name, faraday_object_type, faraday_object_id) return _delete(delete_url) -# XXX: COUCH IT! + @_add_session_cookies def _couch_changes(workspace_name, **params): return CouchChangesStream(workspace_name, @@ -1454,33 +1454,36 @@ def create_workspace(workspace_name, description, start_date, finish_date, duration=duration, type="Workspace") + def delete_host(workspace_name, host_id): """Delete host of id host_id from the database.""" - return _delete_from_couch(workspace_name, host_id) + return _delete_from_server(workspace_name, 'Host', host_id) -def delete_interface(workspace_name, interface_id): - """Delete interface of id interface_id from the database.""" - return _delete_from_couch(workspace_name, interface_id) def delete_service(workspace_name, service_id): """Delete service of id service_id from the database.""" - return _delete_from_couch(workspace_name, service_id) + return _delete_from_server(workspace_name, 'Service', service_id) + def delete_vuln(workspace_name, vuln_id): """Delete vuln of id vuln_id from the database.""" - return _delete_from_couch(workspace_name, vuln_id) + return _delete_from_server(workspace_name, 'Vulnerability', vuln_id) + def delete_note(workspace_name, note_id): """Delete note of id note_id from the database.""" - return _delete_from_couch(workspace_name, note_id) + return _delete_from_server(workspace_name, note_id) + def delete_credential(workspace_name, credential_id): """Delete credential of id credential_id from the database.""" - return _delete_from_couch(workspace_name, credential_id) + return _delete_from_server(workspace_name, 'Credential', credential_id) + def delete_command(workspace_name, command_id): """Delete command of id command_id from the database.""" - return _delete_from_couch(workspace_name, command_id) + return _delete_from_server(workspace_name, 'Command', command_id) + def delete_workspace(workspace_name): """Delete the couch database of id workspace_name""" From f9b3e022d9892f82ce39a8a1fdbe3177811519d8 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 16 Feb 2018 16:14:26 -0300 Subject: [PATCH 0894/1506] Fix tests and remove redudant code. parameters are passed to the requests get --- persistence/server/models.py | 2 +- persistence/server/server.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index e2d6444668f..5528230f39e 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -226,7 +226,7 @@ def get_vulns(workspace_name, **params): def get_vuln(workspace_name, vuln_id): """Return the Vuln of id vuln_id. None if not found.""" - return force_unique(get_vulns(workspace_name, id=vuln_id)) + return force_unique(get_vulns(workspace_name, object_id=vuln_id)) def get_web_vulns(workspace_name, **params): diff --git a/persistence/server/server.py b/persistence/server/server.py index abd3e012299..6160a1d9734 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -106,13 +106,12 @@ def _create_server_get_url(workspace_name, object_name=None, object_id=None, **p Return the get_url as a string. """ - object_name = "/{0}".format(object_name) if object_name else "" - object_name += "/{0}/".format(object_id) if object_id else "" + get_url = "/{0}".format(object_name) if object_name else "" + get_url += "/{0}/".format(object_id) if object_id else "" get_url = '{0}/ws/{1}{2}'.format(_create_server_api_url(), workspace_name, - object_name) - if params: - get_url = '{0}?{1}'.format(get_url, urllib.urlencode(params)) + get_url) + return get_url From 93cc3710c703a1c0e030d5f803d3ff541d1c64e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 19 Feb 2018 12:29:49 -0300 Subject: [PATCH 0895/1506] Fill severity when creating vulns from template --- server/www/scripts/statusReport/controllers/modalNew.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/www/scripts/statusReport/controllers/modalNew.js b/server/www/scripts/statusReport/controllers/modalNew.js index 236bfe01198..41d9f4567c4 100644 --- a/server/www/scripts/statusReport/controllers/modalNew.js +++ b/server/www/scripts/statusReport/controllers/modalNew.js @@ -215,6 +215,8 @@ angular.module('faradayApp') for (var key in item) { if(key != "refs" && key != "policyviolations" && vm.data.hasOwnProperty(key)) { vm.data[key] = item[key]; + }else if (key === 'exploitation'){ + vm.data['severity'] = item['exploitation']; } } From 9f6b32d62719ae8268f84da9a4cf95da91015383 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 19 Feb 2018 12:48:33 -0300 Subject: [PATCH 0896/1506] Parameter unification --- bin/create_note.py | 5 ++++- bin/{getAllIpsInterfaces.py => getAllIps.py} | 0 2 files changed, 4 insertions(+), 1 deletion(-) rename bin/{getAllIpsInterfaces.py => getAllIps.py} (100%) diff --git a/bin/create_note.py b/bin/create_note.py index 737c9b1855b..04fc441c5c6 100644 --- a/bin/create_note.py +++ b/bin/create_note.py @@ -19,7 +19,10 @@ def main(workspace='', args=None, parser=None): logger.warn('Create note will create a comment. fplugin name will be changed to create_comment') parser.add_argument('parent', help='Parent ID') - parser.add_argument('parent_type', help='Parent Type') + parser.add_argument('--parent_type', + help='Vulnerability severity', + choices=['Host', 'Service'], + default='unclassified') parser.add_argument('name', help='Note name') parser.add_argument('text', help='Note content') diff --git a/bin/getAllIpsInterfaces.py b/bin/getAllIps.py similarity index 100% rename from bin/getAllIpsInterfaces.py rename to bin/getAllIps.py From 5d8f59073dd8c74f4267bb89d860eca45277175e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 19 Feb 2018 12:48:49 -0300 Subject: [PATCH 0897/1506] Fix a bug when creating a service --- bin/create_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/create_service.py b/bin/create_service.py index b78bba80641..774ee0f3f89 100644 --- a/bin/create_service.py +++ b/bin/create_service.py @@ -40,7 +40,7 @@ def main(workspace='', args=None, parser=None): parsed_args.name, workspace, protocol=parsed_args.protocol, - ports=ports, + ports=[port], status=parsed_args.status, version=parsed_args.version, description=parsed_args.description, From d7e2be6f1855a4d3929774e110e48c31c48f7854 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 19 Feb 2018 12:49:02 -0300 Subject: [PATCH 0898/1506] Remove interfaces from old command --- bin/getAllIps.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/bin/getAllIps.py b/bin/getAllIps.py index e05cc35b27b..2072e08f051 100644 --- a/bin/getAllIps.py +++ b/bin/getAllIps.py @@ -6,7 +6,7 @@ Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) See the file 'doc/LICENSE' for the license information ''' - +import re from persistence.server import models __description__ = "Get all scanned interfaces" @@ -14,7 +14,14 @@ def main(workspace='', args=None, parser=None): - for interface in models.get_interfaces(workspace): - print(interface.ipv4['address']) + ip_regex = re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$") + not_matching_count = 0 + for host in models.get_hosts(workspace): + if re.match(ip_regex, host.ip): + print(host.ip) + else: + not_matching_count += 1 + if not_matching_count: + print('Hosts that has invalid ip addresses {0}'.format(not_matching_count)) return 0, None From 0807ac01ae30a1825a7906ae94ba4344a0cff786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 19 Feb 2018 15:11:42 -0300 Subject: [PATCH 0899/1506] Allow filtering vulns by service name --- server/api/modules/vulns.py | 19 ++++++++++++++---- server/models.py | 6 ++++-- test_cases/test_api_vulnerability.py | 29 ++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 2d14937f466..790d10ef764 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -13,7 +13,7 @@ from flask_classful import route from marshmallow import Schema, fields, post_load, ValidationError from marshmallow.validate import OneOf -from sqlalchemy.orm import joinedload, selectin_polymorphic, undefer +from sqlalchemy.orm import aliased, joinedload, selectin_polymorphic, undefer from sqlalchemy.orm.exc import NoResultFound from depot.manager import DepotManager @@ -283,14 +283,24 @@ def filter(self, query, model, attr, value): return query.filter(model.creator_command_tool == value) +class ServiceFilter(Filter): + def filter(self, query, model, attr, value): + alias = aliased(Service, name='service_filter') + return query.join( + alias, + alias.id == model.__table__.c.service_id).filter( + alias.name == value + ) + + class VulnerabilityFilterSet(FilterSet): class Meta(FilterSetMeta): model = VulnerabilityWeb # It has all the fields # TODO migration: Check if we should add fields owner, - # command, impact, service, issuetracker, tags, date, - # host, easeofresolution, evidence, policy violations, hostnames + # command, impact, issuetracker, tags, date, host + # evidence, policy violations, hostnames fields = ( - "status", "website", "pname", "query", "path", + "status", "website", "pname", "query", "path", "service", "data", "severity", "confirmed", "name", "request", "response", "parameters", "params", "resolution", "ease_of_resolution", "description", "command_id", "target", "creator", "method", @@ -310,6 +320,7 @@ class Meta(FilterSetMeta): type = TypeFilter(fields.Str(validate=[OneOf(['Vulnerability', 'VulnerabilityWeb'])])) creator = CreatorFilter(fields.Str()) + service = ServiceFilter(fields.Str()) severity = Filter(SeverityField()) easeofresolution = Filter(fields.String( attribute='ease_of_resolution', diff --git a/server/models.py b/server/models.py index 1c8b7909dcf..d77940cfc80 100644 --- a/server/models.py +++ b/server/models.py @@ -751,8 +751,10 @@ class VulnerabilityGeneric(VulnerabilityABC): ) target_host_ip = column_property( case([ - (text('host_id IS NOT null'), _host_vuln_query.as_scalar()), - (text('service_id IS NOT null'), _service_vuln_query.as_scalar()) + (text('vulnerability.host_id IS NOT null'), + _host_vuln_query.as_scalar()), + (text('vulnerability.service_id IS NOT null'), + _service_vuln_query.as_scalar()) ]), deferred=True ) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index a1369f95def..6a19281af3b 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1369,3 +1369,32 @@ def test_creator_filter(workspace, session, 'creator', 'metasploit') assert {v.id for v in filtered} == {v.id for v in vulns} + + +def test_service_filter(workspace, session, host, service_factory, + vulnerability_factory, vulnerability_web_factory): + filter_ = VulnerabilityFilterSet().filters['service'] + + vulnerability_factory.create_batch(5, host=host, service=None, + workspace=workspace) + other_service = service_factory.create(name='ftp', workspace=workspace) + vulnerability_factory.create_batch(10, host=None, service=other_service, + workspace=workspace) + vulnerability_web_factory.create_batch(10, service=other_service, + workspace=workspace) + + service = service_factory.create(name='http', workspace=workspace) + vulns = [] + vulns = vulnerability_factory.create_batch(10, host=None, service=service, + workspace=workspace) + vulns += vulnerability_web_factory.create_batch(10, service=service, + workspace=workspace) + session.commit() + + filtered = filter_.filter(VulnerabilityGeneric.query, + VulnerabilityGeneric, + 'service', + 'http') + assert all(v.service and v.service.name == 'http' + for v in filtered) + assert {v.id for v in filtered} == {v.id for v in vulns} From e41cd19485f4b18700c1d463f79f07a36bbd9646 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 19 Feb 2018 15:29:47 -0300 Subject: [PATCH 0900/1506] Fix fplugin import_csv --- bin/import_csv.py | 100 +++++++++++++++++------------------ persistence/server/models.py | 11 +++- persistence/server/utils.py | 4 +- 3 files changed, 61 insertions(+), 54 deletions(-) diff --git a/bin/import_csv.py b/bin/import_csv.py index 8719b19bde2..1717891f729 100644 --- a/bin/import_csv.py +++ b/bin/import_csv.py @@ -21,12 +21,11 @@ def parse_register(register): host = parse_host(register) - interface = parse_interface(register) service = parse_service(register) vulnerability = parse_vulnerability(register) vulnerability_web = parse_vulnerability_web(register) - return host, interface, service, vulnerability, vulnerability_web + return host, service, vulnerability, vulnerability_web def transform_dict_to_object(columns, register): @@ -43,6 +42,9 @@ def transform_dict_to_object(columns, register): # Default data value = {val : ""} + if val == "service_id": + value["parent"] = register["service_id"] + if val == "owned" or val == "confirmed": value[val] = False @@ -61,6 +63,9 @@ def transform_dict_to_object(columns, register): # Copy data to new object if key in register: + if val == "host_name": + value[val] = register['interface_ipv4_address'] or register['interface_ipv6_address'] + if val == "ports": value[val] = [register[key]] @@ -116,31 +121,6 @@ def parse_host(register): return host -def parse_interface(register): - - columns = { - "interface_name" : "name", - "interface_description" : "description", - "interface_hostnames" : "hostnames", #list - "interface_mac" : "mac", - "interface_network_segment" : "network_segment", - "interface_ipv4_address" : "ipv4_address", - "interface_ipv4_gateway" : "ipv4_gateway", - "interface_ipv4_mask" : "ipv4_mask", - "interface_ipv4_dns" : "ipv4_dns", - "interface_ipv6_address" : "ipv6_address", - "interface_ipv6_gateway" : "ipv6_gateway", - "interface_ipv6_prefix" : "ipv6_prefix", - "interface_ipv6_dns" : "ipv6_dns" - } - - obj = transform_dict_to_object(columns, register) - if obj is None: - return None - interface = models.Interface(obj, WORKSPACE) - return interface - - def parse_service(register): columns = { @@ -232,45 +212,53 @@ def main(workspace="", args=None, parser=None): csv_reader = csv.DictReader(file_csv, delimiter=",", quotechar='"') for register in csv_reader: - host, interface, service, vulnerability, vulnerability_web = parse_register(register) + host, service, vulnerability, vulnerability_web = parse_register(register) # Set all IDs and create objects if host is not None: - - host.setID(None) - if not models.get_host(WORKSPACE, host.getID()): + old_host = models.get_host(WORKSPACE, ip=host.getName()) + if not old_host: counter += 1 print "New host: " + host.getName() models.create_host(WORKSPACE, host) - - if interface is not None: - - interface.setID(host.getID()) - if not models.get_interface(WORKSPACE, interface.getID()): - - counter += 1 - print "New interface: " + interface.getName() - models.create_interface(WORKSPACE, interface) + host = models.get_host(WORKSPACE, ip=host.getName()) if service is not None: - - service.setID(interface.getID()) - if not models.get_service(WORKSPACE, service.getID()): + service.setParent(host.getID()) + service_params = { + 'name': service.getName(), + 'port': service.getPorts()[0], + 'protocol': service.getProtocol(), + 'host_id': service.getParent() + } + old_service = models.get_service(WORKSPACE, **service_params) + if not old_service: counter += 1 print "New service: " + service.getName() models.create_service(WORKSPACE, service) + service = models.get_service(WORKSPACE, **service_params) # Check if Service exist, then create the vuln with parent Service. # If not exist the Service, create the vuln with parent Host. if vulnerability is not None: - - if service is None: - vulnerability.setID(host.getID()) - else: - vulnerability.setID(service.getID()) - if not models.get_vuln(WORKSPACE, vulnerability.getID()): + if host and not service: + parent_type = 'Host' + parent_id = host.getID() + if host and service: + parent_type = 'Service' + parent_id = service.getID() + vulnerability.setParent(parent_id) + vulnerability.setParentType(parent_type) + + vuln_params = { + 'name': vulnerability.getName(), + 'description': vulnerability.getDescription(), + 'parent_type': parent_type, + 'parent': parent_id, + } + if not models.get_vuln(WORKSPACE, **vuln_params): counter += 1 print "New vulnerability: " + vulnerability.getName() @@ -278,8 +266,18 @@ def main(workspace="", args=None, parser=None): elif vulnerability_web is not None: - vulnerability_web.setID(service.getID()) - if not models.get_web_vuln(WORKSPACE, vulnerability_web.getID()): + vuln_web_params = { + 'name': vulnerability_web.getName(), + 'description': vulnerability_web.getDescription(), + 'parent': service.getID(), + 'parent_type': 'Service', + 'method': vulnerability_web.getMethod(), + 'parameter_name': vulnerability_web.getParams(), + 'path': vulnerability_web.getPath(), + 'website': vulnerability_web.getWebsite(), + } + vulnerability_web.setParent(service.getID()) + if not models.get_web_vuln(WORKSPACE, **vuln_web_params): counter += 1 print "New web vulnerability: " + vulnerability_web.getName() diff --git a/persistence/server/models.py b/persistence/server/models.py index 5528230f39e..bfb06958dcc 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -224,9 +224,9 @@ def get_vulns(workspace_name, **params): return _get_faraday_ready_vulns(workspace_name, vulns_dictionaries, vulns_type='vulns') -def get_vuln(workspace_name, vuln_id): +def get_vuln(workspace_name, vuln_id=None, **params): """Return the Vuln of id vuln_id. None if not found.""" - return force_unique(get_vulns(workspace_name, object_id=vuln_id)) + return force_unique(get_vulns(workspace_name, object_id=vuln_id, **params)) def get_web_vulns(workspace_name, **params): @@ -728,6 +728,12 @@ def getParentType(self): def getParent(self): return self.parent_id + def setParent(self, parent_id): + self.parent_id = parent_id + + def setParentType(self, parent_type): + self.parent_type = parent_type + def setID(self, id): self.id = id self.id_available.set() @@ -1113,6 +1119,7 @@ def __init__(self, vuln_web, workspace_name): self.tags = vuln_web.get('tags', list()) self.target = vuln_web.get('target') self.policyviolations = vuln_web.get('policyviolations', list()) + self.parent_type = 'Service' @staticmethod def publicattrsrefs(): diff --git a/persistence/server/utils.py b/persistence/server/utils.py index 37a2edd8d3b..0fd1fbc883e 100644 --- a/persistence/server/utils.py +++ b/persistence/server/utils.py @@ -99,7 +99,9 @@ def get_vuln_web_properties(vuln_web): 'path': vuln_web.getPath(), 'pname': vuln_web.getPname(), 'query': vuln_web.getQuery(), - 'status': vuln_web.getStatus() + 'status': vuln_web.getStatus(), + 'parent': vuln_web.getParent(), + 'parent_type': vuln_web.getParentType(), } vuln_web_dict.update(get_object_properties(vuln_web)) vuln_web_dict.update(get_vuln_properties(vuln_web)) From a532ff6656716d6f14558b303462bd7516159305 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 19 Feb 2018 16:21:09 -0300 Subject: [PATCH 0901/1506] Allow sorting vulns by service This works differently than with CouchDB. It sorts directly by service id instead of by service summary --- server/api/base.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/api/base.py b/server/api/base.py index 7959d3b9026..215a2cdb9d5 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -340,7 +340,11 @@ def _get_order_field(self, **kwargs): # It could be something like fields.Method raise InvalidUsage("Field not in the DB: %s" % order_field) - field = getattr(model_class, order_field) + if hasattr(model_class, order_field + '_id'): + # Ugly hack to allow sorting by a parent + field = getattr(model_class, order_field + '_id') + else: + field = getattr(model_class, order_field) sort_dir = flask.request.args.get(self.sort_direction_paremeter_name, self.default_sort_direction) if sort_dir not in ('asc', 'desc'): From 53cd526bd6ef8b1398d525ef27b0293ad09ed235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 19 Feb 2018 17:56:08 -0300 Subject: [PATCH 0902/1506] Fix remember sorting preferences of the policy violations field. It wasn't properly stored in the cookie, so refreshing the page made it dissapear --- server/www/scripts/statusReport/controllers/statusReport.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 5c1a8637118..1c0fce217e5 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -406,7 +406,11 @@ angular.module('faradayApp') visible: $scope.columns["creator"] }); $scope.gridOptions.columnDefs.push({ name : 'policyviolations', - displayName : "policy violations", + // The following line breaks the remembering of the field (i.e. + // setting it in the SRcolumns cookie), so it is better to + // leave it commented (or to debug the problem, which I don't + // want to) + // displayName : "policy violations", cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/policyviolationscolumn.html', headerCellTemplate: header, width: '100', From eee3d63f7276c77aa47201eef2cbb3c1f11bf1bc Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Mon, 19 Feb 2018 20:23:29 -0300 Subject: [PATCH 0903/1506] Fix vuln amount GTK --- gui/gtk/dialogs.py | 5 ++--- persistence/server/models.py | 8 ++------ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/gui/gtk/dialogs.py b/gui/gtk/dialogs.py index bc48bcdf266..0e14ef45e95 100644 --- a/gui/gtk/dialogs.py +++ b/gui/gtk/dialogs.py @@ -721,7 +721,6 @@ def __init__(self, parent, active_ws_name, host): """Creates a window with the information about a given hosts. The parent is needed so the window can set transient for """ - window_title = "Host " + host.name + " information" Gtk.Window.__init__(self, title=window_title) @@ -880,11 +879,11 @@ def create_model(self, host): # only the ID and the name are needed, but i still need to 'fill' # the other columns with dummy info - display_str = host.getName() + " (" + str(len(host.getVulns())) + ")" + display_str = host.getName() + " (" + str(host.getVulnAmount()) + ")" owned_status = ("Yes" if host.isOwned() else "No") model.append(None, [str(host.getID()), host.getName(), host.getOS(), owned_status, - str(len(host.getVulns())), "", + str(host.getVulnAmount()), "", "", "", "", "", "", "", display_str]) diff --git a/persistence/server/models.py b/persistence/server/models.py index 5528230f39e..12dccf87e50 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -869,12 +869,8 @@ def getDefaultGateway(self): return self.default_gateway def getVulns(self): - return get_all_vulns(self._workspace_name, hostid=self._server_id) - # def getInterface(self, interface_couch_id): - # service = get_interfaces(self._workspace_name, couchid=interface_couch_id) - # return service[0] - # def getAllInterfaces(self): - # return get_interfaces(self._workspace_name, host=self._server_id) + return get_all_vulns(self._workspace_name, target=self._server_id) + def getServices(self): return get_services(self._workspace_name, hostid=self._server_id) From fda2cae3dd9d9b7c53342526d1055a922439141f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 20 Feb 2018 15:05:49 -0300 Subject: [PATCH 0904/1506] Fix race condition on association proxy --- server/models.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/server/models.py b/server/models.py index 7fbf0978acb..d53d92df286 100644 --- a/server/models.py +++ b/server/models.py @@ -18,6 +18,7 @@ Text, UniqueConstraint, event) +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import relationship, undefer from sqlalchemy.sql import select, text, table from sqlalchemy.sql.expression import asc, case, join @@ -53,6 +54,7 @@ 'source_code', 'comment', ] +UNIQUE_VIOLATION = '23505' class SQLAlchemy(OriginalSQLAlchemy): @@ -438,8 +440,28 @@ class takes different arguments than the hardcoded def _create(self, value): parent_instance = self.lazy_collection.ref() - return self.creator(value, parent_instance) - + session = db.session + conflict_objs = session.new + try: + yield self.creator(value, parent_instance) + except IntegrityError as ex: + if ex.orig.pgcode == UNIQUE_VIOLATION: + # unique constraint failed at database + # other process/thread won us on the commit + # we need to fetch already created objs. + session.rollback() + conflict_obj_names = [obj.name for obj in conflict_objs if obj.name != value] + for conflict_obj_name in conflict_obj_names: + conclict_obj = session.query(Reference).filter_by(name=conflict_obj_name).first() + if not conclict_obj: + raise Exception('This should not happend. AssocProxy could not find a conflict obj.') + self.col.add(conclict_obj) + yield self.creator(value, parent_instance) + + def add(self, value): + if value not in self: + for new_value in self._create(value): + self.col.add(new_value) def _build_associationproxy_creator(model_class_name): def creator(name, vulnerability): From 8401fde4803baa397bef458fa5be0fbda7749f8f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 20 Feb 2018 15:07:39 -0300 Subject: [PATCH 0905/1506] Reraise the exception if pgcode != UNIQUE_VIOLATION --- server/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/models.py b/server/models.py index d53d92df286..9147b567ad1 100644 --- a/server/models.py +++ b/server/models.py @@ -457,6 +457,8 @@ def _create(self, value): raise Exception('This should not happend. AssocProxy could not find a conflict obj.') self.col.add(conclict_obj) yield self.creator(value, parent_instance) + else: + raise def add(self, value): if value not in self: From 3ef015ca0ad2229c7d0987dcbeddfa488581d12a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 20 Feb 2018 15:47:17 -0300 Subject: [PATCH 0906/1506] Properly exit the client when a plugin has been run The gtk client didn't exit if a plugin had been run in that session, even if it already finished --- plugins/plugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/plugin.py b/plugins/plugin.py index 8dc98ca138e..a94fb2f3187 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -369,6 +369,7 @@ def __init__(self, plugin_instance, output_queue, isReport=False): self.output_queue = output_queue self.plugin = plugin_instance self.isReport = isReport + self.setDaemon(True) def run(self): proc_name = self.name From a75811a8f959b5e1c9cfc30d9f5eb714e8b6fc37 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 20 Feb 2018 16:53:01 -0300 Subject: [PATCH 0907/1506] Creator tool should be the first tool, not the one that updated the command --- test_cases/test_api_vulnerability.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index cb73db0f17c..f576a06b93b 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1077,7 +1077,7 @@ def test_vulnerability_metadata(self, session, test_client, workspace): expected_metadata = { u'command_id': command.id, u'create_time': int(time.mktime(vuln.create_date.timetuple()) * 1000), - u'creator': update_command.tool, + u'creator': command.tool, u'owner': owner.username, u'update_action': 0, u'update_controller_action': u'', From f5ac29deb7f645d22d28141d54c2012664a576de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 20 Feb 2018 17:18:55 -0300 Subject: [PATCH 0908/1506] Fix error creating services with invalid status --- server/api/modules/services.py | 3 ++- test_cases/test_api_services.py | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/server/api/modules/services.py b/server/api/modules/services.py index 9d09fcb072f..e2af8c61c88 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -5,6 +5,7 @@ from flask import Blueprint from filteralchemy import FilterSet, operators from marshmallow import fields, post_load, ValidationError +from marshmallow.validate import OneOf from sqlalchemy.orm.exc import NoResultFound from server.api.base import AutoSchema, ReadWriteWorkspacedView, FilterSetMeta, \ @@ -32,7 +33,7 @@ class ServiceSchema(AutoSchema): fields.Method(deserialize='load_ports'), required=True, attribute='port') - status = fields.String(default='open') + status = fields.String(default='open', validate=OneOf(Service.STATUSES)) parent = fields.Integer(attribute='host_id') # parent is not required for updates host_id = fields.Integer(attribute='host_id', dump_only=True) vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index f1199e27648..7497d3a60be 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -70,6 +70,22 @@ def test_create_service(self, test_client, host, session): assert service.port == 21 assert service.host is host + def test_create_fails_with_invalid_status(self, test_client, + host, session): + session.commit() + data = { + "name": "ftp", + "description": "test. test", + "owned": False, + "ports": [21], + "protocol": "tcp", + "status": "asdasdasd", + "parent": host.id + } + res = test_client.post(self.url(), data=data) + assert res.status_code == 400 + assert 'Not a valid choice' in res.data + def test_create_fails_with_host_of_other_workspace(self, test_client, host, session, second_workspace): @@ -231,4 +247,4 @@ def test_create_service_without_ost(self, test_client, host, session): } res = test_client.post(self.url(), data=data) assert res.status_code == 400 - assert res.json['messages']['_schema'] == res.json['messages']['_schema'] \ No newline at end of file + assert res.json['messages']['_schema'] == res.json['messages']['_schema'] From 58d729b501db760e22aa3f34186ecf5703fddb74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 20 Feb 2018 18:06:16 -0300 Subject: [PATCH 0909/1506] Remove not useful failing test Now it is possible to sort vuln by service id --- test_cases/test_api_vulnerability.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 6a19281af3b..ae31f9eaed5 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -694,11 +694,6 @@ def test_sort_by_method(self, session, test_client, second_workspace, assert ''.join(v['method'] for v in res.json['data'] if v['method']) == 'abcdefghij'[::-1] - def test_cant_sort_by_service(self, test_client): - res = test_client.get(self.url() + '?sort=service&sort_dir=asc') - assert res.status_code == 400 - assert "service doesn't support sorting" in res.data - def test_create_vuln_with_evidence(self, host_with_hostnames, test_client, session): session.commit() # flush host_with_hostnames From be7e2d6a832c0cf3d00ca5f5db6637d2420a74a0 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 20 Feb 2018 18:46:53 -0300 Subject: [PATCH 0910/1506] Test jenkins notification --- Jenkinsfile | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b585cd80472..41c8e32fcb9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -56,10 +56,11 @@ node (label: "master"){ } finally { junit "**/xunit.xml" - + notifyBuild(currentBuild.result) if (testsError) { throw testsError } + } } @@ -81,10 +82,37 @@ node (label: "master"){ } finally { junit "**/xunit-postgres.xml" - + notifyBuild(currentBuild.result) if (testsError) { throw testsError } + } } } + +def notifyBuild(String buildStatus = 'STARTED') { + // build status of null means successful + buildStatus = buildStatus ?: 'SUCCESSFUL' + + // Default values + def colorName = 'RED' + def colorCode = '#FF0000' + def subject = "${buildStatus}: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'" + def summary = "${subject} (${env.BUILD_URL})" + + // Override default values based on build status + if (buildStatus == 'STARTED') { + color = 'YELLOW' + colorCode = '#FFFF00' + } else if (buildStatus == 'SUCCESSFUL') { + color = 'GREEN' + colorCode = '#00FF00' + } else { + color = 'RED' + colorCode = '#FF0000' + } + + // Send notifications + slackSend (color: colorCode, message: summary) +} \ No newline at end of file From 27d0836c5e66f03d2f62ccae2cd51aaf3d868af9 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Tue, 20 Feb 2018 20:18:41 -0300 Subject: [PATCH 0911/1506] Load services in GTK --- gui/gtk/dialogs.py | 64 +++++++++------------------- persistence/server/changes_stream.py | 1 - persistence/server/models.py | 10 ++++- 3 files changed, 30 insertions(+), 45 deletions(-) diff --git a/gui/gtk/dialogs.py b/gui/gtk/dialogs.py index 0e14ef45e95..ae938bf7bd3 100644 --- a/gui/gtk/dialogs.py +++ b/gui/gtk/dialogs.py @@ -881,7 +881,7 @@ def create_model(self, host): display_str = host.getName() + " (" + str(host.getVulnAmount()) + ")" owned_status = ("Yes" if host.isOwned() else "No") - model.append(None, [str(host.getID()), host.getName(), + host_position = model.append(None, [str(host.getID()), host.getName(), host.getOS(), owned_status, str(host.getVulnAmount()), "", "", "", "", "", "", "", @@ -895,6 +895,24 @@ def lst_to_str(lst): """Convenient function to avoid this long line everywhere""" return ', '.join([str(word) for word in lst if word]) + def add_service_to_host_in_model(service, model): + """Append a service to an host in the given + model. Return None. Modifies the model""" + display_str = service.getName() + " (" + str(len(service.getVulns())) + ")" + model.append(host_position, [str(service.getID()), + service.getName(), + service.getDescription(), + service.getProtocol(), + service.getStatus(), + lst_to_str(service.getPorts()), + service.getVersion(), + "Yes" if service.isOwned() else "No", + "", "", "", "", display_str]) + + services = host.getServices() + for service in services: + add_service_to_host_in_model(service, model) + return model @scrollable(width=250) @@ -1089,20 +1107,8 @@ def safe_wrapper(*args, **kwargs): object_id = selected_object[0] object_ = None - # TODO: fix this code to get services removing interface logic - # if object_type == 'Interface': - # # an interface is a direct child of a host - # object_ = safely(self.host.getInterface)(object_id) - # elif object_type == 'Service': - # # a service is a grand-child of a host, so we should look - # # for its parent interface and ask her about the child - # parent_interface_iter = selected_object.get_parent() - # parent_interface_id = parent_interface_iter[0] - # parent_interface = safely(self.host.getInterface)(parent_interface_id) - # if parent_interface: - # object_ = safely(parent_interface.getService)(object_id) - # else: - # object_ = None + if object_type == 'Service': + object_ = safely(self.host.getService)(object_id) return object_ @@ -1269,34 +1275,6 @@ def save(self, button, keeper): dialog.destroy() self._next_conflict_or_close() - # TODO: refactor or remove this code - # def case_for_interfaces(self, model, n): - # """The custom case for the interfaces. Plays a little - # with the information in the given model to create a solution acceptable - # by resolveConflict. n is the right or left view, should be - # either 1 or 2 as integers""" - # solution = {} - # solution["ipv4"] = {} - # solution["ipv6"] = {} - # for row in model: - # prop_name = row[0].lower() - # if prop_name.startswith("ipv4"): - # prop_name = prop_name.split(" ")[1] - # if not prop_name.startswith("dns"): - # solution["ipv4"][prop_name] = self.uncook(row[n], row[4]) - # elif prop_name.startswith("dns"): - # solution["ipv4"]["DNS"] = self.uncook(row[n], row[4]) - - # elif prop_name.startswith("ipv6"): - # prop_name = prop_name.split(" ")[1] - # if not prop_name.startswith("dns"): - # solution["ipv6"][prop_name] = self.uncook(row[n], row[4]) - # elif prop_name.startswith("dns"): - # solution["ipv6"]["DNS"] = self.uncook(row[n], row[4]) - # else: - # solution[prop_name] = self.uncook(row[n], row[4]) - # return solution - def on_quit(self, button): """Exits the window""" self.destroy() diff --git a/persistence/server/changes_stream.py b/persistence/server/changes_stream.py index 55a3bb15582..d50e107db71 100644 --- a/persistence/server/changes_stream.py +++ b/persistence/server/changes_stream.py @@ -74,7 +74,6 @@ def __init__(self, workspace_name, server_url, **params): self.changes_queue = Queue() self.workspace_name = workspace_name self._response = None - websocket.enableTrace(True) self.ws = websocket.WebSocketApp( "ws://{0}:9000".format(self._base_url), on_message=self.on_message, diff --git a/persistence/server/models.py b/persistence/server/models.py index 3745650b93a..2e43462f004 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -250,7 +250,7 @@ def get_services(workspace_name, **params): Return a list of Services objects """ - services_dictionary = server.get_services(workspace_name, **params) + services_dictionary = server.get_services(workspace_name, **params)[0] return _get_faraday_ready_services(workspace_name, services_dictionary) @@ -878,8 +878,16 @@ def getVulns(self): return get_all_vulns(self._workspace_name, target=self._server_id) def getServices(self): + """ + Get all services of that host + """ return get_services(self._workspace_name, hostid=self._server_id) + def getService(self): + """ + Get a specific service id + """ + return get_service(self._workspace_name, hostid=self._server_id) class Service(ModelBase): """A simple Service class. Should implement all the methods of the From 4b870f626f5f14181b9b0031d81c76c70a1d8205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 21 Feb 2018 13:01:58 -0300 Subject: [PATCH 0912/1506] Fix typo in error when invalid config found --- faraday.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/faraday.py b/faraday.py index b73d9bbaf47..96e7c11d13a 100755 --- a/faraday.py +++ b/faraday.py @@ -498,7 +498,7 @@ def checkServerUrl(): """) sys.exit(-1) except requests.exceptions.MissingSchema as ex: - print("Check ~/.faraday.config/user.xml server url, the following error was found: {0} ".format(ex)) + print("Check ~/.faraday/config/user.xml server url, the following error was found: {0} ".format(ex)) def checkVersion(): From 085a25568cb475c6f76b6e34f80719fdcc5b7092 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 21 Feb 2018 15:47:58 -0300 Subject: [PATCH 0913/1506] Load services in GTK sidebar, fix #4555 --- persistence/server/models.py | 5 ++++- persistence/server/server.py | 10 ++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index 2e43462f004..f3993892b60 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -250,7 +250,10 @@ def get_services(workspace_name, **params): Return a list of Services objects """ - services_dictionary = server.get_services(workspace_name, **params)[0] + services_dictionary = server.get_services(workspace_name, **params) + # List inside of list, use the inside list... + if type(services_dictionary[0]) == list: + services_dictionary = services_dictionary[0] return _get_faraday_ready_services(workspace_name, services_dictionary) diff --git a/persistence/server/server.py b/persistence/server/server.py index 6160a1d9734..c985a0fc5e7 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -283,8 +283,14 @@ def _get_raw_vulns(workspace_name, **params): def _get_raw_services(workspace_name, **params): """Take a workspace_name and an arbitrary number of params and return - a dictionary with the services table.""" - request_url = _create_server_get_url(workspace_name, 'services', **params) + a dictionary with the services table. + If you provide a hostid, it returns all services of that specified host""" + + if params.get("hostid"): + request_url = _create_server_get_url(workspace_name, '', **params) + '/hosts/' + str(params["hostid"]) + "/services" + else: + request_url = _create_server_get_url(workspace_name, 'services', **params) + return _get(request_url, **params) From 7e25589136557afd58255534c63cda09baf1907c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 21 Feb 2018 16:39:32 -0300 Subject: [PATCH 0914/1506] Allow showing host ID in hosts list --- server/www/scripts/hosts/controllers/hosts.js | 1 + server/www/scripts/hosts/partials/list.html | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/server/www/scripts/hosts/controllers/hosts.js b/server/www/scripts/hosts/controllers/hosts.js index 3421857330b..b54f6f0db2a 100644 --- a/server/www/scripts/hosts/controllers/hosts.js +++ b/server/www/scripts/hosts/controllers/hosts.js @@ -16,6 +16,7 @@ angular.module('faradayApp') $scope.columns = JSON.parse($cookies.get('HColumns')) }else{ $scope.columns = { + "id": false, "name": true, "description": false, "hostnames": false, diff --git a/server/www/scripts/hosts/partials/list.html b/server/www/scripts/hosts/partials/list.html index 4a20e1a7a3b..63477b585fe 100644 --- a/server/www/scripts/hosts/partials/list.html +++ b/server/www/scripts/hosts/partials/list.html @@ -66,6 +66,10 @@

    Add columns

     + + From 637627c43f3c6fbda335c63d55a1bd1d685f7e04 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 21 Feb 2018 19:21:58 -0300 Subject: [PATCH 0920/1506] GTK Sidebar working full now! --- persistence/server/models.py | 8 +++++++- persistence/server/server.py | 8 ++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index d593fc95425..1194c0a4d03 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -887,7 +887,7 @@ def getServices(self): """ Get all services of this host. """ - return get_services(self._workspace_name, hostid=self._server_id) + return get_services(self._workspace_name, host_id=self._server_id) def getService(self, service_id): """ @@ -968,6 +968,12 @@ def isOwned(self): def getVulnsAmount(self): return self.vuln_amount + def getVulns(self): + """ + Get all vulns of this service. + """ + return get_all_vulns(self._workspace_name, service_id=self._server_id) + class Vuln(ModelBase): """A simple Vuln class. Should implement all the methods of the diff --git a/persistence/server/server.py b/persistence/server/server.py index c985a0fc5e7..a5af42794f1 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -284,13 +284,9 @@ def _get_raw_vulns(workspace_name, **params): def _get_raw_services(workspace_name, **params): """Take a workspace_name and an arbitrary number of params and return a dictionary with the services table. - If you provide a hostid, it returns all services of that specified host""" - - if params.get("hostid"): - request_url = _create_server_get_url(workspace_name, '', **params) + '/hosts/' + str(params["hostid"]) + "/services" - else: - request_url = _create_server_get_url(workspace_name, 'services', **params) + If you provide a host_id and services_of_host=True, it returns all services of that specified host""" + request_url = _create_server_get_url(workspace_name, 'services', **params) return _get(request_url, **params) From 2d6f6ed926fef47eece4fb3118e94e730887a81e Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 21 Feb 2018 19:47:06 -0300 Subject: [PATCH 0921/1506] Fix GTK clear boxes in HostInfo dialog selections --- gui/gtk/dialogs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gui/gtk/dialogs.py b/gui/gtk/dialogs.py index ef592b191a5..5546408782e 100644 --- a/gui/gtk/dialogs.py +++ b/gui/gtk/dialogs.py @@ -954,9 +954,12 @@ def on_main_tree_selection(self, tree_selection): if object_type == 'Host': self.set_vuln_model(self.create_vuln_model(self.host)) + self.clear(self.specific_info) + self.clear(self.vuln_info) elif object_type == 'Service': self.clear(self.specific_info) + self.clear(self.vuln_info) self.change_label_in_frame(self.specific_info_frame, object_type) prop_names = self.get_properties_names(object_type) self.show_info_in_box(object_info, prop_names, self.specific_info) From 2cf5865188ec7e04419fcde95124c8b4c7cd386e Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 21 Feb 2018 20:14:57 -0300 Subject: [PATCH 0922/1506] Delete print of Websocket changes --- persistence/server/changes_stream.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/persistence/server/changes_stream.py b/persistence/server/changes_stream.py index d50e107db71..1ce4367a618 100644 --- a/persistence/server/changes_stream.py +++ b/persistence/server/changes_stream.py @@ -104,9 +104,7 @@ def on_error(self, ws, error): print error def on_close(selg, ws): - print "### closed ###" - - print 'open' + pass def __enter__(self): return self From 3fa6d05135aeb0be2173adadbbf009c75aee20ca Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 21 Feb 2018 20:16:05 -0300 Subject: [PATCH 0923/1506] Pass workspace name with parameter, not positional model controller --- model/controller.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/model/controller.py b/model/controller.py index 5007093b5c5..1c94c737272 100644 --- a/model/controller.py +++ b/model/controller.py @@ -497,20 +497,20 @@ def _pluginEnd(self, name, command_id): def newHost(self, ip, os="Unknown"): return model.common.factory.createModelObject( models.Host.class_signature, ip, - self.mappers_manager.workspace_name, os=os, parent_id=None) + workspace_name=self.mappers_manager.workspace_name, os=os, parent_id=None) def newService(self, name, protocol="tcp?", ports=[], status="running", version="unknown", description="", parent_id=None): return model.common.factory.createModelObject( models.Service.class_signature, name, - self.mappers_manager.workspace_name, protocol=protocol, ports=ports, status=status, + workspace_name=self.mappers_manager.workspace_name, protocol=protocol, ports=ports, status=status, version=version, description=description, parent_id=parent_id) def newVuln(self, name, desc="", ref=None, severity="", resolution="", confirmed=False, parent_id=None): return model.common.factory.createModelObject( models.Vuln.class_signature, name, - self.mappers_manager.workspace_name, desc=desc, ref=ref, severity=severity, resolution=resolution, + workspace_name=self.mappers_manager.workspace_name, desc=desc, ref=ref, severity=severity, resolution=resolution, confirmed=confirmed, parent_id=parent_id) def newVulnWeb(self, name, desc="", ref=None, severity="", resolution="", @@ -519,7 +519,7 @@ def newVulnWeb(self, name, desc="", ref=None, severity="", resolution="", parent_id=None): return model.common.factory.createModelObject( models.VulnWeb.class_signature, name, - self.mappers_manager.workspace_name, desc=desc, ref=ref, severity=severity, resolution=resolution, + workspace_name=self.mappers_manager.workspace_name, desc=desc, ref=ref, severity=severity, resolution=resolution, website=website, path=path, request=request, response=response, method=method, pname=pname, params=params, query=query, category=category, confirmed=confirmed, parent_id=parent_id) @@ -527,12 +527,12 @@ def newVulnWeb(self, name, desc="", ref=None, severity="", resolution="", def newNote(self, name, text, parent_id=None): return model.common.factory.createModelObject( models.Note.class_signature, name, - self.mappers_manager.workspace_name, text=text, parent_id=parent_id) + workspace_name=self.mappers_manager.workspace_name, text=text, parent_id=parent_id) def newCred(self, username, password, parent_id=None): return model.common.factory.createModelObject( models.Credential.class_signature, name, - username, password=password, parent_id=parent_id) + username, workspace_name=self.mappers_manager.workspace_name, password=password, parent_id=parent_id) def getHost(self, name): hosts_mapper = self.mappers_manager.getMapper(models.Host.class_signature) From f885c3a20a7ae3870d28d29c640c538b8fad3c9a Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 21 Feb 2018 20:37:53 -0300 Subject: [PATCH 0924/1506] Fixed workspace name missing in factory createModelObject plugins/plugin --- plugins/plugin.py | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/plugins/plugin.py b/plugins/plugin.py index a94fb2f3187..ac50f9bfbfc 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -45,6 +45,7 @@ def __init__(self): self.data_path = CONF.getDataPath() self.persistence_path = CONF.getPersistencePath() + self.workspace = CONF.getLastWorkspace() # Must be unique. Check that there is not # an existant plugin with the same id. # TODO: Make script that list current ids. @@ -167,15 +168,18 @@ def __addPendingAction(self, *args): args = args + (self.command_id, ) else: logger.warn('Warning command id not set for action {0}'.format(args)) - print('Addpending', args) + logger.debug('AddPendingAction', args) self._pending_actions.put(args) def createAndAddHost(self, name, os="unknown"): + host_obj = factory.createModelObject( Host.class_signature, name, os=os, - parent_id=None) + parent_id=None, + workspace_name=self.workspace) + host_obj._metadata.creatoserverr = self.id self.__addPendingAction(modelactions.ADDHOST, host_obj) return host_obj.getID() @@ -207,7 +211,8 @@ def createAndAddServiceToInterface(self, host_id, interface_id, name, Service.class_signature, name, protocol=protocol, ports=ports, status=status, version=version, description=description, - parent_type='Service', parent_id=host_id) + parent_type='Service', parent_id=host_id, + workspace_name=self.workspace) serv_obj._metadata.creator = self.id self.__addPendingAction(modelactions.ADDSERVICEHOST, serv_obj) @@ -220,7 +225,8 @@ def createAndAddVulnToHost(self, host_id, name, desc="", ref=[], Vuln.class_signature, name, desc=desc, refs=ref, severity=severity, resolution=resolution, confirmed=False, - parent_id=host_id, parent_type='Host') + parent_id=host_id, parent_type='Host', + workspace_name=self.workspace) vuln_obj._metadata.creator = self.id self.__addPendingAction(modelactions.ADDVULNHOST, vuln_obj) @@ -237,7 +243,8 @@ def createAndAddVulnToInterface(self, host_id, interface_id, name, Vuln.class_signature, name, desc=desc, refs=ref, severity=severity, resolution=resolution, confirmed=False, - parent_type='Host', parent_id=host_id) + parent_type='Host', parent_id=host_id, + workspace_name=self.workspace) vuln_obj._metadata.creator = self.id self.__addPendingAction(modelactions.ADDVULNHOST, vuln_obj) @@ -250,8 +257,8 @@ def createAndAddVulnToService(self, host_id, service_id, name, desc="", Vuln.class_signature, name, desc=desc, refs=ref, severity=severity, resolution=resolution, confirmed=False, - parent_type='Service', parent_id=service_id - ) + parent_type='Service', parent_id=service_id, + workspace_name=self.workspace) vuln_obj._metadata.creator = self.id self.__addPendingAction(modelactions.ADDVULNSRV, vuln_obj) @@ -270,7 +277,7 @@ def createAndAddVulnWebToService(self, host_id, service_id, name, desc="", pname=pname, params=params, query=query, category=category, confirmed=False, parent_id=service_id, parent_type='Service', - ) + workspace_name=self.workspace) vulnweb_obj._metadata.creator = self.id self.__addPendingAction(modelactions.ADDVULNWEBSRV, vulnweb_obj) @@ -280,7 +287,8 @@ def createAndAddNoteToHost(self, host_id, name, text): note_obj = model.common.factory.createModelObject( Note.class_signature, - name, text=text, object_id=host_id, object_type='host') + name, text=text, object_id=host_id, object_type='host', + workspace_name=self.workspace) note_obj._metadata.creator = self.id self.__addPendingAction(modelactions.ADDNOTEHOST, note_obj) @@ -293,7 +301,8 @@ def createAndAddNoteToInterface(self, host_id, interface_id, name, text): note_obj = model.common.factory.createModelObject( Note.class_signature, - name, text=text, object_id=host_id, object_type='host') + name, text=text, object_id=host_id, object_type='host', + workspace_name=self.workspace) note_obj._metadata.creator = self.id self.__addPendingAction(modelactions.ADDNOTEHOST, note_obj) @@ -303,7 +312,8 @@ def createAndAddNoteToService(self, host_id, service_id, name, text): note_obj = model.common.factory.createModelObject( Note.class_signature, - name, text=text, object_id=service_id, object_type='service') + name, text=text, object_id=service_id, object_type='service', + workspace_name=self.workspace) note_obj._metadata.creator = self.id self.__addPendingAction(modelactions.ADDNOTESRV, note_obj) @@ -313,7 +323,8 @@ def createAndAddNoteToNote(self, host_id, service_id, note_id, name, text): note_obj = model.common.factory.createModelObject( Note.class_signature, - name, text=text, object_id=note_id, object_type='comment') + name, text=text, object_id=note_id, object_type='comment', + workspace_name=self.workspace) note_obj._metadata.creator = self.id @@ -325,7 +336,8 @@ def createAndAddCredToService(self, host_id, service_id, username, cred_obj = model.common.factory.createModelObject( Credential.class_signature, - username, password=password, parent_id=service_id, parent_type='Service') + username, password=password, parent_id=service_id, parent_type='Service', + workspace_name=self.workspace) cred_obj._metadata.creator = self.id self.__addPendingAction(modelactions.ADDCREDSRV, cred_obj) From bfee65b9611050f8d75b45e307affa3c76fb9b85 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 21 Feb 2018 20:41:39 -0300 Subject: [PATCH 0925/1506] Fixed issue with empty host in GTK sidebar --- persistence/server/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index 1194c0a4d03..982327b9848 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -252,7 +252,7 @@ def get_services(workspace_name, **params): """ services_dictionary = server.get_services(workspace_name, **params) # List inside of list, use the inside list... - if type(services_dictionary[0]) == list: + if len(services_dictionary) > 0 and type(services_dictionary[0]) == list: services_dictionary = services_dictionary[0] return _get_faraday_ready_services(workspace_name, services_dictionary) From 68a7b56e7aa30e205c18c7b24558d11937fc36cf Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Thu, 22 Feb 2018 20:11:51 -0300 Subject: [PATCH 0926/1506] New parameters keep old and keep new --- faraday.py | 2 ++ model/application.py | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/faraday.py b/faraday.py index a16172c8364..0d3212c44d5 100755 --- a/faraday.py +++ b/faraday.py @@ -172,6 +172,8 @@ def getParserArgs(): help="Enables debug mode. Default = disabled") parser.add_argument('--nodeps', action='store_true', help='Skip dependency check') + parser.add_argument('--keep-old', action='store_true', help='Keep old object in CLI mode if faraday find a conflict') + parser.add_argument('--keep-new', action='store_true', help='Keep new object in CLI mode if faraday find a conflict (DEFAULT ACTION)') f = open(FARADAY_VERSION_FILE) f_version = f.read().strip() diff --git a/model/application.py b/model/application.py index f8a6c872cae..b7986221a93 100644 --- a/model/application.py +++ b/model/application.py @@ -96,8 +96,14 @@ def __init__(self, args): ) if self.args.cli: + self.app = CliApp(self._workspace_manager, self._plugin_controller) - CONF.setMergeStrategy("new") + + if self.args.keep_old: + CONF.setMergeStrategy("old") + else: + CONF.setMergeStrategy("new") + else: self.app = UiFactory.create(self._model_controller, self._plugin_manager, From 6afae219fa0e8e2358b9afaab9719fadae5756b1 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Thu, 22 Feb 2018 20:12:13 -0300 Subject: [PATCH 0927/1506] Fix a bug with MergeStrategy in model --- persistence/server/models.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index b7ec8b374a9..a444ded58ab 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -732,21 +732,33 @@ def tieBreak(self, key, prop1, prop2): return None def addUpdate(self, newModelObject): + conflict = False diff = ModelObjectDiff(self, newModelObject) + for k, v in diff.getPropertiesDiff().items(): attribute = self.publicattrsrefs().get(k) prop_update = self.propertyTieBreaker(attribute, *v) + option_choosen = prop_update + # if there's a strategy set by the user, apply it if not isinstance(prop_update, tuple) or _get_merge_strategy(): - # if there's a strategy set by the user, apply it + if isinstance(prop_update, tuple): - prop_update = MergeSolver(_get_merge_strategy()) - prop_update = prop_update.solve(prop_update[0], prop_update[1]) + #Choose the new attribute based in merge strategy: old or new + merge_solver = MergeSolver(_get_merge_strategy()) + option_choosen = merge_solver.solve(prop_update[0], prop_update[1]) - setattr(self, attribute, prop_update) + #Faraday have duplicated description field, so if we change + #description, we need change also, the desc field used in WEBUI + if attribute == "description" and (self.class_signature == "Vulnerability" or self.class_signature == "VulnerabilityWeb"): + setattr(self, "desc", option_choosen) + + #Set the new choosen attribute + setattr(self, attribute, option_choosen) else: conflict = True + if conflict: self.updates.append(ConflictUpdate(self, newModelObject)) return conflict From 91b8b2b26243e5b2924257944debadfe57dda7fb Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 26 Feb 2018 15:11:25 -0300 Subject: [PATCH 0928/1506] Fix tests for sqlite --- server/api/base.py | 8 ++++++-- server/models.py | 14 ++++++++++++++ server/utils/database.py | 4 ++-- test_cases/conftest.py | 10 ++++++++-- test_cases/test_api_hosts.py | 1 + 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 285db3faa60..1af7b8714f4 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -481,7 +481,8 @@ def _perform_create(self, data, workspace_name): db.session.commit() except sqlalchemy.exc.IntegrityError as ex: db.session.rollback() - conflict_obj = get_conflict_object(db.session, obj, data) + workspace = self._get_workspace(workspace_name) + conflict_obj = get_conflict_object(db.session, obj, data, workspace) if conflict_obj: abort(409, ValidationError( { @@ -520,7 +521,10 @@ def _perform_update(self, object_id, obj, data, workspace_name=None): db.session.commit() except sqlalchemy.exc.IntegrityError as ex: db.session.rollback() - conflict_obj = get_conflict_object(db.session, obj, data) + workspace = None + if workspace_name: + workspace = db.session.query(Workspace).filter_by(name=workspace_name).first() + conflict_obj = get_conflict_object(db.session, obj, data, workspace) if conflict_obj: abort(409, ValidationError( { diff --git a/server/models.py b/server/models.py index 9147b567ad1..2ad07b5ec84 100644 --- a/server/models.py +++ b/server/models.py @@ -1481,11 +1481,25 @@ def parent(self): "COALESCE(website, ''), workspace_id, COALESCE(source_code_id, -1));" ) +vulnerability_uniqueness_sqlite = DDL( + "CREATE UNIQUE INDEX uix_vulnerability ON %(fullname)s " + "(name, (description), COALESCE(host_id, -1), COALESCE(service_id, -1), " + "COALESCE(method, ''), COALESCE(parameter_name, ''), COALESCE(path, ''), " + "COALESCE(website, ''), workspace_id, COALESCE(source_code_id, -1));" +) + + event.listen( VulnerabilityGeneric.__table__, 'after_create', vulnerability_uniqueness.execute_if(dialect='postgresql') ) +event.listen( + VulnerabilityGeneric.__table__, + 'after_create', + vulnerability_uniqueness_sqlite.execute_if(dialect='sqlite') +) + # We have to import this after all models are defined import server.events diff --git a/server/utils/database.py b/server/utils/database.py index 11be686e7d7..0031ea269a7 100644 --- a/server/utils/database.py +++ b/server/utils/database.py @@ -248,7 +248,7 @@ def get_unique_fields(session, instance): yield unique_constraint['column_names'] -def get_conflict_object(session, obj, data): +def get_conflict_object(session, obj, data, workspace=None): unique_fields_gen = get_unique_fields(session, obj) for unique_fields in unique_fields_gen: relations_fields = filter( @@ -262,7 +262,7 @@ def get_conflict_object(session, obj, data): if 'workspace_id' in relations_fields: relations_fields.remove('workspace_id') - filter_data['workspace_id'] = obj.workspace.id + filter_data['workspace_id'] = workspace.id for relations_field in relations_fields: if relations_field not in data and relations_field.strip('_id') in data: diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 2638340ddb1..262f3718600 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -1,3 +1,5 @@ +from tempfile import NamedTemporaryFile + import os import sys import json @@ -15,7 +17,7 @@ from test_cases import factories - +TEMPORATY_SQLITE = NamedTemporaryFile() # Discover factories to automatically register them to pytest-factoryboy and to # override its session enabled_factories = [] @@ -60,7 +62,10 @@ def open(self, *args, **kwargs): def pytest_addoption(parser): - parser.addoption('--connection-string', default='sqlite://', + # currently for tests using sqlite and memory have problem while using transactions + # we need to review sqlite configuraitons for persistence using PRAGMA. + print(TEMPORATY_SQLITE.name) + parser.addoption('--connection-string', default='sqlite:////{0}'.format(TEMPORATY_SQLITE.name), help="Database connection string. Defaults to in-memory " "sqlite if not specified:") parser.addoption('--ignore-nplusone', action='store_true', @@ -78,6 +83,7 @@ def app(request): ctx.push() def teardown(): + TEMPORATY_SQLITE.close() ctx.pop() request.addfinalizer(teardown) diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 72b7454475a..db8490f2931 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -121,6 +121,7 @@ def test_create_a_host_fails_with_existing_ip(self, session, test_client, host): session.add(host) session.commit() + assert Host.query.count() == HOSTS_COUNT + 1 res = test_client.post(self.url(), data={ "ip": host.ip, From 0bad128abd3e6c9386fd516bd7b84a5ef8f02658 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 26 Feb 2018 15:41:50 -0300 Subject: [PATCH 0929/1506] Update DDL --- server/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index 2ad07b5ec84..ee2dfd2913e 100644 --- a/server/models.py +++ b/server/models.py @@ -1483,7 +1483,7 @@ def parent(self): vulnerability_uniqueness_sqlite = DDL( "CREATE UNIQUE INDEX uix_vulnerability ON %(fullname)s " - "(name, (description), COALESCE(host_id, -1), COALESCE(service_id, -1), " + "(name, description, COALESCE(host_id, -1), COALESCE(service_id, -1), " "COALESCE(method, ''), COALESCE(parameter_name, ''), COALESCE(path, ''), " "COALESCE(website, ''), workspace_id, COALESCE(source_code_id, -1));" ) From c189a9536fe31beb193ff31a2e8993e2700f5580 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 26 Feb 2018 16:47:32 -0300 Subject: [PATCH 0930/1506] Fix timezone issues that broke the API when updating licenses Passing timezone aware timestamps (as the web did) raised a 500 error due to an issue with marshmallow. Switched to an alternative implementation of DateTimeField. Explicitly included dateutil dependency, not it is directly used --- requirements_server.txt | 1 + server/api/modules/licenses.py | 7 +++++-- server/schemas.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/requirements_server.txt b/requirements_server.txt index b06a97db300..7e459367d8c 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -9,6 +9,7 @@ Pillow==4.2.1 psycopg2==2.7.3.2 pyasn1-modules>=0.0.11 pyopenssl==17.2.0 +python-dateutil==2.6.0 python-slugify==1.2.4 requests==2.18.4 restkit==4.2.2 diff --git a/server/api/modules/licenses.py b/server/api/modules/licenses.py index 9ff0b5fad8a..adc541228a6 100644 --- a/server/api/modules/licenses.py +++ b/server/api/modules/licenses.py @@ -11,15 +11,17 @@ ReadWriteView, AutoSchema, ) +from server.schemas import StrictDateTimeField license_api = Blueprint('license_api', __name__) class LicenseSchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') - end = fields.DateTime(attribute='end_date') + end = StrictDateTimeField(load_as_tz_aware=False, attribute='end_date') + start = StrictDateTimeField(load_as_tz_aware=False, attribute='start_date') lictype = fields.String(attribute='type') - start = fields.DateTime(attribute='start_date') + class Meta: model = License fields = ('_id', 'id', 'product', @@ -33,4 +35,5 @@ class LicenseView(ReadWriteView): schema_class = LicenseSchema unique_fields = [] + LicenseView.register(license_api) diff --git a/server/schemas.py b/server/schemas.py index 06de430743b..21ba4ea0550 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -2,6 +2,7 @@ import datetime from marshmallow import fields, Schema from marshmallow.exceptions import ValidationError +from dateutil.tz import tzutc from server.models import CommandObject, VulnerabilityABC @@ -144,3 +145,34 @@ class MetadataSchema(Schema): update_user = fields.String(default='', dump_only=True) update_action = fields.Integer(default=0, dump_only=True) update_controller_action = fields.String(default='', dump_only=True) + + +class StrictDateTimeField(fields.DateTime): + """ + Marshmallow DateTime field with extra parameter to control + whether dates should be loaded as tz_aware or not + """ + # Taken from + # https://github.com/Nobatek/umongo/blob/14ec7e40ca517071d9374af39f8409223e097253/umongo/marshmallow_bonus.py + + # TODO migration: write me some tests!!! + + def __init__(self, load_as_tz_aware=False, *args, **kwargs): + super(StrictDateTimeField, self).__init__(*args, **kwargs) + self.load_as_tz_aware = load_as_tz_aware + + def _deserialize(self, value, attr, data): + if isinstance(value, datetime.datetime): + date = value + else: + date = super(StrictDateTimeField, self)._deserialize(value, attr, data) + if self.load_as_tz_aware: + # If datetime is TZ naive, set UTC timezone + if date.tzinfo is None or date.tzinfo.utcoffset(date) is None: + date = date.replace(tzinfo=tzutc()) + else: + # If datetime is TZ aware, convert it to UTC and remove TZ info + if date.tzinfo is not None and date.tzinfo.utcoffset(date) is not None: + date.astimezone(tzutc()) + date = date.replace(tzinfo=None) + return date From 0dff28c009cf5c03dce9930ac90a61f2a74d228b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 27 Feb 2018 14:43:42 -0300 Subject: [PATCH 0931/1506] Avoid stopping the model controller thread. Moved the code to tstop the thread when the plugin end event was processed --- model/application.py | 1 - model/controller.py | 5 +---- persistence/server/models.py | 1 + utils/error_report.py | 1 - 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/model/application.py b/model/application.py index 43add6cc519..0a8ef00ec19 100644 --- a/model/application.py +++ b/model/application.py @@ -192,7 +192,6 @@ def __exit(self, exit_code=0): model.api.devlog("stopping model controller thread...") model.api.stopAPIServer() restapi.stopServer() - self._model_controller.stop() self._model_controller.join() self.timer.stop() model.api.devlog("Waiting for controller threads to end...") diff --git a/model/controller.py b/model/controller.py index 1c94c737272..f182feeb6dd 100644 --- a/model/controller.py +++ b/model/controller.py @@ -145,7 +145,6 @@ def __init__(self, mappers_manager, pending_actions): self._setupActionDispatcher() self.objects_with_updates = [] - self._stop = False def __getattr__(self, name): getLogger(self).debug("ModelObject attribute to refactor: %s" % name) @@ -316,9 +315,6 @@ def setSavingModel(self, value): except RuntimeError: pass - def stop(self): - self._stop = True - def _main(self): """ The main method for the thread. @@ -491,6 +487,7 @@ def _pluginEnd(self, name, command_id): getLogger(self).info("Plugin Ended: {0}".format(name)) self.active_plugins_count -= 1 self.active_plugins_count_lock.release() + self._stop = True return True diff --git a/persistence/server/models.py b/persistence/server/models.py index 982327b9848..e85c94de583 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -738,6 +738,7 @@ def setParentType(self, parent_type): self.parent_type = parent_type def setID(self, id): + print('setID {0}'.format(id)) self.id = id self.id_available.set() diff --git a/utils/error_report.py b/utils/error_report.py index 060f6c1adfd..94a070337d7 100644 --- a/utils/error_report.py +++ b/utils/error_report.py @@ -41,7 +41,6 @@ def exception_handler(type, value, tb): Since this handler may be called from threads, the dialog must be created using gtk idle_add or signals to avoid issues. """ - import requests import hashlib import platform From b19a0543fa3aa43d3160f07209784bee0a11cbf7 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 27 Feb 2018 14:46:48 -0300 Subject: [PATCH 0932/1506] Change User table name to avoid problems with postgresql "user" is a reserved word for postgresql. we change the table name to aovid support issues while debugging the database --- server/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index d77940cfc80..17dd8c677c8 100644 --- a/server/models.py +++ b/server/models.py @@ -1187,7 +1187,7 @@ class Role(Metadata, RoleMixin): class User(db.Model, UserMixin): - __tablename__ = 'user' + __tablename__ = 'faraday_user' id = Column(Integer, primary_key=True) username = Column(String(255), unique=True, nullable=False) password = Column(String(255), nullable=True) From 26f36a8214ad0e5cd3e1431b4b470d46319ae43f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 27 Feb 2018 14:55:14 -0300 Subject: [PATCH 0933/1506] Fix relationship to use new table name --- server/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/models.py b/server/models.py index 17dd8c677c8..e3d4739492a 100644 --- a/server/models.py +++ b/server/models.py @@ -145,7 +145,7 @@ class Metadata(db.Model): @declared_attr def creator_id(cls): - return Column(Integer, ForeignKey('user.id'), nullable=True) + return Column(Integer, ForeignKey('faraday_user.id'), nullable=True) @declared_attr def creator(cls): @@ -1175,7 +1175,7 @@ def get(workspace_name): class RolesUsers(db.Model): __tablename__ = 'roles_users' id = Column(Integer(), primary_key=True) - user_id = Column('user_id', Integer(), ForeignKey('user.id')) + user_id = Column('user_id', Integer(), ForeignKey('faraday_user.id')) role_id = Column('role_id', Integer(), ForeignKey('role.id')) @@ -1249,7 +1249,7 @@ class UserAvatar(Metadata): # photo field will automatically generate thumbnail # if the file is a valid image photo = Column(UploadedFileField(upload_type=FaradayUploadedFile)) - user_id = Column('user_id', Integer(), ForeignKey('user.id')) + user_id = Column('user_id', Integer(), ForeignKey('faraday_user.id')) user = relationship('User', foreign_keys=[user_id]) @@ -1329,7 +1329,7 @@ class Task(TaskABC): 'concrete': True } - assigned_to_id = Column(Integer, ForeignKey('user.id'), nullable=True) + assigned_to_id = Column(Integer, ForeignKey('faraday_user.id'), nullable=True) assigned_to = relationship('User', backref='assigned_tasks', foreign_keys=[assigned_to_id]) methodology_id = Column( From 8f4bc1a76b503cda51ccd675439bb01d3e23c53f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 27 Feb 2018 15:21:51 -0300 Subject: [PATCH 0934/1506] Fix --debug and plugin logging in GTK Why does this have to be so hard??? :( --- model/application.py | 1 - model/controller.py | 24 ++++++++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/model/application.py b/model/application.py index 43add6cc519..e002546efc7 100644 --- a/model/application.py +++ b/model/application.py @@ -78,7 +78,6 @@ def stop(self): class MainApplication(object): def __init__(self, args): - setUpLogger() self._original_excepthook = sys.excepthook self.args = args diff --git a/model/controller.py b/model/controller.py index 1c94c737272..dfa9f2ee0b3 100644 --- a/model/controller.py +++ b/model/controller.py @@ -33,7 +33,7 @@ class modelactions: DELHOST = 2001 ADDSERVICEHOST = 2008 ADDSERVICEHOST = 20008 - ADDCATEGORY = 2011 + ADDCATEGORY = 2011 # TODO migration: check why isn't implemented ADDVULNHOST = 2017 DELVULNHOST = 2018 ADDVULNSRV = 2019 @@ -42,17 +42,17 @@ class modelactions: DELNOTEHOST = 2026 ADDNOTESRV = 2027 DELNOTESRV = 2028 - RENAMEROOT = 2029 + RENAMEROOT = 2029 # TODO migration: check why isn't implemented ADDNOTEVULN = 2030 - DELNOTEVULN = 2031 + DELNOTEVULN = 2031 # TODO migration: check why isn't implemented EDITHOST = 2032 EDITSERVICE = 2035 ADDCREDSRV = 2036 DELCREDSRV = 2037 ADDVULNWEBSRV = 2038 - DELVULNWEBSRV = 2039 + DELVULNWEBSRV = 2039 # TODO migration: check why isn't implemented ADDNOTENOTE = 2040 - DELNOTENOTE = 2041 + DELNOTENOTE = 2041 # TODO migration: check why isn't implemented EDITNOTE = 2042 EDITVULN = 2043 ADDNOTE = 2044 @@ -228,7 +228,9 @@ def _setupActionDispatcher(self): modelactions.DELCRED: self.__del, # Plugin states modelactions.PLUGINSTART: self._pluginStart, - modelactions.PLUGINEND: self._pluginEnd + modelactions.PLUGINEND: self._pluginEnd, + modelactions.DEVLOG: self._devlog, + modelactions.LOG: self._log, } def run(self): @@ -493,6 +495,16 @@ def _pluginEnd(self, name, command_id): self.active_plugins_count_lock.release() return True + def _devlog(self, msg, *args, **kwargs): + # I have no idea what I am doing + api.devlog(msg) + return True + + def _log(self, msg, *args, **kwargs): + # I have no idea what I am doing + api.log(msg, *args[:-1]) + return True + def newHost(self, ip, os="Unknown"): return model.common.factory.createModelObject( From ccb5af8829d90b3db81d71cc3651dc8666350f88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 27 Feb 2018 16:07:19 -0300 Subject: [PATCH 0935/1506] Change web UI path from /_ui/ to / Make /_ui/ redirect to / to keep backwards compatibility --- RELEASE.md | 1 + server/web.py | 10 ++++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 482d2a602a9..f665c2fe30c 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -23,6 +23,7 @@ TBA: * Workspace names can be changed from the Web UI * Changed the scope field of a workspace from a free text input to a list of targets * Exploitation and severity fields only allow certain values. CWE CVEs were fixed to be valid. A script to convert custom CSVs was added. +* Web UI path changed from /_ui/ to / (_ui has now a redirection to / for keeping backwards compatibility) November 17, 2017: --- diff --git a/server/web.py b/server/web.py index a4f3c19fae7..d51f3762a6f 100644 --- a/server/web.py +++ b/server/web.py @@ -69,7 +69,6 @@ def render(self, request): class WebServer(object): - HOME = '' UI_URL_PATH = '_ui' API_URL_PATH = '_api' WEB_UI_LOCAL_PATH = os.path.join(server.config.FARADAY_BASE, 'server/www') @@ -97,15 +96,14 @@ def __load_ssl_certs(self): return ssl.DefaultOpenSSLContextFactory(*certs) def __build_server_tree(self): - self.__root_resource = CleanHttpHeadersResource() - self.__root_resource.putChild(WebServer.HOME, self.__build_web_redirect()) - self.__root_resource.putChild( - WebServer.UI_URL_PATH, self.__build_web_resource()) + self.__root_resource = self.__build_web_resource() + self.__root_resource.putChild(WebServer.UI_URL_PATH, + self.__build_web_redirect()) self.__root_resource.putChild( WebServer.API_URL_PATH, self.__build_api_resource()) def __build_web_redirect(self): - return FaradayRedirectResource(WebServer.UI_URL_PATH) + return FaradayRedirectResource('/') def __build_web_resource(self): return FileWithoutDirectoryListing(WebServer.WEB_UI_LOCAL_PATH) From 00d4a8467104d49f4b5a4cfda2044a42f255dfa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 27 Feb 2018 17:09:31 -0300 Subject: [PATCH 0936/1506] Fix about modal image path --- server/www/scripts/commons/partials/modalAbout.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/commons/partials/modalAbout.html b/server/www/scripts/commons/partials/modalAbout.html index 27e05b03164..9cc79522312 100644 --- a/server/www/scripts/commons/partials/modalAbout.html +++ b/server/www/scripts/commons/partials/modalAbout.html @@ -3,7 +3,7 @@
    -

    About Faraday

    +

    About Faraday

    Faraday {{version}} by Infobyte Security

    From 419dc6f66af51cf78a4cbd3b7e5eab45ef1210e7 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 28 Feb 2018 15:13:33 -0300 Subject: [PATCH 0937/1506] Update users table name on admin creation --- server/commands/initdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index c89f1d6174b..e718260a91e 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -121,7 +121,7 @@ def _create_admin_user(self, conn_string): random_password = self.generate_random_pw(12) already_created = False try: - engine.execute("INSERT INTO \"user\" (username, name, password, " + engine.execute("INSERT INTO \"faraday_user\" (username, name, password, " "is_ldap, active) VALUES ('faraday', 'Administrator', " "'{0}', false, true);".format(random_password)) except sqlalchemy.exc.IntegrityError: From 38e90e99b74be1168c25da814f1e70cba7465438 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 28 Feb 2018 15:21:25 -0300 Subject: [PATCH 0938/1506] Add missing dep for sql_shell --- requirements_server.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_server.txt b/requirements_server.txt index 7e459367d8c..fa7a0219c46 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -24,3 +24,4 @@ git+https://github.com/sh4r3m4n/filteralchemy@dev#egg=filteralchemy filedepot==0.5.0 nplusone==0.8.1 deprecation==1.0.1 +pgcli==1.8.2 From 3ddce18fe39d75a4d0da3151a8affadbb89eaad4 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 28 Feb 2018 15:23:51 -0300 Subject: [PATCH 0939/1506] Show couchdb error when credentials are wrong --- server/importer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/importer.py b/server/importer.py index 094209bfd0b..b74c654d688 100644 --- a/server/importer.py +++ b/server/importer.py @@ -984,6 +984,8 @@ def import_admins(self, admins): def import_users(self, all_users, admins): # Import non admin users + if 'error' in all_users: + raise Exception(all_users['reason']) for user in all_users['rows']: user = user['doc'] if not user['_id'].startswith(COUCHDB_USER_PREFIX): From d42ae2883be4e8eac4c11b4fdc3be0cf50adbf33 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 28 Feb 2018 16:34:44 -0300 Subject: [PATCH 0940/1506] Use multithreading for couchdb importer. Changed logger messages to debug --- server/importer.py | 133 ++++++++++++++++++++++++++------------------- server/models.py | 1 - 2 files changed, 76 insertions(+), 58 deletions(-) diff --git a/server/importer.py b/server/importer.py index b74c654d688..f06ac598ec8 100644 --- a/server/importer.py +++ b/server/importer.py @@ -1,12 +1,16 @@ # Faraday Penetration Test IDE # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information + import os import re import sys import json import logging import datetime +import threading +import multiprocessing + import requests from tempfile import NamedTemporaryFile @@ -42,7 +46,6 @@ MethodologyTemplate, PolicyViolation, Reference, - ReferenceTemplate, Service, Scope, Tag, @@ -55,6 +58,7 @@ VulnerabilityWeb, Workspace, File, + SQLAlchemy, ) from server.utils import invalid_chars from server.utils.database import get_or_create @@ -237,9 +241,9 @@ def map_tool_with_command_id(command_tool_map, document): def update_command_tools(workspace, command_tool_map, id_map): if command_tool_map: - logger.info("Setting the tool to {} commands".format( + logger.debug("Setting the tool to {} commands".format( len(command_tool_map))) - for (command_couchid, tool) in tqdm(command_tool_map.items()): + for (command_couchid, tool) in (command_tool_map.items()): try: map_data = id_map[command_couchid] @@ -894,7 +898,7 @@ def parse(self, document): return None, None def get_importer_from_document(self, doc_type): - logger.info('Getting class importer for {0} in workspace {1}'.format(doc_type, self.workspace_name)) + logger.debug('Getting class importer for {0} in workspace {1}'.format(doc_type, self.workspace_name)) importer_class_mapper = { 'EntityMetadata': EntityMetadataImporter, 'Host': HostImporter, @@ -970,7 +974,7 @@ def get_users_and_admins(self): def import_admins(self, admins): # Import admin users for (username, password) in admins.items(): - logger.info('Creating user {0}'.format(username)) + logger.debug('Creating user {0}'.format(username)) admin = db.session.query(User).filter_by(username=username).first() if not admin: app.user_datastore.create_user( @@ -994,7 +998,7 @@ def import_users(self, all_users, admins): if user['name'] in admins.keys(): # This is an already imported admin user, skip continue - logger.info(u'Importing user {0}'.format(user['name'])) + logger.debug(u'Importing user {0}'.format(user['name'])) old_user = db.session.query(User).filter_by(username=user['name']).first() if not old_user: app.user_datastore.create_user( @@ -1020,7 +1024,7 @@ def __init__(self): self.names = Counter() def run(self): - logger.info("Importing vulnerability templates") + logger.debug("Importing vulnerability templates") cwe_url = "http://{username}:{password}@{hostname}:{port}/{path}".format( username=server.config.couchdb.user, password=server.config.couchdb.password, @@ -1039,7 +1043,7 @@ def run(self): logger.warn(e) return - for cwe in tqdm(cwes.json()['rows']): + for cwe in (cwes.json()['rows']): document = cwe['doc'] new_severity = self.get_severity(document) @@ -1165,16 +1169,27 @@ def run(self): users_import = ImportCouchDBUsers() users_import.run() - - for workspace_name in workspaces_list: - logger.info(u'Setting up workspace {}'.format(workspace_name)) + logger.info('Importing workspaces. Using {0} threads'.format(multiprocessing.cpu_count())) + workspace_threads = [] + for workspace_name in tqdm(workspaces_list): + logger.debug(u'Setting up workspace {}'.format(workspace_name)) if not server.couchdb.server_has_access_to(workspace_name): logger.error(u"Unauthorized access to CouchDB. Make sure faraday-server's"\ " configuration file has CouchDB admin's credentials set") sys.exit(1) - - self.import_workspace_into_database(workspace_name) + thread = threading.Thread(target=self.import_workspace_into_database, args=(workspace_name, )) + thread.daemon = True + thread.start() + workspace_threads.append(thread) + if len(workspace_threads) > multiprocessing.cpu_count(): + for thread in workspace_threads: + thread.join() + workspace_threads.remove(thread) + + for thread in workspace_threads: + thread.join() + #self.import_workspace_into_database(workspace_name) def get_objs(self, host, obj_type, level, workspace): if obj_type == 'Credential': @@ -1223,8 +1238,8 @@ def verify_import_data(self, couchdb_relational_map, couchdb_relational_map_by_t missing_ids = set([x for x in missing_ids if not re.match(r'^\_design', x)]) objs_diff = [] if missing_ids: - logger.info('Downloading missing couchdb docs') - for missing_id in tqdm(missing_ids): + logger.debug('Downloading missing couchdb docs') + for missing_id in (missing_ids): not_imported_obj = get_object_from_couchdb(missing_id, workspace) if not_imported_obj['type'] == 'Interface': @@ -1241,48 +1256,52 @@ def verify_import_data(self, couchdb_relational_map, couchdb_relational_map_by_t missing_objs_file.write(json.dumps(objs_diff)) def import_workspace_into_database(self, workspace_name): + with app.app_context(): - faraday_importer = FaradayEntityImporter(workspace_name) - workspace, created = get_or_create(session, Workspace, name=workspace_name) - session.commit() + faraday_importer = FaradayEntityImporter(workspace_name) + workspace, created = get_or_create(session, Workspace, name=workspace_name) + session.commit() - couch_url = "http://{username}:{password}@{hostname}:{port}/{workspace_name}/_temp_view?include_docs=true".format( - username=server.config.couchdb.user, - password=server.config.couchdb.password, - hostname=server.config.couchdb.host, - port=server.config.couchdb.port, - workspace_name=workspace_name - ) + couch_url = "http://{username}:{password}@{hostname}:{port}/{workspace_name}/_temp_view?include_docs=true".format( + username=server.config.couchdb.user, + password=server.config.couchdb.password, + hostname=server.config.couchdb.host, + port=server.config.couchdb.port, + workspace_name=workspace_name + ) - # obj_types are tuples. the first value is the level on the tree - # for the desired obj. - obj_types = OBJ_TYPES - couchdb_relational_map = defaultdict(list) - couchdb_relational_map_by_type = defaultdict(list) - command_tool_map = {} - for level, obj_type in obj_types: - obj_importer = faraday_importer.get_importer_from_document(obj_type)() - objs_dict = self.get_objs(couch_url, obj_type, level, workspace) - for raw_obj in tqdm(objs_dict.get('rows', [])): - # we use no_autoflush since some queries triggers flush and some relationship are missing in the middle - with session.no_autoflush: - raw_obj = raw_obj['value'] - couchdb_id = raw_obj['_id'] - - # first let's make sure no invalid chars are present in the Raw objects - raw_obj = invalid_chars.clean_dict(raw_obj) - - for new_obj in obj_importer.update_from_document(raw_obj, workspace, level, couchdb_relational_map): - if not new_obj: - continue - set_metadata(raw_obj, new_obj) - map_tool_with_command_id(command_tool_map, - raw_obj) - session.commit() - couchdb_relational_map_by_type[couchdb_id].append({'type': obj_type, 'id': new_obj.id}) - couchdb_relational_map[couchdb_id].append(new_obj.id) - update_command_tools(workspace, command_tool_map, - couchdb_relational_map_by_type) - session.commit() - self.verify_import_data(couchdb_relational_map, couchdb_relational_map_by_type, workspace) - return created + # obj_types are tuples. the first value is the level on the tree + # for the desired obj. + obj_types = OBJ_TYPES + couchdb_relational_map = defaultdict(list) + couchdb_relational_map_by_type = defaultdict(list) + command_tool_map = {} + for level, obj_type in obj_types: + obj_importer = faraday_importer.get_importer_from_document(obj_type)() + objs_dict = self.get_objs(couch_url, obj_type, level, workspace) + for raw_obj in (objs_dict.get('rows', [])): + # we use no_autoflush since some queries triggers flush and some relationship are missing in the middle + with session.no_autoflush: + raw_obj = raw_obj['value'] + couchdb_id = raw_obj['_id'] + + # first let's make sure no invalid chars are present in the Raw objects + raw_obj = invalid_chars.clean_dict(raw_obj) + + for new_obj in obj_importer.update_from_document(raw_obj, workspace, level, couchdb_relational_map): + if not new_obj: + continue + set_metadata(raw_obj, new_obj) + map_tool_with_command_id(command_tool_map, + raw_obj) + session.commit() + couchdb_relational_map_by_type[couchdb_id].append({'type': obj_type, 'id': new_obj.id}) + couchdb_relational_map[couchdb_id].append(new_obj.id) + update_command_tools(workspace, command_tool_map, + couchdb_relational_map_by_type) + session.commit() + self.verify_import_data(couchdb_relational_map, couchdb_relational_map_by_type, workspace) + + session.expunge_all() + session.close() + return created diff --git a/server/models.py b/server/models.py index e3d4739492a..a1b3f7ee43a 100644 --- a/server/models.py +++ b/server/models.py @@ -10,7 +10,6 @@ Column, DateTime, Enum, - event, Float, ForeignKey, Integer, From f1262b9d9c6eddee16ea2223f2bf3086275c9633 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 28 Feb 2018 16:43:05 -0300 Subject: [PATCH 0941/1506] Update tqdm after joining --- server/importer.py | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/server/importer.py b/server/importer.py index f06ac598ec8..dfb0cd3fe7f 100644 --- a/server/importer.py +++ b/server/importer.py @@ -1171,25 +1171,30 @@ def run(self): logger.info('Importing workspaces. Using {0} threads'.format(multiprocessing.cpu_count())) workspace_threads = [] - for workspace_name in tqdm(workspaces_list): - logger.debug(u'Setting up workspace {}'.format(workspace_name)) - - if not server.couchdb.server_has_access_to(workspace_name): - logger.error(u"Unauthorized access to CouchDB. Make sure faraday-server's"\ - " configuration file has CouchDB admin's credentials set") - sys.exit(1) - thread = threading.Thread(target=self.import_workspace_into_database, args=(workspace_name, )) - thread.daemon = True - thread.start() - workspace_threads.append(thread) - if len(workspace_threads) > multiprocessing.cpu_count(): - for thread in workspace_threads: - thread.join() - workspace_threads.remove(thread) - - for thread in workspace_threads: - thread.join() - #self.import_workspace_into_database(workspace_name) + with tqdm(total=len(workspaces_list), + unit='B', unit_scale=True, unit_divisor=1024) as pbar: + for workspace_name in workspaces_list: + logger.debug(u'Setting up workspace {}'.format(workspace_name)) + + if not server.couchdb.server_has_access_to(workspace_name): + logger.error(u"Unauthorized access to CouchDB. Make sure faraday-server's"\ + " configuration file has CouchDB admin's credentials set") + sys.exit(1) + thread = threading.Thread(target=self.import_workspace_into_database, args=(workspace_name, )) + thread.daemon = True + thread.start() + workspace_threads.append(thread) + if len(workspace_threads) > multiprocessing.cpu_count(): + for thread in workspace_threads: + thread.join() + pbar.update(1) + workspace_threads.remove(thread) + + logger.info('Waiting for treads to finish.') + for thread in workspace_threads: + thread.join() + pbar.update(1) + #self.import_workspace_into_database(workspace_name) def get_objs(self, host, obj_type, level, workspace): if obj_type == 'Credential': From 6a49c405a0ec46c1825674518a6acfb6db501bd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 28 Feb 2018 18:10:19 -0300 Subject: [PATCH 0942/1506] Don't allow invalid service status in the plugins --- plugins/plugin.py | 8 +++++++- plugins/repo/sqlmap/plugin.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/plugins/plugin.py b/plugins/plugin.py index ac50f9bfbfc..41357c963a7 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -204,8 +204,14 @@ def createAndAddInterface( details="Interface object removed. Use host or service instead. Service will be attached to Host!") def createAndAddServiceToInterface(self, host_id, interface_id, name, protocol="tcp?", ports=[], - status="running", version="unknown", + status="open", version="unknown", description=""): + if status not in ("open", "closed", "filtered"): + self.log( + 'Unknown service status %s. Using "open" instead' % status, + 'WARNING' + ) + status = 'open' serv_obj = model.common.factory.createModelObject( Service.class_signature, diff --git a/plugins/repo/sqlmap/plugin.py b/plugins/repo/sqlmap/plugin.py index 1628f93d5c5..ea5225df862 100644 --- a/plugins/repo/sqlmap/plugin.py +++ b/plugins/repo/sqlmap/plugin.py @@ -469,7 +469,7 @@ def parseOutputString(self, output, debug=False): i_id, name=dbms_version, protocol="tcp", - status="down", + status="closed", version=str(dbms_version), ports=[str(db_port)], description="DB detect by SQLi") From 192e1725b43af30a4c338de1afd3c522738f60b1 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 1 Mar 2018 12:44:12 -0300 Subject: [PATCH 0943/1506] Add additional messsage --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 41c8e32fcb9..c1a3331f90a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -56,7 +56,7 @@ node (label: "master"){ } finally { junit "**/xunit.xml" - notifyBuild(currentBuild.result) + notifyBuild(currentBuild.result, "SQLite") if (testsError) { throw testsError } @@ -82,7 +82,7 @@ node (label: "master"){ } finally { junit "**/xunit-postgres.xml" - notifyBuild(currentBuild.result) + notifyBuild(currentBuild.result, "PostgreSQL") if (testsError) { throw testsError } @@ -91,7 +91,7 @@ node (label: "master"){ } } -def notifyBuild(String buildStatus = 'STARTED') { +def notifyBuild(String buildStatus = 'STARTED', String extraMessage = '') { // build status of null means successful buildStatus = buildStatus ?: 'SUCCESSFUL' @@ -99,7 +99,7 @@ def notifyBuild(String buildStatus = 'STARTED') { def colorName = 'RED' def colorCode = '#FF0000' def subject = "${buildStatus}: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'" - def summary = "${subject} (${env.BUILD_URL})" + def summary = "${subject} (${env.BUILD_URL}) " + extraMessage // Override default values based on build status if (buildStatus == 'STARTED') { From fd196f48f1147fd2d3f62f4ed57bab7e893eb293 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 1 Mar 2018 12:50:04 -0300 Subject: [PATCH 0944/1506] add build text --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c1a3331f90a..11716711bfa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -56,7 +56,7 @@ node (label: "master"){ } finally { junit "**/xunit.xml" - notifyBuild(currentBuild.result, "SQLite") + notifyBuild(currentBuild.result, "SQLite Build") if (testsError) { throw testsError } @@ -82,7 +82,7 @@ node (label: "master"){ } finally { junit "**/xunit-postgres.xml" - notifyBuild(currentBuild.result, "PostgreSQL") + notifyBuild(currentBuild.result, "PostgreSQL Build") if (testsError) { throw testsError } From c654c84afd45d1602a12f83bb62cc91509e4b93e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 1 Mar 2018 14:07:10 -0300 Subject: [PATCH 0945/1506] warn all the channel when the build is broken --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 11716711bfa..14aabd7352a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -111,6 +111,7 @@ def notifyBuild(String buildStatus = 'STARTED', String extraMessage = '') { } else { color = 'RED' colorCode = '#FF0000' + summary = summary + ' @channel' } // Send notifications From cd293ba992a7ff055dee78f8ff225fca6df6cc23 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 1 Mar 2018 14:19:13 -0300 Subject: [PATCH 0946/1506] showmessage was moved to commonsFact --- .../statusReport/controllers/statusReport.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index c4e80642c62..37f5db2f6e9 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -482,7 +482,7 @@ angular.module('faradayApp') return response.filter(x => !angular.equals(x, {})) }, function(failed) { - showMessage("Something failed searching vulnerability exploits."); + commonsFact.showMessage("Something failed searching vulnerability exploits."); return []; }); } @@ -518,12 +518,12 @@ angular.module('faradayApp') promises.push(self.vulnModelsManager.create(vuln, true)); }); $q.all(promises).then(function(success) { - showMessage("Created " + selected.length + " templates successfully.", true); + commonsFact.showMessage("Created " + selected.length + " templates successfully.", true); }, function(failedMessage) { - showMessage(failedMessage); + commonsFact.showMessage(failedMessage); }); } catch(err) { - showMessage("Something failed when creating some of the templates."); + commonsFact.showMessage("Something failed when creating some of the templates."); } }; @@ -667,7 +667,7 @@ angular.module('faradayApp') $scope.remove(vulns); }); } else { - showMessage('No vulnerabilities were selected to delete'); + commonsFact.showMessage('No vulnerabilities were selected to delete'); } }; @@ -687,7 +687,7 @@ angular.module('faradayApp') loadVulns(); } }, function(errorMsg){ - showMessage("Error updating vuln " + vuln.name + " (" + vuln._id + "): " + errorMsg); + commonsFact.showMessage("Error updating vuln " + vuln.name + " (" + vuln._id + "): " + errorMsg); }); }; @@ -721,7 +721,7 @@ angular.module('faradayApp') }); } else { - showMessage('A vulnerability must be selected in order to edit'); + commonsFact.showMessage('A vulnerability must be selected in order to edit'); } }; From ff1d8b2a680ea0bc147794aa6e9e4dc7116b5cef Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 1 Mar 2018 14:34:52 -0300 Subject: [PATCH 0947/1506] Avoid importer to use null values --- server/api/modules/vulns.py | 8 ++++---- server/importer.py | 8 ++++---- server/models.py | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 790d10ef764..4dcc89dab7e 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -71,10 +71,10 @@ def get_data(self, file_obj): class ImpactSchema(Schema): - accountability = fields.Boolean(attribute='impact_accountability') - availability = fields.Boolean(attribute='impact_availability') - confidentiality = fields.Boolean(attribute='impact_confidentiality') - integrity = fields.Boolean(attribute='impact_integrity') + accountability = fields.Boolean(attribute='impact_accountability', default=False) + availability = fields.Boolean(attribute='impact_availability', default=False) + confidentiality = fields.Boolean(attribute='impact_confidentiality', default=False) + integrity = fields.Boolean(attribute='impact_integrity', default=False) class CustomMetadataSchema(MetadataSchema): diff --git a/server/importer.py b/server/importer.py index dfb0cd3fe7f..c13ca94036e 100644 --- a/server/importer.py +++ b/server/importer.py @@ -573,10 +573,10 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation vulnerability.owned = document.get('owned', False) #vulnerability.attachments = json.dumps(document.get('_attachments', {})) - vulnerability.impact_accountability = document.get('impact', {}).get('accountability') - vulnerability.impact_availability = document.get('impact', {}).get('availability') - vulnerability.impact_confidentiality = document.get('impact', {}).get('confidentiality') - vulnerability.impact_integrity = document.get('impact', {}).get('integrity') + vulnerability.impact_accountability = document.get('impact', {}).get('accountability') or False + vulnerability.impact_availability = document.get('impact', {}).get('availability') or False + vulnerability.impact_confidentiality = document.get('impact', {}).get('confidentiality') or False + vulnerability.impact_integrity = document.get('impact', {}).get('integrity') or False if command and created: session.flush() CommandObject.create(vulnerability, command) diff --git a/server/models.py b/server/models.py index a1b3f7ee43a..cef757cdb00 100644 --- a/server/models.py +++ b/server/models.py @@ -394,10 +394,10 @@ class VulnerabilityABC(Metadata): severity = Column(Enum(*SEVERITIES, name='vulnerability_severity'), nullable=False) risk = Column(Float(3, 1), nullable=True) - impact_accountability = Column(Boolean, default=False) - impact_availability = Column(Boolean, default=False) - impact_confidentiality = Column(Boolean, default=False) - impact_integrity = Column(Boolean, default=False) + impact_accountability = Column(Boolean, default=False, nullable=False) + impact_availability = Column(Boolean, default=False, nullable=False) + impact_confidentiality = Column(Boolean, default=False, nullable=False) + impact_integrity = Column(Boolean, default=False, nullable=False) __table_args__ = ( CheckConstraint('1.0 <= risk AND risk <= 10.0', From 37079b4dfe7581656120bc02c55aa47676d45e7c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 1 Mar 2018 15:53:11 -0300 Subject: [PATCH 0948/1506] Avoid stopping the model controller is when a plugin is on queue --- model/application.py | 1 + model/controller.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/model/application.py b/model/application.py index 3bb5b65e59a..e002546efc7 100644 --- a/model/application.py +++ b/model/application.py @@ -191,6 +191,7 @@ def __exit(self, exit_code=0): model.api.devlog("stopping model controller thread...") model.api.stopAPIServer() restapi.stopServer() + self._model_controller.stop() self._model_controller.join() self.timer.stop() model.api.devlog("Waiting for controller threads to end...") diff --git a/model/controller.py b/model/controller.py index 21bf7fef1d9..a17ed032d6a 100644 --- a/model/controller.py +++ b/model/controller.py @@ -145,6 +145,7 @@ def __init__(self, mappers_manager, pending_actions): self._setupActionDispatcher() self.objects_with_updates = [] + self.processing = False def __getattr__(self, name): getLogger(self).debug("ModelObject attribute to refactor: %s" % name) @@ -325,7 +326,8 @@ def _main(self): This will make host addition and removal "thread-safe" and will avoid locking components that need to interact with the model """ - while not self._stop: + + while not self._stop or self.processing: # check if thread must finish # no plugin should be active to stop the controller if self._stop and self.active_plugins_count == 0: @@ -478,6 +480,7 @@ def addPluginEnd(self, name): self.__addPendingAction(modelactions.PLUGINEND, name) def _pluginStart(self, name, command_id): + self.processing = True self.active_plugins_count_lock.acquire() getLogger(self).info("Plugin Started: {0}. ".format(name, command_id)) self.active_plugins_count += 1 @@ -485,6 +488,7 @@ def _pluginStart(self, name, command_id): return True def _pluginEnd(self, name, command_id): + self.processing = False self.active_plugins_count_lock.acquire() getLogger(self).info("Plugin Ended: {0}".format(name)) self.active_plugins_count -= 1 From ef76c5e42eb99d40d7c777809c10a24b53c2a932 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 1 Mar 2018 17:47:33 -0300 Subject: [PATCH 0949/1506] Properly exit GTK if any plugin API request failed Detailed in #4563 --- model/controller.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/model/controller.py b/model/controller.py index a17ed032d6a..d39235b4b44 100644 --- a/model/controller.py +++ b/model/controller.py @@ -411,8 +411,11 @@ def find(self, obj_id): return self.mappers_manager.find(obj_id) def _save_new_object(self, new_object, command_id): - res = self.mappers_manager.save(new_object, command_id) - new_object.setID(res) + res = None + try: + res = self.mappers_manager.save(new_object, command_id) + finally: + new_object.setID(res) if res: notifier.addObject(new_object) return res @@ -436,6 +439,9 @@ def __add(self, new_obj, command_id=None, *args): old_obj = new_obj.__class__(conflict.answer.json()['object'], new_obj._workspace_name) new_obj.setID(old_obj.getID()) return self._handle_conflict(old_obj, new_obj, command_id) + except Exception: + new_obj.setID(None) + raise def __edit(self, obj, command_id=None, *args, **kwargs): obj.updateAttributes(*args, **kwargs) From 9ff7bfaa69e7e5b158dd99fdf9d43ed27567fdb3 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 1 Mar 2018 18:22:05 -0300 Subject: [PATCH 0950/1506] Remove _stop to avoid stopping the model controller --- model/controller.py | 1 - 1 file changed, 1 deletion(-) diff --git a/model/controller.py b/model/controller.py index a17ed032d6a..4af6ce47e1d 100644 --- a/model/controller.py +++ b/model/controller.py @@ -493,7 +493,6 @@ def _pluginEnd(self, name, command_id): getLogger(self).info("Plugin Ended: {0}".format(name)) self.active_plugins_count -= 1 self.active_plugins_count_lock.release() - self._stop = True return True def _devlog(self, msg, *args, **kwargs): From efaf227b43cb354a3ebf443b1147e08a73916964 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 2 Mar 2018 12:29:49 -0300 Subject: [PATCH 0951/1506] remove parenthesis --- server/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/models.py b/server/models.py index 36b7346c180..7abfd1fed6f 100644 --- a/server/models.py +++ b/server/models.py @@ -1475,9 +1475,9 @@ def parent(self): vulnerability_uniqueness = DDL( "CREATE UNIQUE INDEX uix_vulnerability ON %(fullname)s " - "(name, md5(description), COALESCE(host_id, -1), COALESCE(service_id, -1), " + "name, md5(description), COALESCE(host_id, -1), COALESCE(service_id, -1), " "COALESCE(method, ''), COALESCE(parameter_name, ''), COALESCE(path, ''), " - "COALESCE(website, ''), workspace_id, COALESCE(source_code_id, -1));" + "COALESCE(website, ''), workspace_id, COALESCE(source_code_id, -1);" ) vulnerability_uniqueness_sqlite = DDL( From fcdd5525612aea42f50e6636f6b7b0edf50b07fa Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 2 Mar 2018 12:35:43 -0300 Subject: [PATCH 0952/1506] remove parenthesis for sqlite only --- server/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/models.py b/server/models.py index 7abfd1fed6f..4f82fa07cbe 100644 --- a/server/models.py +++ b/server/models.py @@ -1475,16 +1475,16 @@ def parent(self): vulnerability_uniqueness = DDL( "CREATE UNIQUE INDEX uix_vulnerability ON %(fullname)s " - "name, md5(description), COALESCE(host_id, -1), COALESCE(service_id, -1), " + "(name, md5(description), COALESCE(host_id, -1), COALESCE(service_id, -1), " "COALESCE(method, ''), COALESCE(parameter_name, ''), COALESCE(path, ''), " - "COALESCE(website, ''), workspace_id, COALESCE(source_code_id, -1);" + "COALESCE(website, ''), workspace_id, COALESCE(source_code_id, -1));" ) vulnerability_uniqueness_sqlite = DDL( "CREATE UNIQUE INDEX uix_vulnerability ON %(fullname)s " - "(name, description, COALESCE(host_id, -1), COALESCE(service_id, -1), " + "name, description, COALESCE(host_id, -1), COALESCE(service_id, -1), " "COALESCE(method, ''), COALESCE(parameter_name, ''), COALESCE(path, ''), " - "COALESCE(website, ''), workspace_id, COALESCE(source_code_id, -1));" + "COALESCE(website, ''), workspace_id, COALESCE(source_code_id, -1);" ) From 01b3b6a673e7a47f05135300ee60e3261d3b7ab2 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 2 Mar 2018 14:46:57 -0300 Subject: [PATCH 0953/1506] there was an issue with an old sqlite3 version --- server/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/models.py b/server/models.py index 4f82fa07cbe..36b7346c180 100644 --- a/server/models.py +++ b/server/models.py @@ -1482,9 +1482,9 @@ def parent(self): vulnerability_uniqueness_sqlite = DDL( "CREATE UNIQUE INDEX uix_vulnerability ON %(fullname)s " - "name, description, COALESCE(host_id, -1), COALESCE(service_id, -1), " + "(name, description, COALESCE(host_id, -1), COALESCE(service_id, -1), " "COALESCE(method, ''), COALESCE(parameter_name, ''), COALESCE(path, ''), " - "COALESCE(website, ''), workspace_id, COALESCE(source_code_id, -1);" + "COALESCE(website, ''), workspace_id, COALESCE(source_code_id, -1));" ) From 40e0ee4e86f33079568e04e2666843c64ff18f07 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 2 Mar 2018 15:01:17 -0300 Subject: [PATCH 0954/1506] Add release note for dirb plugin --- RELEASE.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index ed48d9c7146..07fc855f80d 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -8,6 +8,11 @@ Make sure you run ```./faraday.py --update``` the first time after an update! New features in the latest update ===================================== +TBA +--- + +* dirb plugin should creates a vulnerability type information instead of a note. + November 17, 2017: --- * Fix bug with tags in models. From b3c34fe528d99877c1c84b7e7f6a65426084a9ec Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 2 Mar 2018 15:03:59 -0300 Subject: [PATCH 0955/1506] Add release note for new plugin --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index 07fc855f80d..da813ef7f9a 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -12,6 +12,7 @@ TBA --- * dirb plugin should creates a vulnerability type information instead of a note. +* Add new plugin ip360 November 17, 2017: --- From b7ab9d22113f0f9a0db03db5965bb3244d4d6bea Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 2 Mar 2018 15:06:48 -0300 Subject: [PATCH 0956/1506] Add release note for new column in webui (csv) --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index da813ef7f9a..cb66fc02e08 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -13,6 +13,7 @@ TBA * dirb plugin should creates a vulnerability type information instead of a note. * Add new plugin ip360 +* Add confirmed column to exported csv from webui November 17, 2017: --- From 0739cc581e6e4b231e4b72da31e54ed1a87314d7 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 2 Mar 2018 15:09:06 -0300 Subject: [PATCH 0957/1506] Add release note for Fixes in Arachni plugin --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index cb66fc02e08..29474f7adc1 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -14,6 +14,7 @@ TBA * dirb plugin should creates a vulnerability type information instead of a note. * Add new plugin ip360 * Add confirmed column to exported csv from webui +* Fixes in Arachni plugin November 17, 2017: --- From 508646de261fe8a0f40a91abd5ef3575e5790814 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 2 Mar 2018 15:12:04 -0300 Subject: [PATCH 0958/1506] Add release note for new parameters for faraday cli --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index 29474f7adc1..e17b73320ad 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -15,6 +15,7 @@ TBA * Add new plugin ip360 * Add confirmed column to exported csv from webui * Fixes in Arachni plugin +* Add new parameters --keep-old and --keep-new for faraday CLI November 17, 2017: --- From 3df40781f7ecfe93fa30ef827092580c7e17b0e4 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 2 Mar 2018 15:23:51 -0300 Subject: [PATCH 0959/1506] Add release notes for new fplugin and fix typo --- RELEASE.md | 1 + ...eenshot_server.py => screenshot_server.py} | 47 +++++++++---------- requirements_extras.txt | 3 +- 3 files changed, 26 insertions(+), 25 deletions(-) rename bin/{scheenshot_server.py => screenshot_server.py} (62%) diff --git a/RELEASE.md b/RELEASE.md index e17b73320ad..5d2d9109ab6 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -16,6 +16,7 @@ TBA * Add confirmed column to exported csv from webui * Fixes in Arachni plugin * Add new parameters --keep-old and --keep-new for faraday CLI +* Add new screenshot fplugin which takes a screenshot of the ip:ports of a given protocol November 17, 2017: --- diff --git a/bin/scheenshot_server.py b/bin/screenshot_server.py similarity index 62% rename from bin/scheenshot_server.py rename to bin/screenshot_server.py index f191fbdf019..714baed30c7 100644 --- a/bin/scheenshot_server.py +++ b/bin/screenshot_server.py @@ -5,28 +5,28 @@ Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) See the file 'doc/LICENSE' for the license information """ -from __future__ import print_function -from persistence.server.server_io_exceptions import ResourceDoesNotExist -from persistence.server import models -from utils.user_input import query_yes_no +from __future__ import print_function import os +import sys +from persistence.server.server_io_exceptions import ResourceDoesNotExist +from persistence.server import models try: from selenium import webdriver -except Exception: - print ("Missing dependencies: (selenium)") - +except ImportError: + print ("Missing dependencies: (selenium). install it with pip install selenium. ") + sys.exit(1) +__description__ = 'Takes a Screenshot of the ip:ports of a given protocol' +__prettyname__ = 'Screenshot_server' -__description__ = 'Takes a Schreenshot of the ip:ports of a given protocol' -__prettyname__ = 'Scheenshot_server' -def scheenshot(path, protocol, ip, port): +def screenshot(path, protocol, ip, port): driver = webdriver.PhantomJS() - driver.set_window_size(1024, 768) # set the window size that you need + driver.set_window_size(1024, 768) # set the window size that you need driver.set_page_load_timeout(5) try: driver.get(protocol + "://" + ip + ":" + port + "/") - driver.get_screenshot_as_file (os.path.join( path , ip + "_" + port + ".png")) + driver.get_screenshot_as_file(os.path.join(path, ip + "_" + port + ".png")) except Exception: print("Coudn't connect") finally: @@ -34,39 +34,38 @@ def scheenshot(path, protocol, ip, port): return 0 + def main(workspace='', args=None, parser=None): - parser.add_argument( 'protocol', help="Desired protocol" , default="") - parser.add_argument( '--path', help="Saves the Image in a given path", default="." ) + parser.add_argument('protocol', help="Desired protocol", default="") + parser.add_argument('--path', help="Saves the Image in a given path", default=".") parsed_args = parser.parse_args(args) protocols = parsed_args.protocol.split(",") print (protocols) path = parsed_args.path - + for protocol in protocols: - + if not os.path.exists(path): print ("Invalid Path") exit() - - + try: services = models.get_services(workspace) except ResourceDoesNotExist: print ("Invalid workspace name: ", workspace) return 1, None - for service in services: service_protocol = service.protocol.lower() - + if service_protocol == protocol: port = str(service.ports[0]) - + interface_id = ".".join(service.id.split(".")[:2]) interface = models.get_interface(workspace, interface_id) ip = interface.ipv4["address"] - + print (protocol + "://" + ip + ":" + port) - scheenshot(path, protocol , ip, port ) - return 0, None \ No newline at end of file + screenshot(path, protocol, ip, port) + return 0, None diff --git a/requirements_extras.txt b/requirements_extras.txt index 373b7543794..5389e69f17d 100644 --- a/requirements_extras.txt +++ b/requirements_extras.txt @@ -1,3 +1,4 @@ beautifulsoup4>=4.6.0 psycopg2>=2.7.3 -w3af_api_client>=1.1.7 \ No newline at end of file +w3af_api_client>=1.1.7 +selenium==3.9.0 From 76e9187637b9f503bdc49da80ab9a218cd8c1d1e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 2 Mar 2018 15:31:25 -0300 Subject: [PATCH 0960/1506] Add release note for net sparker --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index 5d2d9109ab6..85093bd23b4 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -17,6 +17,7 @@ TBA * Fixes in Arachni plugin * Add new parameters --keep-old and --keep-new for faraday CLI * Add new screenshot fplugin which takes a screenshot of the ip:ports of a given protocol +* Add fix for net sparker cloud fix on severity November 17, 2017: --- From a1fc03f9c5eb1021beebf1212aee7f2d6b8b6fc0 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 2 Mar 2018 15:38:30 -0300 Subject: [PATCH 0961/1506] Add release notes for netsparker regular --- RELEASE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index 85093bd23b4..2da1d30663b 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -17,7 +17,7 @@ TBA * Fixes in Arachni plugin * Add new parameters --keep-old and --keep-new for faraday CLI * Add new screenshot fplugin which takes a screenshot of the ip:ports of a given protocol -* Add fix for net sparker cloud fix on severity +* Add fix for net sparker regular and cloud fix on severity November 17, 2017: --- From 4ba5f94d61851a20176f07d7ac3ab1d321d6b703 Mon Sep 17 00:00:00 2001 From: Winna_Z Date: Fri, 2 Mar 2018 17:53:45 -0300 Subject: [PATCH 0962/1506] fixed encoding --- plugins/repo/netsparker/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/repo/netsparker/plugin.py b/plugins/repo/netsparker/plugin.py index 9c45d86d9aa..79cc736f0e8 100644 --- a/plugins/repo/netsparker/plugin.py +++ b/plugins/repo/netsparker/plugin.py @@ -121,9 +121,9 @@ def __init__(self, item_node): self.request = self.get_text_from_subnode("rawrequest") self.response = self.get_text_from_subnode("rawresponse") if self.response: - self.response = self.response.encode("base64")[:-1] + self.response = self.response.encode("ascii",errors="backslashreplace") if self.request: - self.request = self.request.encode("base64")[:-1] + self.request = self.request.encode("ascii",errors="backslashreplace") self.kvulns = [] for v in self.node.findall("knownvulnerabilities/knownvulnerability"): From 92ade8cca7540a6378810b72345a31727aa8d670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 5 Mar 2018 13:58:54 -0300 Subject: [PATCH 0963/1506] Fix syntax error in CSV provider --- server/www/scripts/csv/providers/csv.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/www/scripts/csv/providers/csv.js b/server/www/scripts/csv/providers/csv.js index 61d4a062493..91d679cee81 100644 --- a/server/www/scripts/csv/providers/csv.js +++ b/server/www/scripts/csv/providers/csv.js @@ -31,8 +31,9 @@ angular.module('faradayApp') if(typeof(v[prop]) === "object") v[prop] = parseObject(v[prop]); if(typeof(v[prop]) != "undefined" && v[prop] != null && typeof(v[prop]) != "number" && prop != "confirmed") { object[prop] = cleanCSV(v[prop]); - else + }else{ object[prop] = ""; + } if(prop === "confirmed") object[prop] = v[prop].toString(); if(prop === "date") object[prop] = parseDate(v["metadata"]["create_time"] * 1000); From 4055afb5d5ad4487c560902b5e05bb7a3ef5a418 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 5 Mar 2018 15:01:06 -0300 Subject: [PATCH 0964/1506] Fix a syntax error on csv --- server/www/scripts/csv/providers/csv.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/www/scripts/csv/providers/csv.js b/server/www/scripts/csv/providers/csv.js index 61d4a062493..5bda007557c 100644 --- a/server/www/scripts/csv/providers/csv.js +++ b/server/www/scripts/csv/providers/csv.js @@ -31,8 +31,10 @@ angular.module('faradayApp') if(typeof(v[prop]) === "object") v[prop] = parseObject(v[prop]); if(typeof(v[prop]) != "undefined" && v[prop] != null && typeof(v[prop]) != "number" && prop != "confirmed") { object[prop] = cleanCSV(v[prop]); - else + } else { object[prop] = ""; + } + if(prop === "confirmed") object[prop] = v[prop].toString(); if(prop === "date") object[prop] = parseDate(v["metadata"]["create_time"] * 1000); From 508982bb2cf417e145d5d932de9e12fd0e234324 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 5 Mar 2018 18:36:21 -0300 Subject: [PATCH 0965/1506] Fix a bug found using hypothesis --- server/api/modules/vulns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 4dcc89dab7e..bc323ef6311 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -462,7 +462,7 @@ def _get_schema_class(self): if request.method == 'POST': requested_type = request.json.get('type', None) if not requested_type: - raise ValidationError('Type is required.') + raise InvalidUsage('Type is required.') if requested_type not in self.schema_class_dict: raise InvalidUsage('Invalid vulnerability type.') return self.schema_class_dict[requested_type] From f47048f2300db6f4982eecdb7c836aefbff53122 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 6 Mar 2018 13:28:35 -0300 Subject: [PATCH 0966/1506] Fix XML API. Cleanup code and fix some PEP8 warnings. * XML API was using some deleted ASYNC functions. now XML API feeds the model controller queue directly. * Move Modelactions and rename the class * Removed import * * Add parent_type for Notes --- model/__init__.py | 72 +++++++++++++++++ model/api.py | 55 ++++++++----- model/controller.py | 151 ++++++++++------------------------- persistence/server/models.py | 11 +-- plugins/controller.py | 32 ++++---- plugins/plugin.py | 52 ++++++++---- scripts/shodan_faraday.py | 26 +++--- server/api/modules/vulns.py | 1 - 8 files changed, 221 insertions(+), 179 deletions(-) diff --git a/model/__init__.py b/model/__init__.py index 9b5f4486690..2852160385e 100644 --- a/model/__init__.py +++ b/model/__init__.py @@ -5,3 +5,75 @@ ''' +class Modelactions: + ADDHOST = 2000 + DELHOST = 2001 + ADDSERVICEHOST = 2008 + ADDSERVICEHOST = 20008 + ADDCATEGORY = 2011 # TODO migration: check why isn't implemented + ADDVULNHOST = 2017 + DELVULNHOST = 2018 + ADDVULNSRV = 2019 + DELVULNSRV = 2020 + ADDNOTEHOST = 2025 + DELNOTEHOST = 2026 + ADDNOTESRV = 2027 + DELNOTESRV = 2028 + RENAMEROOT = 2029 # TODO migration: check why isn't implemented + ADDNOTEVULN = 2030 + DELNOTEVULN = 2031 # TODO migration: check why isn't implemented + EDITHOST = 2032 + EDITSERVICE = 2035 + ADDCREDSRV = 2036 + DELCREDSRV = 2037 + ADDVULNWEBSRV = 2038 + DELVULNWEBSRV = 2039 # TODO migration: check why isn't implemented + ADDNOTENOTE = 2040 + DELNOTENOTE = 2041 # TODO migration: check why isn't implemented + EDITNOTE = 2042 + EDITVULN = 2043 + ADDNOTE = 2044 + DELNOTE = 2045 + ADDVULN = 2046 + DELVULN = 2047 + EDITCRED = 2048 + ADDCRED = 2049 + DELCRED = 2050 + PLUGINSTART = 3000 + PLUGINEND = 3001 + LOG = 3002 + DEVLOG = 3003 + + __descriptions = { + ADDHOST: "ADDHOST", + DELHOST: "DELHOST", + ADDCATEGORY: "ADDCATEGORY", + ADDVULNHOST: "ADDVULNHOST", + DELVULNHOST: "DELVULNHOST", + ADDVULNSRV: "ADDVULNSRV", + DELVULNSRV: "DELVULNSRV", + ADDNOTEVULN: "ADDNOTEVULN", + DELNOTEVULN: "DELNOTEVULN", + ADDNOTENOTE: "ADDNOTENOTE", + DELNOTENOTE: "DELNOTENOTE", + EDITHOST: "EDITHOST", + ADDCREDSRV: "ADDCREDSRV", + DELCREDSRV: "DELCREDSRV", + ADDVULNWEBSRV: "ADDVULNSWEBRV", + DELVULNWEBSRV: "DELVULNWEBSRV", + EDITNOTE: "EDITNOTE", + EDITVULN: "EDITVULN", + EDITCRED: "EDITCRED", + ADDNOTE: "ADDNOTE", + DELNOTE: "DELNOTE", + ADDVULN: "ADDVULN", + DELVULN: "DELVULN", + ADDCRED: "ADDCRED", + DELCRED: "DELCRED", + PLUGINSTART: "PLUGINSTART", + PLUGINEND: "PLUGINEND" + } + + @staticmethod + def getDescription(action): + return modelactions.__descriptions.get(action, "") \ No newline at end of file diff --git a/model/api.py b/model/api.py index 2e61b9ba2f5..12893c9a5ce 100644 --- a/model/api.py +++ b/model/api.py @@ -7,16 +7,15 @@ ''' import os -import socket -import zipfile import logging import model.common from config.configuration import getInstanceConfiguration #from workspace import Workspace import model.log +from model import Modelactions from utils.logs import getLogger -from utils.common import * +from utils.common import socket, gateway import shutil #from plugins.api import PluginControllerAPI @@ -90,6 +89,7 @@ def _setUpAPIServer(hostname=None, port=None): # register all the api functions to be exposed by the server _xmlrpc_api_server.register_function(createAndAddHost) + _xmlrpc_api_server.register_function(createAndAddServiceToHost) _xmlrpc_api_server.register_function(createAndAddNoteToService) _xmlrpc_api_server.register_function(createAndAddNoteToHost) _xmlrpc_api_server.register_function(createAndAddNoteToNote) @@ -139,6 +139,17 @@ def createAndAddHost(ip, os="Unknown"): return None +def createAndAddServiceToHost(host_id, name, + protocol="tcp?", ports=[], + status="open", version="unknown", + description=""): + service = newService(name, protocol, ports, status, version, description, host_id) + + if addServiceToHost(service): + return service.getID() + return None + + # Vulnerability def createAndAddVulnToHost(host_id, name, desc, ref, severity, resolution): vuln = newVuln(name, desc, ref, severity, resolution, parent_id=host_id) @@ -168,20 +179,20 @@ def createAndAddVulnWebToService(host_id, service_id, name, desc, ref, severity, # Note def createAndAddNoteToHost(host_id, name, text): - note = newNote(name, text, parent_id=host_id) + note = newNote(name, text, parent_id=host_id, parent_type='host') if addNoteToHost(host_id, note): return note.getID() return None def createAndAddNoteToService(host_id, service_id, name, text): - note = newNote(name, text, parent_id=service_id) + note = newNote(name, text, parent_id=service_id, parent_type='service') if addNoteToService(host_id, service_id, note): return note.getID() return None def createAndAddNoteToNote(host_id, service_id, note_id, name, text): - note = newNote(name, text, parent_id=note_id) + note = newNote(name, text, parent_id=note_id, parent_type='comment') if addNoteToNote(host_id, service_id, note_id, note): return note.getID() return None @@ -200,33 +211,37 @@ def createAndAddCredToService(host_id, service_id, username, password): def addHost(host): if host is not None: - __model_controller.addHostASYNC(host) + __model_controller.add_action((Modelactions.ADDHOST, host)) + #addHostASYNC(host) return True return False -def addServiceToHost(host_id, service): - pass +def addServiceToHost(service): + if service is not None: + __model_controller.add_action((Modelactions.ADDSERVICEHOST, service)) + return True + return False # Vulnerability def addVulnToHost(host_id, vuln): if vuln is not None: - __model_controller.addVulnToHostASYNC(host_id, vuln) + __model_controller.add_action((Modelactions.ADDVULNHOST, vuln)) return True return False def addVulnToService(host_id, service_id, vuln): if vuln is not None: - __model_controller.addVulnToServiceASYNC(host_id, service_id, vuln) + __model_controller.add_action((Modelactions.ADDVULNSRV, vuln)) return True return False #VulnWeb def addVulnWebToService(host_id, service_id, vuln): if vuln is not None: - __model_controller.addVulnWebToServiceASYNC(host_id, service_id, vuln) + __model_controller.add_action((Modelactions.ADDVULNWEBSRV, vuln)) return True return False @@ -234,33 +249,35 @@ def addVulnWebToService(host_id, service_id, vuln): # Notes def addNoteToHost(host_id, note): if note is not None: - __model_controller.addNoteToHostASYNC(host_id, note) + __model_controller.add_action( + (Modelactions.ADDNOTEHOST, note)) return True return False def addNoteToService(host_id, service_id, note): if note is not None: - __model_controller.addNoteToServiceASYNC(host_id, service_id, note) + __model_controller.add_action( + (Modelactions.ADDNOTESRV, note)) return True return False def addNoteToNote(host_id, service_id, note_id, note): if note is not None: - __model_controller.addNoteToNoteASYNC(host_id, service_id, note_id, note) + __model_controller.add_action((Modelactions.ADDNOTENOTE, note)) return True return False def addCredToService(host_id, service_id, cred): if cred is not None: - __model_controller.addCredToServiceASYNC(host_id, service_id, cred) + __model_controller.add_action((Modelactions.ADDCREDSRV, cred)) return True return False #------------------------------------------------------------------------------- # APIs to delete elements to model #------------------------------------------------------------------------------- -#TODO: delete funcitons are still missing +#TODO: delete functions are still missing def delHost(hostname): __model_controller.delHostASYNC(hostname) return True @@ -343,12 +360,12 @@ def newVulnWeb(name, desc="", ref=None, severity="", resolution="", website="", parent_id) -def newNote(name, text, parent_id=None): +def newNote(name, text, parent_id=None, parent_type=None): """ It creates and returns a Note object. The created object is not added to the model. """ - return __model_controller.newNote(name, text, parent_id) + return __model_controller.newNote(name, text, parent_id, parent_type) def newCred(username, password, parent_id=None): diff --git a/model/controller.py b/model/controller.py index 5515e06aec4..5d305b619cf 100644 --- a/model/controller.py +++ b/model/controller.py @@ -13,6 +13,7 @@ from threading import Thread from config.configuration import getInstanceConfiguration +from model import Modelactions from persistence.server.server_io_exceptions import ConflictInDatabase from utils.logs import getLogger import model.api as api @@ -28,80 +29,6 @@ logger = logging.getLogger(__name__) -class modelactions: - ADDHOST = 2000 - DELHOST = 2001 - ADDSERVICEHOST = 2008 - ADDSERVICEHOST = 20008 - ADDCATEGORY = 2011 # TODO migration: check why isn't implemented - ADDVULNHOST = 2017 - DELVULNHOST = 2018 - ADDVULNSRV = 2019 - DELVULNSRV = 2020 - ADDNOTEHOST = 2025 - DELNOTEHOST = 2026 - ADDNOTESRV = 2027 - DELNOTESRV = 2028 - RENAMEROOT = 2029 # TODO migration: check why isn't implemented - ADDNOTEVULN = 2030 - DELNOTEVULN = 2031 # TODO migration: check why isn't implemented - EDITHOST = 2032 - EDITSERVICE = 2035 - ADDCREDSRV = 2036 - DELCREDSRV = 2037 - ADDVULNWEBSRV = 2038 - DELVULNWEBSRV = 2039 # TODO migration: check why isn't implemented - ADDNOTENOTE = 2040 - DELNOTENOTE = 2041 # TODO migration: check why isn't implemented - EDITNOTE = 2042 - EDITVULN = 2043 - ADDNOTE = 2044 - DELNOTE = 2045 - ADDVULN = 2046 - DELVULN = 2047 - EDITCRED = 2048 - ADDCRED = 2049 - DELCRED = 2050 - PLUGINSTART = 3000 - PLUGINEND = 3001 - LOG = 3002 - DEVLOG = 3003 - - __descriptions = { - ADDHOST: "ADDHOST", - DELHOST: "DELHOST", - ADDCATEGORY: "ADDCATEGORY", - ADDVULNHOST: "ADDVULNHOST", - DELVULNHOST: "DELVULNHOST", - ADDVULNSRV: "ADDVULNSRV", - DELVULNSRV: "DELVULNSRV", - ADDNOTEVULN: "ADDNOTEVULN", - DELNOTEVULN: "DELNOTEVULN", - ADDNOTENOTE: "ADDNOTENOTE", - DELNOTENOTE: "DELNOTENOTE", - EDITHOST: "EDITHOST", - ADDCREDSRV: "ADDCREDSRV", - DELCREDSRV: "DELCREDSRV", - ADDVULNWEBSRV: "ADDVULNSWEBRV", - DELVULNWEBSRV: "DELVULNWEBSRV", - EDITNOTE: "EDITNOTE", - EDITVULN: "EDITVULN", - EDITCRED: "EDITCRED", - ADDNOTE: "ADDNOTE", - DELNOTE: "DELNOTE", - ADDVULN: "ADDVULN", - DELVULN: "DELVULN", - ADDCRED: "ADDCRED", - DELCRED: "DELCRED", - PLUGINSTART: "PLUGINSTART", - PLUGINEND: "PLUGINEND" - } - - @staticmethod - def getDescription(action): - return modelactions.__descriptions.get(action, "") - - class ModelController(Thread): def __init__(self, mappers_manager, pending_actions): @@ -196,41 +123,41 @@ def addWrapper(new_obj, parent_id=None, *args): def _setupActionDispatcher(self): self._actionDispatcher = { - modelactions.ADDHOST: self.__add, - modelactions.DELHOST: self.__del, - modelactions.EDITHOST: self.__edit, - modelactions.EDITSERVICE: self.__edit, + Modelactions.ADDHOST: self.__add, + Modelactions.DELHOST: self.__del, + Modelactions.EDITHOST: self.__edit, + Modelactions.EDITSERVICE: self.__edit, # Vulnerability - modelactions.ADDVULNHOST: self.__add, - modelactions.DELVULNHOST: self.__del, - modelactions.ADDVULNSRV: self.__add, - modelactions.DELVULNSRV: self.__del, - modelactions.ADDVULN: self.__add, - modelactions.DELVULN: self.__del, - modelactions.ADDVULNWEBSRV: self.__add, - modelactions.EDITVULN: self.__edit, + Modelactions.ADDVULNHOST: self.__add, + Modelactions.DELVULNHOST: self.__del, + Modelactions.ADDVULNSRV: self.__add, + Modelactions.DELVULNSRV: self.__del, + Modelactions.ADDVULN: self.__add, + Modelactions.DELVULN: self.__del, + Modelactions.ADDVULNWEBSRV: self.__add, + Modelactions.EDITVULN: self.__edit, #Service - modelactions.ADDSERVICEHOST: self.__add, + Modelactions.ADDSERVICEHOST: self.__add, # Note - modelactions.ADDNOTEHOST: self.__add, - modelactions.DELNOTEHOST: self.__del, - modelactions.ADDNOTESRV: self.__add, - modelactions.DELNOTESRV: self.__del, - modelactions.ADDNOTEVULN: self.__add, - modelactions.ADDNOTE: self.__add, - modelactions.DELNOTE: self.__del, - modelactions.ADDCREDSRV: self.__add, - modelactions.DELCREDSRV: self.__del, - modelactions.ADDNOTENOTE: self.__add, - modelactions.EDITNOTE: self.__edit, - modelactions.EDITCRED: self.__edit, - modelactions.ADDCRED: self.__add, - modelactions.DELCRED: self.__del, + Modelactions.ADDNOTEHOST: self.__add, + Modelactions.DELNOTEHOST: self.__del, + Modelactions.ADDNOTESRV: self.__add, + Modelactions.DELNOTESRV: self.__del, + Modelactions.ADDNOTEVULN: self.__add, + Modelactions.ADDNOTE: self.__add, + Modelactions.DELNOTE: self.__del, + Modelactions.ADDCREDSRV: self.__add, + Modelactions.DELCREDSRV: self.__del, + Modelactions.ADDNOTENOTE: self.__add, + Modelactions.EDITNOTE: self.__edit, + Modelactions.EDITCRED: self.__edit, + Modelactions.ADDCRED: self.__add, + Modelactions.DELCRED: self.__del, # Plugin states - modelactions.PLUGINSTART: self._pluginStart, - modelactions.PLUGINEND: self._pluginEnd, - modelactions.DEVLOG: self._devlog, - modelactions.LOG: self._log, + Modelactions.PLUGINSTART: self._pluginStart, + Modelactions.PLUGINEND: self._pluginEnd, + Modelactions.DEVLOG: self._devlog, + Modelactions.LOG: self._log, } def run(self): @@ -379,6 +306,9 @@ def sync_unlock(self): # TODO: >>> APIs <<< we have to know which plugin called the apis to store # in the history + def add_action(self, action): + self._pending_actions.put(action) + def __addPendingAction(self, *args): """ Adds a new pending action to the queue @@ -439,7 +369,8 @@ def __add(self, new_obj, command_id=None, *args): old_obj = new_obj.__class__(conflict.answer.json()['object'], new_obj._workspace_name) new_obj.setID(old_obj.getID()) return self._handle_conflict(old_obj, new_obj, command_id) - except Exception: + except Exception as ex: + logger.exception(ex) new_obj.setID(None) raise @@ -480,10 +411,10 @@ def __editService(self, service, name=None, description=None, return res def addPluginStart(self, name): - self.__addPendingAction(modelactions.PLUGINSTART, name) + self.__addPendingAction(Modelactions.PLUGINSTART, name) def addPluginEnd(self, name): - self.__addPendingAction(modelactions.PLUGINEND, name) + self.__addPendingAction(Modelactions.PLUGINEND, name) def _pluginStart(self, name, command_id): self.processing = True @@ -542,10 +473,10 @@ def newVulnWeb(self, name, desc="", ref=None, severity="", resolution="", method=method, pname=pname, params=params, query=query, category=category, confirmed=confirmed, parent_id=parent_id) - def newNote(self, name, text, parent_id=None): + def newNote(self, name, text, parent_id=None, parent_type=None): return model.common.factory.createModelObject( models.Note.class_signature, name, - workspace_name=self.mappers_manager.workspace_name, text=text, parent_id=parent_id) + workspace_name=self.mappers_manager.workspace_name, text=text, parent_id=parent_id, parent_type=parent_type) def newCred(self, username, password, parent_id=None): return model.common.factory.createModelObject( diff --git a/persistence/server/models.py b/persistence/server/models.py index 88c55c9eb51..f05d4df3299 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -739,8 +739,9 @@ def setParentType(self, parent_type): def setID(self, id): print('setID {0}'.format(id)) - self.id = id - self.id_available.set() + if id: + self.id = id + self.id_available.set() def getID(self): # getId will wait until the id is not None @@ -748,8 +749,8 @@ def getID(self): retries = 1 max_retries = 6 while retries <= max_retries and self.id is None: - self.id_available.wait(timeout=timeout) print('Retrying getID timeout {0}'.format(timeout)) + self.id_available.wait(timeout=timeout) timeout = timeout << retries - 1 retries += 1 return self.id @@ -1307,8 +1308,8 @@ class Note(ModelBase): def __init__(self, note, workspace_name): ModelBase.__init__(self, note, workspace_name) self.text = note['text'] - self.object_id = note['object_id'] - self.object_type = note['object_type'] + self.object_id = note.get('object_id') or note.get('parent') + self.object_type = note.get('object_type') or note.get('parent_type') def updateAttributes(self, name=None, text=None): if name is not None: diff --git a/plugins/controller.py b/plugins/controller.py index 88a83b563cc..1b51799a522 100644 --- a/plugins/controller.py +++ b/plugins/controller.py @@ -18,7 +18,7 @@ from plugins.plugin import PluginProcess import model.api from model.commands_history import CommandRunInformation -from model.controller import modelactions +from model import Modelactions from utils.logs import getLogger from config.globals import ( @@ -50,7 +50,7 @@ def stop(self): def commit(self): logger.debug('Plugin end. Commiting to faraday server.') self.pending_actions.put( - (modelactions.PLUGINEND, self.plugin.id, self.command.getID())) + (Modelactions.PLUGINEND, self.plugin.id, self.command.getID())) self.command.duration = time.time() - self.command.itime self.mapper_manager.update(self.command) @@ -165,7 +165,7 @@ def processOutput(self, plugin, output, command, isReport=False): "Created plugin_process (%d) for plugin instance (%d)" % (id(plugin_process), id(plugin))) - self.pending_actions.put((modelactions.PLUGINSTART, plugin.id, command.getID())) + self.pending_actions.put((Modelactions.PLUGINSTART, plugin.id, command.getID())) output_queue.put((output, command.getID())) plugin_commiter = PluginCommiter( @@ -192,25 +192,25 @@ def _processAction(self, action, parameters): def _setupActionDispatcher(self): self._actionDispatcher = { - modelactions.ADDHOST: model.api.addHost, - modelactions.ADDSERVICEHOST: model.api.addServiceToHost, + Modelactions.ADDHOST: model.api.addHost, + Modelactions.ADDSERVICEHOST: model.api.addServiceToHost, #Vulnerability - modelactions.ADDVULNHOST: model.api.addVulnToHost, - modelactions.ADDVULNSRV: model.api.addVulnToService, + Modelactions.ADDVULNHOST: model.api.addVulnToHost, + Modelactions.ADDVULNSRV: model.api.addVulnToService, #VulnWeb - modelactions.ADDVULNWEBSRV: model.api.addVulnWebToService, + Modelactions.ADDVULNWEBSRV: model.api.addVulnWebToService, #Note - modelactions.ADDNOTEHOST: model.api.addNoteToHost, - modelactions.ADDNOTESRV: model.api.addNoteToService, - modelactions.ADDNOTENOTE: model.api.addNoteToNote, + Modelactions.ADDNOTEHOST: model.api.addNoteToHost, + Modelactions.ADDNOTESRV: model.api.addNoteToService, + Modelactions.ADDNOTENOTE: model.api.addNoteToNote, #Creds - modelactions.ADDCREDSRV: model.api.addCredToService, + Modelactions.ADDCREDSRV: model.api.addCredToService, #LOG - modelactions.LOG: model.api.log, - modelactions.DEVLOG: model.api.devlog, + Modelactions.LOG: model.api.log, + Modelactions.DEVLOG: model.api.devlog, # Plugin state - modelactions.PLUGINSTART: model.api.pluginStart, - modelactions.PLUGINEND: model.api.pluginEnd + Modelactions.PLUGINSTART: model.api.pluginStart, + Modelactions.PLUGINEND: model.api.pluginEnd } def updatePluginSettings(self, plugin_id, new_settings): diff --git a/plugins/plugin.py b/plugins/plugin.py index 41357c963a7..9be3eb8ff9f 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -28,7 +28,7 @@ Credential, Note, ) -from model.controller import modelactions +from model import Modelactions #from plugins.modelactions import modelactions from config.configuration import getInstanceConfiguration @@ -181,7 +181,7 @@ def createAndAddHost(self, name, os="unknown"): workspace_name=self.workspace) host_obj._metadata.creatoserverr = self.id - self.__addPendingAction(modelactions.ADDHOST, host_obj) + self.__addPendingAction(Modelactions.ADDHOST, host_obj) return host_obj.getID() @deprecation.deprecated(deprecated_in="3.0", removed_in="3.5", @@ -217,11 +217,33 @@ def createAndAddServiceToInterface(self, host_id, interface_id, name, Service.class_signature, name, protocol=protocol, ports=ports, status=status, version=version, description=description, - parent_type='Service', parent_id=host_id, + parent_type='Host', parent_id=host_id, + workspace_name=self.workspace) + + serv_obj._metadata.creator = self.id + self.__addPendingAction(Modelactions.ADDSERVICEHOST, serv_obj) + return serv_obj.getID() + + def createAndAddServiceToHost(self, host_id, name, + protocol="tcp?", ports=[], + status="open", version="unknown", + description=""): + if status not in ("open", "closed", "filtered"): + self.log( + 'Unknown service status %s. Using "open" instead' % status, + 'WARNING' + ) + status = 'open' + + serv_obj = model.common.factory.createModelObject( + Service.class_signature, + name, protocol=protocol, ports=ports, status=status, + version=version, description=description, + parent_type='Host', parent_id=host_id, workspace_name=self.workspace) serv_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDSERVICEHOST, serv_obj) + self.__addPendingAction(Modelactions.ADDSERVICEHOST, serv_obj) return serv_obj.getID() def createAndAddVulnToHost(self, host_id, name, desc="", ref=[], @@ -235,7 +257,7 @@ def createAndAddVulnToHost(self, host_id, name, desc="", ref=[], workspace_name=self.workspace) vuln_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDVULNHOST, vuln_obj) + self.__addPendingAction(Modelactions.ADDVULNHOST, vuln_obj) return vuln_obj.getID() @deprecation.deprecated(deprecated_in="3.0", removed_in="3.5", @@ -253,7 +275,7 @@ def createAndAddVulnToInterface(self, host_id, interface_id, name, workspace_name=self.workspace) vuln_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDVULNHOST, vuln_obj) + self.__addPendingAction(Modelactions.ADDVULNHOST, vuln_obj) return vuln_obj.getID() def createAndAddVulnToService(self, host_id, service_id, name, desc="", @@ -267,7 +289,7 @@ def createAndAddVulnToService(self, host_id, service_id, name, desc="", workspace_name=self.workspace) vuln_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDVULNSRV, vuln_obj) + self.__addPendingAction(Modelactions.ADDVULNSRV, vuln_obj) return vuln_obj.getID() def createAndAddVulnWebToService(self, host_id, service_id, name, desc="", @@ -286,7 +308,7 @@ def createAndAddVulnWebToService(self, host_id, service_id, name, desc="", workspace_name=self.workspace) vulnweb_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDVULNWEBSRV, vulnweb_obj) + self.__addPendingAction(Modelactions.ADDVULNWEBSRV, vulnweb_obj) return vulnweb_obj.getID() def createAndAddNoteToHost(self, host_id, name, text): @@ -297,7 +319,7 @@ def createAndAddNoteToHost(self, host_id, name, text): workspace_name=self.workspace) note_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDNOTEHOST, note_obj) + self.__addPendingAction(Modelactions.ADDNOTEHOST, note_obj) return note_obj.getID() @deprecation.deprecated(deprecated_in="3.0", removed_in="3.5", @@ -311,7 +333,7 @@ def createAndAddNoteToInterface(self, host_id, interface_id, name, text): workspace_name=self.workspace) note_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDNOTEHOST, note_obj) + self.__addPendingAction(Modelactions.ADDNOTEHOST, note_obj) return note_obj.getID() def createAndAddNoteToService(self, host_id, service_id, name, text): @@ -322,7 +344,7 @@ def createAndAddNoteToService(self, host_id, service_id, name, text): workspace_name=self.workspace) note_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDNOTESRV, note_obj) + self.__addPendingAction(Modelactions.ADDNOTESRV, note_obj) return note_obj.getID() def createAndAddNoteToNote(self, host_id, service_id, note_id, name, text): @@ -334,7 +356,7 @@ def createAndAddNoteToNote(self, host_id, service_id, note_id, name, text): note_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDNOTENOTE, note_obj) + self.__addPendingAction(Modelactions.ADDNOTENOTE, note_obj) return note_obj.getID() def createAndAddCredToService(self, host_id, service_id, username, @@ -346,14 +368,14 @@ def createAndAddCredToService(self, host_id, service_id, username, workspace_name=self.workspace) cred_obj._metadata.creator = self.id - self.__addPendingAction(modelactions.ADDCREDSRV, cred_obj) + self.__addPendingAction(Modelactions.ADDCREDSRV, cred_obj) return cred_obj.getID() def log(self, msg, level='INFO'): - self.__addPendingAction(modelactions.LOG, msg, level) + self.__addPendingAction(Modelactions.LOG, msg, level) def devlog(self, msg): - self.__addPendingAction(modelactions.DEVLOG, msg) + self.__addPendingAction(Modelactions.DEVLOG, msg) class PluginTerminalOutput(PluginBase): diff --git a/scripts/shodan_faraday.py b/scripts/shodan_faraday.py index 5106fa714f7..9ff1d3b1417 100644 --- a/scripts/shodan_faraday.py +++ b/scripts/shodan_faraday.py @@ -21,7 +21,7 @@ __status__ = "Development" # Configuration -SHODAN_API_KEY = "lT3whkyVHH7iAtP28iNIq7hVNlK638vR" +SHODAN_API_KEY = "" def strip_non_ascii(string): ''' Returns the string without non ASCII characters''' @@ -32,27 +32,27 @@ def send_faraday(result): print 'IP: %s' % result['ip_str'] if result['data'] is not None: - result['data'] = base64.b64encode(strip_non_ascii(str(result['data']))) #fix: to avoid non ascii caracters - + result['data'] = base64.b64encode(str(strip_non_ascii(result['data']))) #fix: to avoid non ascii caracters if args.debug == "1": print '===============' for key in result.keys(): print "kname:" + key + ", value:" + str(result[key]) h_id = api.createAndAddHost(str(result['ip_str']),str(result['os']) if result['os'] is not None else "") - i_id = api.createAndAddInterface(h_id,str(result['ip_str']),"00:00:00:00:00:00", str(result['ip_str']), "0.0.0.0", "0.0.0.0",[], - "0000:0000:0000:0000:0000:0000:0000:0000","00","0000:0000:0000:0000:0000:0000:0000:0000", - [],"",result['hostnames'] if result['hostnames'] is not None else []) - s_id = api.createAndAddServiceToInterface(h_id, i_id, str(result['product']) if result.has_key('product') else str(result['port']), - "tcp",str(result['port']),"open",str(result['version']) if result.has_key('version') else "") + #i_id = api.createAndAddInterface(h_id,str(result['ip_str']),"00:00:00:00:00:00", str(result['ip_str']), "0.0.0.0", "0.0.0.0",[], + # "0000:0000:0000:0000:0000:0000:0000:0000","00","0000:0000:0000:0000:0000:0000:0000:0000", + # [],"",result['hostnames'] if result['hostnames'] is not None else []) + s_id = api.createAndAddServiceToHost(h_id, str(result['product']) if result.has_key('product') else str(result['port']), + "tcp",[str(result['port'])],"open",str(result['version']) if result.has_key('version') else "") + if result['data'] is not None: - n_id = api.createAndAddNoteToService(h_id,s_id,"shadon_response",str(result['data'])) + n_id = api.createAndAddNoteToService(h_id, s_id, "shadon_response", str(result['data'])) #Notes - Information geo/shadon - n_id = api.createAndAddNoteToHost(h_id,"geo_country",result['location']['country_name'] if result['location']['country_name'] is not None else "" ) - n_id = api.createAndAddNoteToHost(h_id,"geo_latitude",result['location']['latitude'] if result['location']['latitude'] is not None else "") - n_id = api.createAndAddNoteToHost(h_id,"geo_longitude",result['location']['longitude'] if result['location']['longitude'] is not None else "") - n_id = api.createAndAddNoteToHost(h_id,"shadon_q",args.shodan_query) + n_id = api.createAndAddNoteToHost(h_id, "geo_country", result['location']['country_name'] if result['location']['country_name'] is not None else "" ) + n_id = api.createAndAddNoteToHost(h_id, "geo_latitude", result['location']['latitude'] if result['location']['latitude'] is not None else "") + n_id = api.createAndAddNoteToHost(h_id, "geo_longitude", result['location']['longitude'] if result['location']['longitude'] is not None else "") + n_id = api.createAndAddNoteToHost(h_id, "shadon_q", args.shodan_query) # Input validation diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index bc323ef6311..2080e64065e 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -27,7 +27,6 @@ from server.fields import FaradayUploadedFile from server.models import ( db, - CommandObject, File, Host, Service, From ab60e56701af75ed62996391401956884625231c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 6 Mar 2018 17:59:54 -0300 Subject: [PATCH 0967/1506] Add hypothesis tests --- requirements.txt | 1 + test_cases/test_api_vulnerability.py | 258 +++++++++++++++++---------- 2 files changed, 168 insertions(+), 91 deletions(-) diff --git a/requirements.txt b/requirements.txt index c4948082d6c..fa253b4b92c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ restkit==4.2.2 tornado==4.5.1 tqdm==4.15.0 whoosh==2.7.4 +hypothesis==3.48.0 diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index ae31f9eaed5..060912d842f 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -5,6 +5,9 @@ import os from base64 import b64encode +from hypothesis import given +from hypothesis.strategies import text, lists, integers, one_of, none + from test_cases.conftest import ignore_nplusone try: @@ -14,6 +17,7 @@ import pytest import time +from hypothesis import given, assume, settings, strategies as st from server.api.modules.vulns import ( TypeFilter, @@ -36,6 +40,76 @@ CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) +def _create_post_data_vulnerability(name, vuln_type, parent_id, + parent_type, refs, policyviolations, + status='opened', + attachments=None, impact=None, + description='desc1234', + confirmed=True, data='data1234', + easeofresolution= + Vulnerability.EASE_OF_RESOLUTIONS[0], + owned=False, resolution='res1234', + severity='critical', + update_controller_action='UI Web' + ): + if not impact: + impact = {'accountability': False, 'availability': False, + 'confidentiality': False, + 'integrity': False} + data = { + 'metadata': { + 'update_time': 1508254070.211, + 'update_user': '', + 'update_action': 0, + 'creator': 'UI Web', + 'create_time': 1508254070.211, + 'update_controller_action': update_controller_action, + 'owner': ''}, + 'obj_id': '5a60af7f01dde6d3acfa8e9d3bef265c361a49d2', + 'owner': '', + 'parent': parent_id, + 'parent_type': parent_type, + 'type': vuln_type, + 'ws': 'airbnb', + 'confirmed': confirmed, + 'data': data, + 'desc': description, + 'easeofresolution': easeofresolution, + 'impact': impact, + 'name': name, + 'owned': owned, + 'policyviolations': policyviolations, + 'refs': refs, + 'resolution': resolution, + 'severity': severity, + 'status': status, + '_attachments': {}, + 'description': '', + 'protocol': '', + 'version': ''} + + if vuln_type == 'VulnerabilityWeb': + data.update({ + "method": "GET", + "params": "pepe", + "path": "/pepep", + "pname": "pepe", + "query": "queue&dfsa", + "request": "", + "response": "", + "website": "www.pepe.com"}) + + if attachments: + data['_attachments'] = {} + for attachment in attachments: + data['_attachments'][attachment.name] = { + "content_type": "application/x-shellscript", + "data": b64encode(attachment.read()) + } + + return data + + @pytest.mark.usefixtures('logged_user') class TestListVulnerabilityView(ReadOnlyAPITests): # TODO migration: use read write api tests model = Vulnerability @@ -154,72 +228,6 @@ def test_hostnames(self, host_with_hostnames, test_client, session, assert isinstance(res.json['hostnames'], list) assert set(res.json['hostnames']) == set(hostname.name for hostname in host_with_hostnames.hostnames) - - def _create_post_data_vulnerability(self, name, vuln_type, parent_id, - parent_type, refs, policyviolations, - status='opened', - attachments=None, impact=None, description='desc1234', - confirmed=True, data='data1234', - easeofresolution=Vulnerability.EASE_OF_RESOLUTIONS[0], - owned=False, resolution='res1234', severity='critical', - update_controller_action='UI Web' - ): - if not impact: - impact = {'accountability': False, 'availability': False, 'confidentiality': False, - 'integrity': False} - data = { - 'metadata': { - 'update_time': 1508254070.211, - 'update_user': '', - 'update_action': 0, - 'creator': 'UI Web', - 'create_time': 1508254070.211, - 'update_controller_action': update_controller_action, - 'owner': ''}, - 'obj_id': '5a60af7f01dde6d3acfa8e9d3bef265c361a49d2', - 'owner': '', - 'parent': parent_id, - 'parent_type': parent_type, - 'type': vuln_type, - 'ws': 'airbnb', - 'confirmed': confirmed, - 'data': data, - 'desc': description, - 'easeofresolution': easeofresolution, - 'impact': impact, - 'name': name, - 'owned': owned, - 'policyviolations': policyviolations, - 'refs': refs, - 'resolution': resolution, - 'severity': severity, - 'status': status, - '_attachments': {}, - 'description': '', - 'protocol': '', - 'version': ''} - - if vuln_type == 'VulnerabilityWeb': - data.update({ - "method": "GET", - "params": "pepe", - "path": "/pepep", - "pname": "pepe", - "query": "queue&dfsa", - "request": "", - "response": "", - "website": "www.pepe.com"}) - - if attachments: - data['_attachments'] = {} - for attachment in attachments: - data['_attachments'][attachment.name] = { - "content_type": "application/x-shellscript", - "data": b64encode(attachment.read()) - } - - return data - def test_create_vuln(self, host_with_hostnames, test_client, session): """ This one should only check basic vuln properties @@ -229,7 +237,7 @@ def test_create_vuln(self, host_with_hostnames, test_client, session): :return: """ session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id=host_with_hostnames.id, @@ -258,7 +266,7 @@ def test_create_vuln_with_attachments(self, host_with_hostnames, test_client, se file_content = 'test file' attachment.write(file_content) attachment.seek(0) - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id=host_with_hostnames.id, @@ -344,7 +352,7 @@ def test_create_vuln_props(self, host_with_hostnames, test_client, session): 'status': 'closed' } vuln_props_excluded = ['owned'] - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id=host_with_hostnames.id, @@ -372,7 +380,7 @@ def test_create_idempotent(self, host_with_hostnames, test_client, session): :return: """ session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='Vulnerability name goes here', vuln_type='Vulnerability', parent_id=host_with_hostnames.id, @@ -391,10 +399,9 @@ def test_create_idempotent(self, host_with_hostnames, test_client, session): assert res.status_code == 409 assert vuln_count_previous + 1 == session.query(Vulnerability).count() - def test_create_vuln_with_closed_status(self, host_with_hostnames, test_client, session): session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id=host_with_hostnames.id, @@ -416,7 +423,7 @@ def test_create_vuln_with_closed_status(self, host_with_hostnames, test_client, def test_create_vuln_with_closed_status(self, host_with_hostnames, test_client, session): session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id=host_with_hostnames.id, @@ -515,7 +522,7 @@ def test_update_vuln_from_open_to_close(self, test_client, session, host_with_ho def test_create_vuln_web(self, host_with_hostnames, test_client, session): service = ServiceFactory.create(host=host_with_hostnames, workspace=host_with_hostnames.workspace) session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='VulnerabilityWeb', parent_id=service.id, @@ -701,7 +708,7 @@ def test_create_vuln_with_evidence(self, host_with_hostnames, test_client, open(os.path.join(CURRENT_PATH, 'data', 'faraday.png'), 'r'), open(os.path.join(CURRENT_PATH, 'data', 'test.html'), 'r') ] - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id=host_with_hostnames.id, @@ -721,7 +728,7 @@ def test_create_vuln_with_evidence(self, host_with_hostnames, test_client, def test_create_vuln_with_refs(self, host_with_hostnames, test_client, session): session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id=host_with_hostnames.id, @@ -738,7 +745,7 @@ def test_create_vuln_with_refs(self, host_with_hostnames, test_client, session): def test_create_vuln_with_policyviolations(self, host_with_hostnames, test_client, session): session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id=host_with_hostnames.id, @@ -756,7 +763,7 @@ def test_create_vuln_with_policyviolations(self, host_with_hostnames, test_clien def test_create_vuln_imapct_verification(self, host_with_hostnames, test_client, session): session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id=host_with_hostnames.id, @@ -784,7 +791,7 @@ def test_create_vuln_imapct_verification(self, host_with_hostnames, test_client, def test_handles_invalid_impact(self, host_with_hostnames, test_client, session): session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id=host_with_hostnames.id, @@ -807,7 +814,7 @@ def test_create_vuln_with_invalid_type(self, test_client, session): session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='VulnerabilitySarasa', parent_id=host_with_hostnames.id, @@ -829,7 +836,7 @@ def test_create_vuln_with_invalid_severity(self, host_with_hostnames, test_client, session): session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id=host_with_hostnames.id, @@ -850,7 +857,7 @@ def test_create_vuln_with_invalid_ease_of_resolution(self, test_client, session): session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id=host_with_hostnames.id, @@ -872,7 +879,7 @@ def test_create_vuln_with_null_ease_of_resolution(self, test_client, session): session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id=host_with_hostnames.id, @@ -1091,7 +1098,7 @@ def test_create_with_parent_of_other_workspace( parent = parent_factory.create(workspace=second_workspace) session.commit() assert parent.workspace_id != self.workspace.id - data = self._create_post_data_vulnerability( + data = _create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id=parent.id, @@ -1137,7 +1144,7 @@ def test_create_vuln_multiple_times_returns_conflict(self, host_with_hostnames, :return: """ session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id=host_with_hostnames.id, @@ -1164,7 +1171,7 @@ def test_create_webvuln_multiple_times_returns_conflict(self, host_with_hostname """ service = ServiceFactory.create(workspace=self.workspace) session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulnsweb', vuln_type='VulnerabilityWeb', parent_id=service.id, @@ -1190,7 +1197,7 @@ def test_create_and_update_webvuln(self, host_with_hostnames, test_client, sessi command = CommandFactory.create(workspace=self.workspace) service = ServiceFactory.create(workspace=self.workspace) session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulnsweb', vuln_type='VulnerabilityWeb', parent_id=service.id, @@ -1203,7 +1210,7 @@ def test_create_and_update_webvuln(self, host_with_hostnames, test_client, sessi ws_name = host_with_hostnames.workspace.name res = test_client.post('/v2/ws/{0}/vulns/?command_id={1}'.format(ws_name, command.id), data=raw_data) assert res.status_code == 201 - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='Update vulnsweb', vuln_type='VulnerabilityWeb', parent_id=service.id, @@ -1223,7 +1230,7 @@ def test_create_vuln_from_command(self, test_client, session): session.commit() assert len(command.command_objects) == 0 url = self.url(workspace=command.workspace) + '?' + urlencode({'command_id': command.id}) - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='Update vulnsweb', vuln_type='VulnerabilityWeb', parent_id=service.id, @@ -1247,7 +1254,7 @@ def test_with_invalid_id_returns_400(self, session, test_client): Bug found on hackaton. """ session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id='', @@ -1263,7 +1270,7 @@ def test_with_invalid_id_returns_400(self, session, test_client): def test_vuln_created_without_command_has_webui_in_metadata(self, test_client, session): host = HostFactory.create(workspace=self.workspace) session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id=host.id, @@ -1393,3 +1400,72 @@ def test_service_filter(workspace, session, host, service_factory, assert all(v.service and v.service.name == 'http' for v in filtered) assert {v.id for v in filtered} == {v.id for v in vulns} + + +def vulnerability_json(parent_id, parent_type): + return st.fixed_dictionaries( + { + 'metadata': st.fixed_dictionaries({ + 'update_time': st.floats(), + 'update_user': st.one_of(st.none(), st.text()), + 'update_action': st.integers(), + 'creator':st.text(), + 'create_time': st.floats(), + 'update_controller_action': st.text(), + 'owner': st.one_of(st.none(), st.text())}), + 'obj_id': st.integers(), + 'owner': st.one_of(st.none(), st.text()), + 'parent': st.sampled_from([parent_id]), + 'parent_type': st.sampled_from([parent_type]), + 'type': st.one_of( + st.sampled_from([ + "Vulnerability", "Invalid", None]), + st.text() + ), + 'ws': st.one_of(st.none(), st.text()), + 'confirmed': st.booleans(), + 'data': st.one_of(st.none(), st.text()), + 'desc': st.one_of(st.none(), st.text()), + 'easeofresolution': st.sampled_from(['trivial', + 'simple', + 'moderate', + 'difficult', + 'infeasible']), + 'impact': st.fixed_dictionaries({'accountability': st.booleans(), 'availability': st.booleans(), + 'confidentiality': st.booleans(), + 'integrity': st.booleans()}), + 'name': st.one_of(st.none(), st.text()), + 'owned': st.booleans(), + 'policyviolations': st.lists(st.one_of(st.none(), st.text())), + 'refs': st.lists(st.one_of(st.none(), st.text())), + 'resolution': st.one_of(st.none(), st.text()), + 'severity': st.sampled_from(['critical', + 'high', + 'medium', + 'low', + 'informational', + 'unclassified']), + 'status': st.sampled_from(['open', + 'closed', + 're-opened', + 'risk-accepted']), + '_attachments': st.fixed_dictionaries({}), + 'description': st.one_of(st.none(), st.text()), + 'protocol': st.one_of(st.none(), st.text()), + 'version': st.one_of(st.none(), st.text())}) + + +@pytest.mark.usefixtures('logged_user') +def test_hypothesis(host_with_hostnames, test_client, session): + session.commit() + VulnerabilityData = vulnerability_json(host_with_hostnames.id, 'Host') + + @given(VulnerabilityData) + def send_api_request(raw_data): + + ws_name = host_with_hostnames.workspace.name + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), + data=raw_data) + assert res.status_code in [201, 400, 409] + + send_api_request() \ No newline at end of file From e94a1e8afaf98030ad33e0700b80c76544c5f1c6 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 6 Mar 2018 18:07:30 -0300 Subject: [PATCH 0968/1506] Add hypothesis to Hosts api --- test_cases/test_api_hosts.py | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 72b7454475a..fbd21dfd7bd 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -7,6 +7,7 @@ import urllib.parse as urlparse from urllib.parse import urlencode from sqlalchemy.orm.util import was_deleted +from hypothesis import given, assume, settings, strategies as st import pytest @@ -627,3 +628,49 @@ def test_service_summaries(self, test_client, session, service_factory): '(53/udp) dns', '(5353/udp) dns', ] + +def host_json(): + return st.fixed_dictionaries( + { + "metadata": + st.fixed_dictionaries({ + "update_time": st.floats(), + "update_user": st.one_of(st.none(), st.text()), + "update_action": st.integers(), + "creator": st.text(), + "create_time": st.integers(), + "update_controller_action": st.text(), + "owner": st.one_of(st.none(), st.text()), + "command_id": st.one_of(st.none(), st.text(), st.integers()),}), + "name": st.one_of(st.none(), st.text()), + "ip": st.one_of(st.none(), st.text()), + "_rev": st.one_of(st.none(), st.text()), + "description": st.one_of(st.none(), st.text()), + "default_gateway": st.one_of(st.none(), st.text()), + "owned": st.booleans(), + "services": st.one_of(st.none(), st.integers()), + "hostnames": st.lists(st.text()), + "vulns": st.one_of(st.none(), st.integers()), + "owner": st.one_of(st.none(), st.text()), + "credentials": st.one_of(st.none(), st.integers()), + "_id": st.one_of(st.none(), st.integers()), + "os": st.one_of(st.none(), st.text()), + "id": st.one_of(st.none(), st.integers()), + "icon": st.one_of(st.none(), st.text())} + ) + + +@pytest.mark.usefixtures('logged_user') +def test_hypothesis(host_with_hostnames, test_client, session): + session.commit() + HostData = host_json() + + @given(HostData) + def send_api_request(raw_data): + + ws_name = host_with_hostnames.workspace.name + res = test_client.post('/v2/ws/{0}/vulns/'.format(ws_name), + data=raw_data) + assert res.status_code in [201, 400, 409] + + send_api_request() \ No newline at end of file From 6ab395bb035d6bda659d51877599f035e7240c4c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 7 Mar 2018 13:44:04 -0300 Subject: [PATCH 0969/1506] Add server.config tests --- test_cases/test_server_config.py | 51 ++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 test_cases/test_server_config.py diff --git a/test_cases/test_server_config.py b/test_cases/test_server_config.py new file mode 100644 index 00000000000..f58e7a13ac8 --- /dev/null +++ b/test_cases/test_server_config.py @@ -0,0 +1,51 @@ +import random +import string +import mock + +from server.config import ( + copy_default_config_to_local, + gen_web_config +) + + +@mock.patch('os.makedirs') +@mock.patch('shutil.copyfile') +def test_copy_default_config_to_local_does_not_exist(copyfile, makedirs): + random_name = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in + range(25)) + + new_file = '/tmp/{0}'.format(random_name) + with mock.patch('server.config.LOCAL_CONFIG_FILE', new_file): + assert copy_default_config_to_local() is None + assert makedirs.called + assert copyfile.called + + # the second call will re use the file just created. + makedirs.reset_mock() + copyfile.reset_mock() + assert copy_default_config_to_local() is None + assert not makedirs.called + assert not copyfile.called + + +@mock.patch('os.path.isfile') +@mock.patch('os.remove') +@mock.patch('json.dump') +def test_gen_web_config(dump, remove, isfile): + gen_web_config() + assert isfile.called + assert remove.called + assert dump.called + assert len(dump.call_args_list) == 1 + for call in dump.call_args_list: + call[0][0] == { + 'lic_db': 'faraday_licenses', + 'osint': {u'host': u'shodan.io', + u'icon': u'shodan', + u'label': u'Shodan', + u'prefix': u'/search?query=', + u'suffix': u'', + u'use_external_icon': False}, + 'ver': '2.7.1', + 'vuln_model_db': 'cwe'} + From fbe43e05b5c8df8947a0db3e54fdf2bac9afc913 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 7 Mar 2018 14:56:10 -0300 Subject: [PATCH 0970/1506] Add tests for ModelController. improve vuln json strategy --- model/controller.py | 6 ++- test_cases/test_api_hosts.py | 1 + test_cases/test_api_vulnerability.py | 3 +- test_cases/test_model_controller.py | 64 ++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 3 deletions(-) diff --git a/model/controller.py b/model/controller.py index 5d305b619cf..f52435b86dd 100644 --- a/model/controller.py +++ b/model/controller.py @@ -417,18 +417,20 @@ def addPluginEnd(self, name): self.__addPendingAction(Modelactions.PLUGINEND, name) def _pluginStart(self, name, command_id): - self.processing = True self.active_plugins_count_lock.acquire() + self.processing = True getLogger(self).info("Plugin Started: {0}. ".format(name, command_id)) self.active_plugins_count += 1 self.active_plugins_count_lock.release() return True def _pluginEnd(self, name, command_id): - self.processing = False + self.active_plugins_count_lock.acquire() getLogger(self).info("Plugin Ended: {0}".format(name)) self.active_plugins_count -= 1 + if self.active_plugins_count == 0: + self.processing = False self.active_plugins_count_lock.release() return True diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index fbd21dfd7bd..0908daf3567 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -629,6 +629,7 @@ def test_service_summaries(self, test_client, session, service_factory): '(5353/udp) dns', ] + def host_json(): return st.fixed_dictionaries( { diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 060912d842f..31f851e6872 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1409,7 +1409,7 @@ def vulnerability_json(parent_id, parent_type): 'update_time': st.floats(), 'update_user': st.one_of(st.none(), st.text()), 'update_action': st.integers(), - 'creator':st.text(), + 'creator': st.text(), 'create_time': st.floats(), 'update_controller_action': st.text(), 'owner': st.one_of(st.none(), st.text())}), @@ -1441,6 +1441,7 @@ def vulnerability_json(parent_id, parent_type): 'resolution': st.one_of(st.none(), st.text()), 'severity': st.sampled_from(['critical', 'high', + 'med', 'medium', 'low', 'informational', diff --git a/test_cases/test_model_controller.py b/test_cases/test_model_controller.py index e69de29bb2d..ad8258378a8 100644 --- a/test_cases/test_model_controller.py +++ b/test_cases/test_model_controller.py @@ -0,0 +1,64 @@ +from Queue import Queue + +import time + +from managers.mapper_manager import MapperManager +from model import Modelactions +from model.controller import ModelController + + +def test_controller_stop_when_is_not_processing(): + mappers_manager = MapperManager() + pending_actions = Queue() + controller = ModelController(mappers_manager, pending_actions) + assert controller.processing is False + assert controller._stop is False + controller.start() + assert controller.isAlive() + controller.stop() + assert controller._stop is True + controller.join() + assert controller.isAlive() is False + + +def test_controller_cant_be_stopped_when_is_processing(): + """ + If someone tells the controller to stop and it is processing then it + will stop when the processing finishes + """ + + mappers_manager = MapperManager() + pending_actions = Queue() + controller = ModelController(mappers_manager, pending_actions) + assert controller.processing is False + assert controller._stop is False + controller.start() + controller.processing = True + controller.active_plugins_count = 1 + assert controller.isAlive() + controller.stop() + assert controller._stop + assert controller.processing + controller.join(timeout=2) + assert controller.isAlive() + controller.processing = False + controller.join() + assert controller.isAlive() is False + + +def test_controller_plugin_start_action_updates_internal_state(): + mappers_manager = MapperManager() + pending_actions = Queue() + controller = ModelController(mappers_manager, pending_actions) + controller.start() + controller.add_action((Modelactions.PLUGINSTART, "test", None)) + time.sleep(1) + assert controller.active_plugins_count == 1 + assert controller.processing + controller.add_action((Modelactions.PLUGINEND, "test", None)) + time.sleep(1) + assert controller.active_plugins_count == 0 + assert controller.processing is False + controller.stop() + controller.join() + assert controller.isAlive() is False \ No newline at end of file From 30ae1956126da5e021d07e6f17678b74facb92b6 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 7 Mar 2018 17:50:38 -0300 Subject: [PATCH 0971/1506] Add cascade delete. leave ref and policy violations when vuln are deleted. --- server/models.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/server/models.py b/server/models.py index cef757cdb00..f9b2f683235 100644 --- a/server/models.py +++ b/server/models.py @@ -293,7 +293,7 @@ class Hostname(Metadata): name = Column(Text, nullable=False) host_id = Column(Integer, ForeignKey('host.id'), index=True, nullable=False) - host = relationship('Host', backref='hostnames') + host = relationship('Host', backref=backref("hostnames", cascade="all, delete-orphan")) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship( 'Workspace', @@ -767,10 +767,10 @@ class Vulnerability(VulnerabilityGeneric): __tablename__ = None host_id = Column(Integer, ForeignKey(Host.id), index=True) host = relationship( - 'Host', - backref='vulnerabilities', - foreign_keys=[host_id], - ) + 'Host', + backref=backref("vulnerabilities", cascade="all, delete-orphan"), + foreign_keys=[host_id], + ) @declared_attr def service_id(cls): @@ -778,7 +778,7 @@ def service_id(cls): @declared_attr def service(cls): - return relationship('Service', backref='vulnerabilities') + return relationship('Service', backref=backref("vulnerabilities", cascade="all, delete-orphan")) @property def hostnames(self): @@ -816,7 +816,7 @@ def service_id(cls): @declared_attr def service(cls): - return relationship('Service', backref='vulnerabilities_web') + return relationship('Service', backref=backref("vulnerabilities_web", cascade="all, delete-orphan"), cascade='all') @property def parent(self): @@ -884,7 +884,8 @@ class Reference(Metadata): ) workspace = relationship( 'Workspace', - backref='references', + backref=backref("references", + cascade="all, delete-orphan"), foreign_keys=[workspace_id], ) @@ -910,8 +911,13 @@ class ReferenceVulnerabilityAssociation(db.Model): vulnerability_id = Column(Integer, ForeignKey('vulnerability.id'), primary_key=True) reference_id = Column(Integer, ForeignKey('reference.id'), primary_key=True) - reference = relationship("Reference", backref="reference_associations", foreign_keys=[reference_id]) - vulnerability = relationship("Vulnerability", backref="reference_vulnerability_associations", foreign_keys=[vulnerability_id]) + reference = relationship("Reference", + backref="reference_associations", + foreign_keys=[reference_id]) + vulnerability = relationship("Vulnerability", + backref=backref("reference_vulnerability_associations", + cascade="all, delete-orphan"), + foreign_keys=[vulnerability_id]) class PolicyViolationVulnerabilityAssociation(db.Model): @@ -922,7 +928,7 @@ class PolicyViolationVulnerabilityAssociation(db.Model): policy_violation_id = Column(Integer, ForeignKey('policy_violation.id'), primary_key=True) policy_violation = relationship("PolicyViolation", backref="policy_violation_associations", foreign_keys=[policy_violation_id]) - vulnerability = relationship("Vulnerability", backref="policy_violationvulnerability_associations", + vulnerability = relationship("Vulnerability", backref=backref("policy_violationvulnerability_associations", cascade="all, delete-orphan"), foreign_keys=[vulnerability_id]) @@ -978,7 +984,8 @@ class PolicyViolation(Metadata): ) workspace = relationship( 'Workspace', - backref='policy_violations', + backref=backref("policy_violations", + cascade="all, delete-orphan"), foreign_keys=[workspace_id], ) From f28c2366b5c5178dbce0e167c5bd18a76593cc78 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 8 Mar 2018 13:16:50 -0300 Subject: [PATCH 0972/1506] Fix some bugs on model controller. add unit tests --- model/controller.py | 9 +++--- test_cases/test_model_controller.py | 47 ++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/model/controller.py b/model/controller.py index f52435b86dd..0c1df6250b3 100644 --- a/model/controller.py +++ b/model/controller.py @@ -336,9 +336,8 @@ def addUpdate(self, old_object, new_object, command_id): notifier.editHost(old_object) return True - # XXX: THIS DOESNT WORK - def find(self, obj_id): - return self.mappers_manager.find(obj_id) + def find(self, class_signature, obj_id): + return self.mappers_manager.find(class_signature, obj_id) def _save_new_object(self, new_object, command_id): res = None @@ -425,9 +424,11 @@ def _pluginStart(self, name, command_id): return True def _pluginEnd(self, name, command_id): - self.active_plugins_count_lock.acquire() getLogger(self).info("Plugin Ended: {0}".format(name)) + if self.active_plugins_count == 0: + getLogger(self).warn("All plugins ended, but a plugin end action was received.") + return True self.active_plugins_count -= 1 if self.active_plugins_count == 0: self.processing = False diff --git a/test_cases/test_model_controller.py b/test_cases/test_model_controller.py index ad8258378a8..a185dbb2f90 100644 --- a/test_cases/test_model_controller.py +++ b/test_cases/test_model_controller.py @@ -2,9 +2,11 @@ import time +import mock from managers.mapper_manager import MapperManager from model import Modelactions from model.controller import ModelController +from test_cases.factories import WorkspaceFactory, HostFactory def test_controller_stop_when_is_not_processing(): @@ -61,4 +63,47 @@ def test_controller_plugin_start_action_updates_internal_state(): assert controller.processing is False controller.stop() controller.join() - assert controller.isAlive() is False \ No newline at end of file + assert controller.isAlive() is False + +def test_only_start_plugin(): + mappers_manager = MapperManager() + pending_actions = Queue() + controller = ModelController(mappers_manager, pending_actions) + controller._pluginStart('test', None) + assert controller.active_plugins_count == 1 + assert controller.processing + controller._pluginStart('test', None) + assert controller.active_plugins_count == 2 + +def test_only_end_pluging(): + mappers_manager = MapperManager() + pending_actions = Queue() + controller = ModelController(mappers_manager, pending_actions) + controller._pluginStart('test', None) + controller._pluginEnd('test', None) + assert controller.active_plugins_count == 0 + assert controller.processing is False + +def test_end_pluging_multiple_times(): + mappers_manager = MapperManager() + pending_actions = Queue() + controller = ModelController(mappers_manager, pending_actions) + controller._pluginEnd('test', None) + controller._pluginEnd('test', None) + assert controller.active_plugins_count == 0 + assert controller.processing is False + + +@mock.patch('persistence.server.server._get') +def test_find(get, session): + mappers_manager = MapperManager() + pending_actions = Queue() + controller = ModelController(mappers_manager, pending_actions) + workspace = WorkspaceFactory.create() + mappers_manager.createMappers(workspace.name) + host = HostFactory.create() + controller.find("Host", host.id) + assert get.called + assert get.mock_calls[0][1][0] == 'http://localhost:5985/_api/v2/ws/{0}/hosts/{1}/'.format(workspace.name, host.id) + assert get.mock_calls[0][2] == {'object_id': host.id} + From 8866dbe5495d8a69ac3b6dd3e32d1f61e924235e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 8 Mar 2018 13:17:11 -0300 Subject: [PATCH 0973/1506] Update code to work with python3 --- model/controller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/model/controller.py b/model/controller.py index 0c1df6250b3..1ceb0f35d0c 100644 --- a/model/controller.py +++ b/model/controller.py @@ -272,7 +272,8 @@ def _main(self): time.sleep(0.01) def processAllPendingActions(self): - [self.processAction() for i in range(self._pending_actions.qsize())] + for _ in range(self._pending_actions.qsize()): + self.processAction() def processAction(self): # check the queue for new actions From be6576073396a55ed0366e8c76f9b15df9c5ae49 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 8 Mar 2018 13:18:06 -0300 Subject: [PATCH 0974/1506] Add missing lock release --- model/controller.py | 1 + 1 file changed, 1 insertion(+) diff --git a/model/controller.py b/model/controller.py index 1ceb0f35d0c..daf700db9df 100644 --- a/model/controller.py +++ b/model/controller.py @@ -428,6 +428,7 @@ def _pluginEnd(self, name, command_id): self.active_plugins_count_lock.acquire() getLogger(self).info("Plugin Ended: {0}".format(name)) if self.active_plugins_count == 0: + self.active_plugins_count_lock.release() getLogger(self).warn("All plugins ended, but a plugin end action was received.") return True self.active_plugins_count -= 1 From 2ccb38e3b2d01ea61ca9a6d6dcabae86068491e3 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 8 Mar 2018 13:19:19 -0300 Subject: [PATCH 0975/1506] Set the workspace of the host. commit session --- test_cases/test_model_controller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_cases/test_model_controller.py b/test_cases/test_model_controller.py index a185dbb2f90..e6876e9b77b 100644 --- a/test_cases/test_model_controller.py +++ b/test_cases/test_model_controller.py @@ -101,7 +101,8 @@ def test_find(get, session): controller = ModelController(mappers_manager, pending_actions) workspace = WorkspaceFactory.create() mappers_manager.createMappers(workspace.name) - host = HostFactory.create() + host = HostFactory.create(workspace=workspace) + session.commit() controller.find("Host", host.id) assert get.called assert get.mock_calls[0][1][0] == 'http://localhost:5985/_api/v2/ws/{0}/hosts/{1}/'.format(workspace.name, host.id) From 64b39b9c6d858602243bcef5802f49bf2788ea5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 8 Mar 2018 13:54:09 -0300 Subject: [PATCH 0976/1506] Move hypothesis dependency to requirements_dev --- requirements.txt | 1 - requirements_dev.txt | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fa253b4b92c..c4948082d6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,3 @@ restkit==4.2.2 tornado==4.5.1 tqdm==4.15.0 whoosh==2.7.4 -hypothesis==3.48.0 diff --git a/requirements_dev.txt b/requirements_dev.txt index 0b258cbb70e..10e96477513 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -2,3 +2,4 @@ factory-boy pytest pytest-factoryboy responses +hypothesis==3.48.0 From 7511f097836f859d0f835716e7ab14d93b09d4fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 8 Mar 2018 14:03:43 -0300 Subject: [PATCH 0977/1506] Fix test_find test Don't use a hardcoded host name because it fails on my computer --- test_cases/test_model_controller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_cases/test_model_controller.py b/test_cases/test_model_controller.py index e6876e9b77b..c08a0cc54d2 100644 --- a/test_cases/test_model_controller.py +++ b/test_cases/test_model_controller.py @@ -105,6 +105,7 @@ def test_find(get, session): session.commit() controller.find("Host", host.id) assert get.called - assert get.mock_calls[0][1][0] == 'http://localhost:5985/_api/v2/ws/{0}/hosts/{1}/'.format(workspace.name, host.id) + assert get.mock_calls[0][1][0].endswith( + '/_api/v2/ws/{0}/hosts/{1}/'.format(workspace.name, host.id)) assert get.mock_calls[0][2] == {'object_id': host.id} From e8d7617656be39bddc2789ed49f746c35c3a3433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 8 Mar 2018 14:08:24 -0300 Subject: [PATCH 0978/1506] Move selenium import of screenshot fplugin and fix styling --- bin/screenshot_server.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/bin/screenshot_server.py b/bin/screenshot_server.py index 714baed30c7..9c861d97348 100644 --- a/bin/screenshot_server.py +++ b/bin/screenshot_server.py @@ -10,17 +10,18 @@ import sys from persistence.server.server_io_exceptions import ResourceDoesNotExist from persistence.server import models -try: - from selenium import webdriver -except ImportError: - print ("Missing dependencies: (selenium). install it with pip install selenium. ") - sys.exit(1) __description__ = 'Takes a Screenshot of the ip:ports of a given protocol' __prettyname__ = 'Screenshot_server' def screenshot(path, protocol, ip, port): + try: + from selenium import webdriver + except ImportError: + print("Missing dependencies: (selenium). " + "Install it with pip install selenium. ") + sys.exit(1) driver = webdriver.PhantomJS() driver.set_window_size(1024, 768) # set the window size that you need driver.set_page_load_timeout(5) @@ -41,19 +42,19 @@ def main(workspace='', args=None, parser=None): parsed_args = parser.parse_args(args) protocols = parsed_args.protocol.split(",") - print (protocols) + print(protocols) path = parsed_args.path for protocol in protocols: if not os.path.exists(path): - print ("Invalid Path") + print("Invalid Path") exit() try: services = models.get_services(workspace) except ResourceDoesNotExist: - print ("Invalid workspace name: ", workspace) + print("Invalid workspace name: ", workspace) return 1, None for service in services: @@ -66,6 +67,6 @@ def main(workspace='', args=None, parser=None): interface = models.get_interface(workspace, interface_id) ip = interface.ipv4["address"] - print (protocol + "://" + ip + ":" + port) + print(protocol + "://" + ip + ":" + port) screenshot(path, protocol, ip, port) return 0, None From 885275ebcecc91bd3e4c84fa1ab4596fed93d4b0 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 8 Mar 2018 15:44:06 -0300 Subject: [PATCH 0979/1506] Add parametrize to find test --- model/controller.py | 1 - test_cases/test_model_controller.py | 75 ++++++++++++++++++++++++++--- 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/model/controller.py b/model/controller.py index daf700db9df..2c9895b0863 100644 --- a/model/controller.py +++ b/model/controller.py @@ -447,7 +447,6 @@ def _log(self, msg, *args, **kwargs): api.log(msg, *args[:-1]) return True - def newHost(self, ip, os="Unknown"): return model.common.factory.createModelObject( models.Host.class_signature, ip, diff --git a/test_cases/test_model_controller.py b/test_cases/test_model_controller.py index c08a0cc54d2..a768b7733e8 100644 --- a/test_cases/test_model_controller.py +++ b/test_cases/test_model_controller.py @@ -3,11 +3,69 @@ import time import mock +import pytest + from managers.mapper_manager import MapperManager from model import Modelactions from model.controller import ModelController -from test_cases.factories import WorkspaceFactory, HostFactory +from test_cases.factories import ( + WorkspaceFactory, + VulnerabilityFactory, + HostFactory, + VulnerabilityWebFactory, + CredentialFactory, + ServiceFactory +) +TEST_CASES = { + 'hosts': { + 'factory': HostFactory, + 'class_signature': 'Host', + 'expected_result': {} + }, + 'vulns': { + 'factory':VulnerabilityFactory, + 'class_signature': 'Vulnerability', + 'api_result': { + 'name': 'Vuln 1', + 'desc': 'Description', + 'data': 'Data', + 'severity': 'critical', + 'confirmed': True, + }, + }, + 'vulns': { + 'factory': VulnerabilityWebFactory, + 'class_signature': 'VulnerabilityWeb', + 'api_result': { + 'name': 'Vuln 1', + 'desc': 'Description', + 'data': 'Data', + 'severity': 'critical', + 'confirmed': True, + }, + }, + 'credential': { + 'factory': CredentialFactory, + 'class_signature': 'Cred', + 'api_result': { + 'name': 'test', + 'username': 'test', + 'password': 'test' + }, + }, + 'services': { + 'factory': ServiceFactory, + 'class_signature': 'Service', + 'api_result': { + 'name': 'SSH', + 'protocol': 'tcp', + 'ports': [22], + 'version': '2.1', + 'status': 'open' + } + } +} def test_controller_stop_when_is_not_processing(): mappers_manager = MapperManager() @@ -94,18 +152,21 @@ def test_end_pluging_multiple_times(): assert controller.processing is False + +@pytest.mark.parametrize("url_endpoint, test_data", TEST_CASES.items()) @mock.patch('persistence.server.server._get') -def test_find(get, session): +def test_find(get, url_endpoint, test_data, session): + if 'api_result' in test_data: + get.return_value = test_data['api_result'] mappers_manager = MapperManager() pending_actions = Queue() controller = ModelController(mappers_manager, pending_actions) workspace = WorkspaceFactory.create() mappers_manager.createMappers(workspace.name) - host = HostFactory.create(workspace=workspace) + obj = test_data['factory'].create(workspace=workspace) session.commit() - controller.find("Host", host.id) + result = controller.find(test_data['class_signature'], obj.id) assert get.called + print(get.mock_calls[0][1][0]) assert get.mock_calls[0][1][0].endswith( - '/_api/v2/ws/{0}/hosts/{1}/'.format(workspace.name, host.id)) - assert get.mock_calls[0][2] == {'object_id': host.id} - + '/_api/v2/ws/{0}/{1}/{2}/'.format(workspace.name, url_endpoint, obj.id)) From b8cf8d114ca28bc3d639ebef1a9d452ab674abfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 8 Mar 2018 16:02:55 -0300 Subject: [PATCH 0980/1506] Improve fplugin load error logging --- plugins/fplugin_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/fplugin_utils.py b/plugins/fplugin_utils.py index 1427d435f06..eb04b1e7726 100644 --- a/plugins/fplugin_utils.py +++ b/plugins/fplugin_utils.py @@ -3,10 +3,12 @@ import sys from colorama import Fore +from utils.logs import getLogger from config.configuration import getInstanceConfiguration CONF = getInstanceConfiguration() +logger = getLogger() def get_available_plugins(): @@ -63,7 +65,7 @@ def get_available_plugins(): } except Exception: - sys.stderr.write("Unable to import module %s\n" % plugin_path) + logger.exception("Unable to import module %s\n" % plugin_path) return plugins_dic From d69ebc4a2082557bb503de660bf2d33722c5ed0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 8 Mar 2018 16:04:50 -0300 Subject: [PATCH 0981/1506] Fix indentation typo on screenshot fplugin --- bin/screenshot_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/screenshot_server.py b/bin/screenshot_server.py index 9c861d97348..224662f3168 100644 --- a/bin/screenshot_server.py +++ b/bin/screenshot_server.py @@ -21,7 +21,7 @@ def screenshot(path, protocol, ip, port): except ImportError: print("Missing dependencies: (selenium). " "Install it with pip install selenium. ") - sys.exit(1) + sys.exit(1) driver = webdriver.PhantomJS() driver.set_window_size(1024, 768) # set the window size that you need driver.set_page_load_timeout(5) From f5070c65060a41be36e1dba7aa378bbf7208fb64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 8 Mar 2018 18:06:35 -0300 Subject: [PATCH 0982/1506] Change last vulns dashboard widget's thead color It doesn't support sorting, show it black so the user doesn't confuse with a clickable element --- server/www/estilos.css | 3 +++ server/www/scripts/dashboard/partials/last-vulns.html | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/server/www/estilos.css b/server/www/estilos.css index 45c7e436785..a044f4b9bcf 100644 --- a/server/www/estilos.css +++ b/server/www/estilos.css @@ -389,6 +389,9 @@ ul.label-list { font-size: 12px !important; margin: 5px; } +table thead th span { + color: black; +} table thead th a { color: #2E91D8; font-weight: normal; diff --git a/server/www/scripts/dashboard/partials/last-vulns.html b/server/www/scripts/dashboard/partials/last-vulns.html index 86a7728a2d0..d6b695925cd 100644 --- a/server/www/scripts/dashboard/partials/last-vulns.html +++ b/server/www/scripts/dashboard/partials/last-vulns.html @@ -14,11 +14,11 @@

    Last Vulnerabilities

    + Name + + Hostnames + + Open Services + + Vulns + + Credentials + + OS + + Owned + + Last modified +
    + {{host.name}} + {{hostname}}
    		+ {{host.credentials}} + + owned not yet +
    + Description + + Hostnames + MAC + + Open Services @@ -93,6 +101,10 @@

    Add columns

    Owned     		+ Creation time + + Last modified @@ -110,11 +122,17 @@

    Add columns

    {{host.name}} +     		+ {{host.description}} + {{hostname}}
    		+ {{host.mac}} + @@ -131,6 +149,9 @@

    Add columns

    owned not yet     		+ + + OS + + Hostnames @@ -93,10 +97,6 @@

    Add columns

    Credentials     		- OS - - Owned @@ -125,6 +125,12 @@

    Add columns

    {{host.description}} +     		+ + + + + {{hostname}}
    @@ -138,12 +144,6 @@

    Add columns

    {{host.credentials}}     		- - - - - owned From 4cc4ba5d3f1eabafa6aea8795363ae5f2e66692d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 12 Jan 2018 13:15:28 -0300 Subject: [PATCH 0785/1506] Add table margin in hosts view --- server/www/scripts/hosts/partials/list.html | 226 ++++++++++---------- 1 file changed, 113 insertions(+), 113 deletions(-) diff --git a/server/www/scripts/hosts/partials/list.html b/server/www/scripts/hosts/partials/list.html index 584bf5bfe44..3a7fc87b8d7 100644 --- a/server/www/scripts/hosts/partials/list.html +++ b/server/www/scripts/hosts/partials/list.html @@ -61,120 +61,120 @@

    Add columns

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - Name - - - Description - - - OS - - - Hostnames - - - MAC - - - Open Services - - - Vulns - - - Credentials - - - Owned - - - Creation time - - - Last modified - -
    - {{host.name}} - - - {{host.description}} - - - - - - - - {{hostname}}
    -     -     		- {{host.mac}} - - {{host.credentials}} - - - owned - not yet - - - - -
    -
    -
    - -
    -
    - -
    - - -
    -
    -
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Name + + + Description + + + OS + + + Hostnames + + + MAC + + + Open Services + + + Vulns + + + Credentials + + + Owned + + + Creation time + + + Last modified + +
    + {{host.name}} + + + {{host.description}} + + + + + + + + {{hostname}}
    +     +     		+ {{host.mac}} + + {{host.credentials}} + + + owned + not yet + + + + +
    +
    +
    + +
    +
    + +
    + + +
    +
    +
    From 93658eaef04e6e4466cb9fdfee8c13ce47920e96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 12 Jan 2018 14:30:01 -0300 Subject: [PATCH 0786/1506] Add release information for #4417 --- RELEASE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index 38f57dfd040..69e35cea461 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -18,6 +18,8 @@ TBA: * Show credentials count in summarized report on the dashboard * Remove vuln template CWE fields, join it with references * Allow to search hosts by hostname +* Allow the user to specify the desired fields of the host list table +* Add optional hostnames, MAC and description fields to the host list * Workspace names can be changed from the Web UI * Changed the scope field of a workspace from a free text input to a list of targets From fdfa54cd1af7f70650cc30f7ca2f97a6ac28589f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 12 Jan 2018 16:58:25 -0300 Subject: [PATCH 0787/1506] Use enum for object_type. add unit tests --- server/api/base.py | 6 ++++-- server/models.py | 10 +++++++++- test_cases/test_api_hosts.py | 2 +- test_cases/test_api_services.py | 29 ++++++++++++++++++++++++++- test_cases/test_api_vulnerability.py | 30 +++++++++++++++++++++++++++- 5 files changed, 71 insertions(+), 6 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index f6f6e622b17..b7758f95202 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -468,16 +468,18 @@ def _set_command_id(self, obj, created): # if the object is created and updated in the same command # the command object already exists # we skip the creation. + object_type = obj.__class__.__table__.name + command_object = CommandObject.query.filter_by( object_id=obj.id, - object_type=obj.__class__.__name__, + object_type=object_type, command=command, workspace=obj.workspace, ).first() if created or not command_object: command_object = CommandObject( object_id=obj.id, - object_type=obj.__class__.__name__, + object_type=object_type, command=command, workspace=obj.workspace, created_persistent=created diff --git a/server/models.py b/server/models.py index 3e16d85408c..9e8d5b39504 100644 --- a/server/models.py +++ b/server/models.py @@ -45,6 +45,14 @@ ) from server.utils.database import BooleanToIntColumn +OBJECT_TYPES = [ + 'vulnerability', + 'host', + 'credential', + 'service', + 'source_code', +] + class SQLAlchemy(OriginalSQLAlchemy): """Override to fix issues when doing a rollback with sqlite driver @@ -500,7 +508,7 @@ class CommandObject(db.Model): id = Column(Integer, primary_key=True) object_id = Column(Integer, nullable=False) - object_type = Column(Text, nullable=False) + object_type = Column(Enum(*OBJECT_TYPES, name='object_types'), nullable=False) command = relationship('Command', backref='command_objects') command_id = Column(Integer, ForeignKey('command.id'), index=True) diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 25bcd6878f2..caf10306653 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -567,7 +567,7 @@ def test_create_host_from_command(self, test_client, session): assert res.status_code == 201 assert len(command.command_objects) == 1 cmd_obj = command.command_objects[0] - assert cmd_obj.object_type == 'Host' + assert cmd_obj.object_type == 'host' assert cmd_obj.object_id == res.json['id'] def test_create_host_cant_assign_command_from_another_workspace(self, test_client, session): diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index 2cc3d48f268..b88dc47628d 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -1,5 +1,9 @@ # -*- coding: utf8 -*- """Tests for many API endpoints that do not depend on workspace_name""" +try: + from urllib import urlencode +except ImportError: + from urllib.parse import urlencode import pytest import json @@ -10,7 +14,7 @@ from server.models import ( Service ) -from test_cases.factories import HostFactory +from test_cases.factories import HostFactory, EmptyCommandFactory @pytest.mark.usefixtures('logged_user') @@ -190,3 +194,26 @@ def test_update_cant_change_id(self, test_client, session): res = test_client.put(self.url(service, workspace=service.workspace), data=raw_data) assert res.status_code == 200 assert res.json['id'] == service.id + + def test_create_service_from_command(self, test_client, session): + host = HostFactory.create(workspace=self.workspace) + command = EmptyCommandFactory.create(workspace=self.workspacegoogle) + session.commit() + assert len(command.command_objects) == 0 + url = self.url(workspace=command.workspace) + '?' + urlencode({'command_id': command.id}) + raw_data = { + "name": "SSH", + "description": "SSH service", + "owned": False, + "ports": [22], + "protocol": "tcp", + "status": "open", + "parent": host.id + } + res = test_client.post(url, data=raw_data) + + assert res.status_code == 201 + assert len(command.command_objects) == 1 + cmd_obj = command.command_objects[0] + assert cmd_obj.object_type == 'service' + assert cmd_obj.object_id == res.json['id'] \ No newline at end of file diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 78eb05b1d3d..b2ac187f354 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1,6 +1,10 @@ #-*- coding: utf8 -*- import os from base64 import b64encode +try: + from urllib import urlencode +except ImportError: + from urllib.parse import urlencode import pytest import time @@ -1105,6 +1109,30 @@ def test_create_and_update_webvuln(self, host_with_hostnames, test_client, sessi data=raw_data) assert res.status_code == 200 + def test_create_vuln_from_command(self, test_client, session): + command = EmptyCommandFactory.create(workspace=self.workspace) + service = ServiceFactory.create(workspace=self.workspace) + session.commit() + assert len(command.command_objects) == 0 + url = self.url(workspace=command.workspace) + '?' + urlencode({'command_id': command.id}) + raw_data = self._create_post_data_vulnerability( + name='Update vulnsweb', + vuln_type='VulnerabilityWeb', + parent_id=service.id, + parent_type='Service', + refs=[], + policyviolations=[], + description='helloworld', + severity='high', + ) + res = test_client.post(url, data=raw_data) + + assert res.status_code == 201 + assert len(command.command_objects) == 1 + cmd_obj = command.command_objects[0] + assert cmd_obj.object_type == 'vulnerability' + assert cmd_obj.object_id == res.json['_id'] + def test_type_filter(workspace, session, vulnerability_factory, @@ -1154,4 +1182,4 @@ def test_creator_filter(workspace, session, VulnerabilityGeneric, 'creator', 'metasploit') - assert {v.id for v in filtered} == {v.id for v in vulns} + assert {v.id for v in filtered} == {v.id for v in vulns} \ No newline at end of file From 16d8c8fe595ad3f7838e14e861701ec416aa597a Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 12 Jan 2018 18:17:48 -0300 Subject: [PATCH 0788/1506] Fix bug: import_source was not set. Command was missing from object_types --- model/commands_history.py | 1 + persistence/server/server.py | 7 +++++-- persistence/server/utils.py | 3 ++- plugins/controller.py | 3 +++ requirements.txt | 1 + server/api/modules/vulns.py | 3 ++- server/models.py | 1 + 7 files changed, 15 insertions(+), 4 deletions(-) diff --git a/model/commands_history.py b/model/commands_history.py index b0910a36fb3..6e3f105171f 100644 --- a/model/commands_history.py +++ b/model/commands_history.py @@ -55,6 +55,7 @@ def __init__(self, **kwargs): self.duration = None self.params = None self.workspace = None + self.import_source = None self._id = None self.id_available = Event() diff --git a/persistence/server/server.py b/persistence/server/server.py index 689c7b74d2b..c9f94c73d4c 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -1358,7 +1358,7 @@ def update_credential(workspace_name, command_id, id, name, username, password, password=password, type="Cred") -def create_command(workspace_name, command, tool, duration=None, hostname=None, +def create_command(workspace_name, command, tool, import_source, duration=None, hostname=None, ip=None, itime=None, params=None, user=None): """Creates a command. @@ -1378,6 +1378,7 @@ def create_command(workspace_name, command, tool, duration=None, hostname=None, return _save_to_server(workspace_name, command=command, tool=tool, + import_source=import_source, duration=duration, hostname=hostname, ip=ip, @@ -1387,7 +1388,8 @@ def create_command(workspace_name, command, tool, duration=None, hostname=None, workspace=workspace_name, type="CommandRunInformation") -def update_command(workspace_name, command_id, command, tool, duration=None, hostname=None, +def update_command(workspace_name, command_id, command, tool, import_source +,duration=None, hostname=None, ip=None, itime=None, params=None, user=None): """Updates a command. @@ -1409,6 +1411,7 @@ def update_command(workspace_name, command_id, command, tool, duration=None, hos command_id, command=command, tool=tool, + import_source=import_source, duration=duration, hostname=hostname, ip=ip, diff --git a/persistence/server/utils.py b/persistence/server/utils.py index 9a50eb8b7d2..37a2edd8d3b 100644 --- a/persistence/server/utils.py +++ b/persistence/server/utils.py @@ -139,5 +139,6 @@ def get_command_properties(command): 'hostname': command.hostname, 'itime': command.itime, 'duration': command.duration, - 'params': command.params + 'params': command.params, + 'import_source': command.import_source, } diff --git a/plugins/controller.py b/plugins/controller.py index 60378b93027..67353311c4e 100644 --- a/plugins/controller.py +++ b/plugins/controller.py @@ -55,6 +55,7 @@ def commit(self): self.mapper_manager.update(self.command) def run(self): + name = '' try: self.output_queue.join() self.commit() @@ -239,6 +240,7 @@ def processCommandInput(self, pid, cmd, pwd): cmd_info = CommandRunInformation( **{'workspace': model.api.getActiveWorkspace().name, 'itime': time.time(), + 'import_source': 'shell', 'command': cmd.split()[0], 'params': ' '.join(cmd.split()[1:])}) cmd_info.setID(self._mapper_manager.save(cmd_info)) @@ -272,6 +274,7 @@ def processReport(self, plugin, filepath, ws_name=None): cmd_info = CommandRunInformation( **{'workspace': ws_name, 'itime': time.time(), + 'import_source': 'report', 'command': 'Import %s:' % plugin, 'params': filepath}) self._mapper_manager.createMappers(ws_name) diff --git a/requirements.txt b/requirements.txt index d8369b5a08a..d141627c414 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ restkit==4.2.2 tornado==4.5.1 tqdm==4.15.0 whoosh==2.7.4 +deprecation==1.0.1 \ No newline at end of file diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 8370f97f421..1f937456ef1 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -287,7 +287,8 @@ class Meta(FilterSetMeta): default_operator = operators.ILike column_overrides = { - field: _strict_filtering for field in strict_fields} + field: _strict_filtering for field in strict_fields + } operators = (operators.ILike, operators.Equal) target = TargetFilter(fields.Str()) type = TypeFilter(fields.Str(validate=[OneOf(['Vulnerability', diff --git a/server/models.py b/server/models.py index 9e8d5b39504..db2fcd5503e 100644 --- a/server/models.py +++ b/server/models.py @@ -51,6 +51,7 @@ 'credential', 'service', 'source_code', + 'comment', ] From 58201049747566c9df17a25ba420e8c0a4cbdd42 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 12 Jan 2018 18:28:49 -0300 Subject: [PATCH 0789/1506] Remove typo --- test_cases/test_api_services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index b88dc47628d..91d971bc00d 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -197,7 +197,7 @@ def test_update_cant_change_id(self, test_client, session): def test_create_service_from_command(self, test_client, session): host = HostFactory.create(workspace=self.workspace) - command = EmptyCommandFactory.create(workspace=self.workspacegoogle) + command = EmptyCommandFactory.create(workspace=self.workspace) session.commit() assert len(command.command_objects) == 0 url = self.url(workspace=command.workspace) + '?' + urlencode({'command_id': command.id}) From 086f6c9896f7570d18529f4b9b8e7c64c575b3a9 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 12 Jan 2018 18:30:34 -0300 Subject: [PATCH 0790/1506] add import_source to tests --- persistence/server/models.py | 1 + test_cases/test_managers_mapper_manager.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/persistence/server/models.py b/persistence/server/models.py index 45d9c04f75c..0d5ba5e611c 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -1318,6 +1318,7 @@ def __init__(self, command, workspace_name): self.params = command['params'] self.user = command['user'] self.workspace = command['workspace'] + self.import_source = command['import_source'] def getID(self): return self.id diff --git a/test_cases/test_managers_mapper_manager.py b/test_cases/test_managers_mapper_manager.py index fed8b9a4f23..2510eb31913 100644 --- a/test_cases/test_managers_mapper_manager.py +++ b/test_cases/test_managers_mapper_manager.py @@ -253,6 +253,7 @@ 'hostname': "mandarina", 'command': "Import Nexpose:", 'tool': "Import Nexpose:", + 'import_source': "report", 'user': "leonardo", 'workspace': "airbnb", 'duration': 0.164561, @@ -262,6 +263,7 @@ 'expected_payload':{ 'command': 'Import Nexpose:', 'tool': "Import Nexpose:", + 'import_source': "report", 'duration': 0.164561, 'hostname': 'mandarina', 'ip': '192.168.124.1', @@ -640,6 +642,7 @@ "hostname": "mandarina", "command": "Import Nessus:", "tool": "Import Nessus:", + "import_source": "report", "user": "lcubo", "workspace": "dsadsa", "duration": "In progress", @@ -650,6 +653,7 @@ 'command': 'Import Nessus:', "tool": "Import Nessus:", 'duration': 'In progress', + "import_source": "report", 'hostname': 'mandarina', 'ip': '192.168.20.53', 'itime': 1513365824, From 0e27825b6869ba0551540b5ddec40c4e38dd37c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 15 Jan 2018 12:55:31 -0300 Subject: [PATCH 0791/1506] Add extra log file for the couchdb importer Now there is an exclussive ~/.faraday/logs/couchdb-importer.log log file. The importers logs still in ~/.faraday/logs/faraday-server.log too. --- server/importer.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/importer.py b/server/importer.py index 99224eba600..2ee7dec09dc 100644 --- a/server/importer.py +++ b/server/importer.py @@ -5,6 +5,7 @@ import re import sys import json +import logging import datetime import requests @@ -64,6 +65,17 @@ logger = server.utils.logger.get_logger(__name__) +importer_logfile = os.path.expanduser(os.path.join( + server.config.CONSTANTS.CONST_FARADAY_HOME_PATH, + server.config.CONSTANTS.CONST_FARADAY_LOGS_PATH, 'couchdb-importer.log')) +importer_file_handler = logging.FileHandler(importer_logfile) +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +importer_file_handler.setFormatter(formatter) +importer_file_handler.setLevel(logging.DEBUG) +logger.addHandler(importer_file_handler) + + session = db.session MAPPED_VULN_SEVERITY = OrderedDict([ From 29c6f8ac3256da523c46bef2fbaf21e7baf4bf6e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 15 Jan 2018 14:21:23 -0300 Subject: [PATCH 0792/1506] Fix bug: processOutput takes command obj and not id --- plugins/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/controller.py b/plugins/controller.py index 60378b93027..94fed774511 100644 --- a/plugins/controller.py +++ b/plugins/controller.py @@ -262,7 +262,7 @@ def onCommandFinished(self, pid, exit_code, term_output): cmd_info.duration = time.time() - cmd_info.itime self._mapper_manager.update(cmd_info) - self.processOutput(plugin, term_output, cmd_info.getID()) + self.processOutput(plugin, term_output, cmd_info) del self._active_plugins[pid] return True From ba09c1230a10248597b143568382796fbbc53cad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 15 Jan 2018 14:50:40 -0300 Subject: [PATCH 0793/1506] Add create_user management command --- manage.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/manage.py b/manage.py index 9c0163f87ea..c80b2831752 100755 --- a/manage.py +++ b/manage.py @@ -1,17 +1,19 @@ #!/usr/bin/env python +import re import click import requests from requests import ConnectionError from sqlalchemy.exc import OperationalError from persistence.server.server import _conf, FARADAY_UP, SERVER_URL -from server.importer import ImportCouchDB from server.commands.initdb import InitDB from server.commands.faraday_schema_display import DatabaseSchema from server.commands.app_urls import show_all_urls from server.commands.reset_db import reset_db_all from server.commands.reports import import_external_reports +from server.models import db, User +from server.importer import ImportCouchDB from server.web import app from utils.logs import setUpLogger @@ -73,6 +75,38 @@ def database_schema(): DatabaseSchema().run() +def validate_user_unique_field(ctx, param, value): + with app.app_context(): + if User.query.filter_by(**{param.name: value}).count(): + raise click.ClickException("User already exists") + return value + + +def validate_email(ctx, param, value): + if not re.match(r'[^@]+@[^@]+\.[^@]+', value): + raise click.BadParameter('Invalid email') + + # Also validate that the email doesn't exist in the database + return validate_user_unique_field(ctx, param, value) + + +@click.command() +@click.option('--username', prompt=True, callback=validate_user_unique_field) +@click.option('--email', prompt=True, callback=validate_email) +@click.option('--password', prompt=True, hide_input=True, + confirmation_prompt=True) +def create_user(username, email, password): + with app.app_context(): + app.user_datastore.create_user(username=username, + email=email, + password=password, + is_ldap=False) + db.session.commit() + click.echo(click.style( + 'User {} created successfully!'.format(username), + fg='green', bold=True)) + + cli.add_command(process_reports) cli.add_command(reset_db) cli.add_command(show_urls) @@ -80,6 +114,7 @@ def database_schema(): cli.add_command(initdb) cli.add_command(import_from_couchdb) cli.add_command(database_schema) +cli.add_command(create_user) if __name__ == '__main__': From 3b0f88322ea534ace894ef108782697225aa7598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 15 Jan 2018 14:51:55 -0300 Subject: [PATCH 0794/1506] Remove textual SQL warning --- server/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index 3e16d85408c..d64c1ec8d72 100644 --- a/server/models.py +++ b/server/models.py @@ -194,7 +194,7 @@ class Host(Metadata): __host_vulnerabilities = ( select([func.count(text('vulnerability.id'))]). - select_from('vulnerability'). + select_from(text('vulnerability')). where(text('vulnerability.host_id = host.id')). as_scalar() ) From 44f03b7a96399c5b07a76930247ccfc8f258d725 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 15 Jan 2018 17:24:27 -0300 Subject: [PATCH 0795/1506] Bug fix: avoid duplicate hosts on gtk sidebar when a new host was created. add name <-> ip host compatibility --- gui/gtk/mainwidgets.py | 9 +++++---- persistence/server/models.py | 4 ++++ server/events.py | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/gui/gtk/mainwidgets.py b/gui/gtk/mainwidgets.py index 8d6bc33a78d..7703543282c 100644 --- a/gui/gtk/mainwidgets.py +++ b/gui/gtk/mainwidgets.py @@ -253,9 +253,10 @@ def _add_single_host_to_model(self, host): vuln_count = host.getVulnAmount() os_icon, os_str = self.__decide_icon(host.getOS()) display_str = str(host) - host_iter = self.model.append([str(host.id), os_icon, os_str, display_str, vuln_count]) - self.host_id_to_iter[host.id] = host_iter - self.host_amount_in_model += 1 + if str(host.id) not in map(lambda host_data: host_data[0], self.model): + host_iter = self.model.append([str(host.id), os_icon, os_str, display_str, vuln_count]) + self.host_id_to_iter[host.id] = host_iter + self.host_amount_in_model += 1 def add_relevant_hosts_to_model(self, hosts): """Takes a list of hosts. Add the hosts to the model without going @@ -481,7 +482,7 @@ def update_progress_label(self): def create_search_entry(self): """Returns a simple search entry""" self.search_entry = Gtk.Entry() - self.search_entry.set_placeholder_text("Search a host by name...") + self.search_entry.set_placeholder_text("Search a host by ip...") self.search_entry.connect("activate", self.on_search_enter_key) self.search_entry.show() return self.search_entry diff --git a/persistence/server/models.py b/persistence/server/models.py index 45d9c04f75c..964f93618d8 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -826,6 +826,10 @@ def __init__(self, host, workspace_name): self.default_gateway = host.get('default_gateway') self.os = host.get('os') if host.get('os') else 'unknown' self.vuln_amount = int(host.get('vulns', 0)) + self.ip = host.get('ip', self.name) + + def getName(self): + return self.ip @staticmethod def publicattrsrefs(): diff --git a/server/events.py b/server/events.py index cc14955584e..5760beaa1f6 100644 --- a/server/events.py +++ b/server/events.py @@ -42,7 +42,7 @@ def update_object_event(mapper, connection, instance): # sometimes apis will commit to db to have fk. # this will avoid duplicate messages on websockets return - name = getattr(instance, 'ip', None) or instance.name + name = getattr(instance, 'ip', None) or getattr(instance, 'name', None) msg = { 'id': instance.id, 'action': 'UPDATE', From 869a09e7bbe364495fee52cc047012c4346c06b0 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 15 Jan 2018 17:45:36 -0300 Subject: [PATCH 0796/1506] Fix more info for a host in gtk --- gui/gtk/dialogs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gui/gtk/dialogs.py b/gui/gtk/dialogs.py index 553b60a7ab5..bc48bcdf266 100644 --- a/gui/gtk/dialogs.py +++ b/gui/gtk/dialogs.py @@ -881,9 +881,8 @@ def create_model(self, host): # the other columns with dummy info display_str = host.getName() + " (" + str(len(host.getVulns())) + ")" - # display_str = str(host) owned_status = ("Yes" if host.isOwned() else "No") - host_position = model.append(None, [host.getID(), host.getName(), + model.append(None, [str(host.getID()), host.getName(), host.getOS(), owned_status, str(len(host.getVulns())), "", "", "", "", "", "", "", @@ -897,6 +896,8 @@ def lst_to_str(lst): """Convenient function to avoid this long line everywhere""" return ', '.join([str(word) for word in lst if word]) + return model + @scrollable(width=250) def create_main_tree_view(self, model): """Return a box containing the main tree (the one showing From dbb4d9e29246d8fc69bfbf1740b253dc86ff158d Mon Sep 17 00:00:00 2001 From: micabot Date: Mon, 15 Jan 2018 18:50:18 -0300 Subject: [PATCH 0797/1506] Fix alphabetical order --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d141627c414..740131981af 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ websocket-client==0.46.0 autobahn==17.10.1 argparse==1.4.0 colorama==0.3.9 +deprecation==1.0.1 flask==0.12.2 IPy==0.83 mockito==1.0.12 @@ -10,4 +11,3 @@ restkit==4.2.2 tornado==4.5.1 tqdm==4.15.0 whoosh==2.7.4 -deprecation==1.0.1 \ No newline at end of file From 452039bb0bbfc61497546b47dc336351ae8894ff Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Mon, 15 Jan 2018 19:35:52 -0300 Subject: [PATCH 0798/1506] New plugin: Ip360 --- plugins/repo/ip360/__init__.py | 5 ++ plugins/repo/ip360/plugin.py | 122 +++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 plugins/repo/ip360/__init__.py create mode 100644 plugins/repo/ip360/plugin.py diff --git a/plugins/repo/ip360/__init__.py b/plugins/repo/ip360/__init__.py new file mode 100644 index 00000000000..19a0420d0c8 --- /dev/null +++ b/plugins/repo/ip360/__init__.py @@ -0,0 +1,5 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2018 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information +''' diff --git a/plugins/repo/ip360/plugin.py b/plugins/repo/ip360/plugin.py new file mode 100644 index 00000000000..9f37341e7a0 --- /dev/null +++ b/plugins/repo/ip360/plugin.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +''' +Faraday Penetration Test IDE +Copyright (C) 2018 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information +''' + +import csv +import StringIO +from plugins import core + +def calculate_severity(number): + + if number is None: + return "info" + + number = float(number) + + # Based in CVSS V2 + if number >= 0 and number <= 3.9: + return "low" + elif number >= 4.0 and number <= 6.9: + return "med" + elif number >= 7.0 and number <= 10: + return "high" + +class Ip360Parser: + + def __init__(self, csv_content): + self.csv_content = StringIO.StringIO(csv_content.decode('ascii', 'ignore')) + self.csv_reader = csv.DictReader(self.csv_content, delimiter=',', quotechar='"') + + def parse(self): + + result = [] + for row in self.csv_reader: + + host = { + "name": row.get("IP"), + "os": row.get("OS") + } + + interface = { + "name": row.get("IP"), + "hostname_resolution": [row.get("NetBIOS Name")], + "network_segment": row.get("NetBIOS Domain"), + } + + service = {"port": row.get("Port")} + + vulnerability = { + "name": row.get("Vulnerability"), + "description": row.get("Description"), + "resolution": row.get("Remediation"), + "ref": [ + row.get("CVE"), + "Vuln ID: " + row.get("Vulnerability ID"), + "Risk: " + row.get("Risk"), + "Skill: " + row.get("Skill"), + "CVSS V2: " + row.get("CVSS V2"), + "CVSS V3: " + row.get("CVSS V3")], + "severity": row.get("CVSS V2") + } + + result.append((host, interface, service, vulnerability)) + + return result + +class Ip360Plugin(core.PluginBase): + """ + Example plugin to parse Ip360 output. + """ + + def __init__(self): + core.PluginBase.__init__(self) + self.id = "Ip360" + self.name = "Ip360 CSV Output Plugin" + self.plugin_version = "0.0.1" + self.options = None + + def parseOutputString(self, output, debug=False): + + parser = Ip360Parser(output) + for host, interface, service, vulnerability in parser.parse(): + + h_id = self.createAndAddHost(host.get("name"), host.get("os")) + + i_id = self.createAndAddInterface( + h_id, + interface.get("name"), + ipv4_address=interface.get("name"), + hostname_resolution=interface.get("hostname_resolution"), + network_segment=interface.get("network_segment")) + + + if service.get("port") == "-": + port = "0" + protocol = "unknown" + else: + port = service.get("port").split("/")[0] + protocol = service.get("port").split("/")[1] + + s_id = self.createAndAddServiceToInterface( + h_id, + i_id, + service.get("port"), + protocol=protocol, + ports=[port]) + + self.createAndAddVulnToService( + h_id, + s_id, + vulnerability.get("name"), + desc=vulnerability.get("description"), + resolution=vulnerability.get("resolution"), + severity=calculate_severity(vulnerability.get("severity")), + ref=vulnerability.get("ref")) + +def createPlugin(): + return Ip360Plugin() \ No newline at end of file From 5aeffeb30aaa5c40becad8af5fca6b30430eaf2b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 16 Jan 2018 11:35:03 -0300 Subject: [PATCH 0799/1506] Add sql shell. Uses db conn from server.ini --- manage.py | 12 ++++++++++++ requirements.txt | 1 + 2 files changed, 13 insertions(+) diff --git a/manage.py b/manage.py index c80b2831752..dc2384be387 100755 --- a/manage.py +++ b/manage.py @@ -1,11 +1,14 @@ #!/usr/bin/env python import re + import click import requests from requests import ConnectionError from sqlalchemy.exc import OperationalError +from pgcli.main import PGCli +import server.config from persistence.server.server import _conf, FARADAY_UP, SERVER_URL from server.commands.initdb import InitDB from server.commands.faraday_schema_display import DatabaseSchema @@ -74,6 +77,14 @@ def import_from_couchdb(): def database_schema(): DatabaseSchema().run() +@click.command() +def sql_shell(): + conn_string = server.config.database.connection_string.strip("'") + + pgcli = PGCli() + pgcli.connect_uri(conn_string) + pgcli.run_cli() + def validate_user_unique_field(ctx, param, value): with app.app_context(): @@ -115,6 +126,7 @@ def create_user(username, email, password): cli.add_command(import_from_couchdb) cli.add_command(database_schema) cli.add_command(create_user) +cli.add_command(sql_shell) if __name__ == '__main__': diff --git a/requirements.txt b/requirements.txt index 740131981af..c4948082d6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ deprecation==1.0.1 flask==0.12.2 IPy==0.83 mockito==1.0.12 +pgcli==1.8.2 requests==2.18.4 restkit==4.2.2 tornado==4.5.1 From 36812facb57b905b3f65f546529c208621f422e3 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 17 Jan 2018 10:55:18 -0300 Subject: [PATCH 0800/1506] Add unit test: trying to create service without parent --- test_cases/test_api_services.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index 91d971bc00d..f1199e27648 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -216,4 +216,19 @@ def test_create_service_from_command(self, test_client, session): assert len(command.command_objects) == 1 cmd_obj = command.command_objects[0] assert cmd_obj.object_type == 'service' - assert cmd_obj.object_id == res.json['id'] \ No newline at end of file + assert cmd_obj.object_id == res.json['id'] + + + def test_create_service_without_ost(self, test_client, host, session): + session.commit() + data = { + "name": "ftp", + "description": "test. test", + "owned": False, + "ports": [21], + "protocol": "tcp", + "status": "open", + } + res = test_client.post(self.url(), data=data) + assert res.status_code == 400 + assert res.json['messages']['_schema'] == res.json['messages']['_schema'] \ No newline at end of file From 7602463fee064b85936b051138be077bd3b320ef Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 17 Jan 2018 11:08:35 -0300 Subject: [PATCH 0801/1506] Add unit test to create a credential with invalid parent type --- test_cases/test_api_credentials.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test_cases/test_api_credentials.py b/test_cases/test_api_credentials.py index b0bb6855e8c..26aabe36e7e 100644 --- a/test_cases/test_api_credentials.py +++ b/test_cases/test_api_credentials.py @@ -136,6 +136,30 @@ def test_update_credentials_with_invalid_parent(self, test_client, session): res = test_client.put(self.url(workspace=credential.workspace) + str(credential.id) + '/', data=raw_data) assert res.status_code == 400 + def test_create_with_invalid_parent_type( + self, session, test_client, workspace, service_factory): + service = service_factory.create(workspace=workspace) + session.commit() + raw_data = { + "_id":"1.e5069bb0718aa519852e6449448eedd717f1b90d", + "name":"name", + "username":"username", + "metadata":{"update_time":1508794240799,"update_user":"", + "update_action":0,"creator":"UI Web", + "create_time":1508794240799,"update_controller_action":"", + "owner":""}, + "password":"pass", + "type":"Cred", + "owner":"", + "description":"", + "parent": service.id, + "parent_type": "Vulnerability" + } + res = test_client.post(self.url(), data=raw_data) + assert res.status_code == 400 + assert res.json['messages']['_schema'] == ['Unknown parent type: Vulnerability'] + + def test_update_credentials(self, test_client, session, host): credential = self.factory.create(host=host, service=None, workspace=self.workspace) From 1570779627cf32fcf548ff17fe7499542e318020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 17 Jan 2018 16:29:15 -0300 Subject: [PATCH 0802/1506] Fix filteralchemy requirement It is required to use my fork with some fixes to the library --- Jenkinsfile | 2 -- requirements_server.txt | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9a318a66531..b585cd80472 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,8 +24,6 @@ node (label: "master"){ pip install -r $WORKSPACE/requirements_server.txt pip install -r $WORKSPACE/requirements_extras.txt pip install -r $WORKSPACE/requirements_dev.txt - pip uninstall -y filteralchemy - pip install -e git+https://github.com/sh4r3m4n/filteralchemy@dev#egg=filteralchemy deactivate """ } diff --git a/requirements_server.txt b/requirements_server.txt index bc68387a4ff..d121bc269a0 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -18,7 +18,7 @@ tqdm==4.15.0 twisted==17.5.0 webargs marshmallow-sqlalchemy -filteralchemy +git+https://github.com/sh4r3m4n/filteralchemy@dev#egg=filteralchemy filedepot==0.5.0 nplusone==0.8.1 deprecation==1.0.1 From 99c73f28e17d064c77566096c1ae4d434970d8fb Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 17 Jan 2018 16:59:34 -0300 Subject: [PATCH 0803/1506] Fix to ignore github sources in dependencies files --- utils/dependencies.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/dependencies.py b/utils/dependencies.py index a974752a087..0d43f0ab149 100644 --- a/utils/dependencies.py +++ b/utils/dependencies.py @@ -12,7 +12,8 @@ def check_dependencies(requirements_file='requirements.txt'): dependencies_file = open(requirements_file, 'r') - requirements = list(pkg_resources.parse_requirements(dependencies_file)) + requirements = list(pkg_resources.parse_requirements([x for x in + dependencies_file.readlines() if not x.startswith('git+')])) installed = [] missing = [] From d1d146d205f0a84723aded642fbcd523299d4c69 Mon Sep 17 00:00:00 2001 From: micabot Date: Wed, 17 Jan 2018 17:30:57 -0300 Subject: [PATCH 0804/1506] Remove "name" form host, it is now IP --- server/www/scripts/hosts/partials/list.html | 2 +- server/www/scripts/hosts/partials/modalEdit.html | 4 ++-- server/www/scripts/services/partials/list.html | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/www/scripts/hosts/partials/list.html b/server/www/scripts/hosts/partials/list.html index 3a7fc87b8d7..ffb5fbb7997 100644 --- a/server/www/scripts/hosts/partials/list.html +++ b/server/www/scripts/hosts/partials/list.html @@ -67,7 +67,7 @@

    Add columns

    		 - Name + IP diff --git a/server/www/scripts/hosts/partials/modalEdit.html b/server/www/scripts/hosts/partials/modalEdit.html index 716004c1e78..3c831a340b7 100644 --- a/server/www/scripts/hosts/partials/modalEdit.html +++ b/server/www/scripts/hosts/partials/modalEdit.html @@ -14,8 +14,8 @@

    Edit host

    - - + + Example: 192.168.0.1 diff --git a/server/www/scripts/services/partials/list.html b/server/www/scripts/services/partials/list.html index 56345fd6f7a..cee8d22861f 100644 --- a/server/www/scripts/services/partials/list.html +++ b/server/www/scripts/services/partials/list.html @@ -152,9 +152,9 @@

    - Name -
    Name
    - + IP +
    IP v4 or v6
    +
    {{host.ip}} From 772abb7e21d198a586f7781af49e863c7f395110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 18 Jan 2018 13:44:47 -0300 Subject: [PATCH 0805/1506] Show service summaries in the hosts API --- server/api/modules/hosts.py | 7 +++++-- server/api/modules/services.py | 4 ---- server/models.py | 4 ++++ test_cases/test_api_hosts.py | 22 ++++++++++++++++++++++ 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 99562e48b6c..a5e437e8907 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -52,13 +52,16 @@ class HostSchema(AutoSchema): fields.List(fields.String)) metadata = SelfNestedField(MetadataSchema()) type = fields.Function(lambda obj: 'Host', dump_only=True) + service_summaries = PrimaryKeyRelatedField('summary', attribute='services', + many=True, + dump_only=True) class Meta: model = Host fields = ('id', '_id', '_rev', 'ip', 'description', 'mac', 'credentials', 'default_gateway', 'metadata', 'name', 'os', 'owned', 'owner', 'services', 'vulns', - 'hostnames', 'type', + 'hostnames', 'type', 'service_summaries' ) @@ -89,7 +92,7 @@ class HostsView(PaginatedMixin, get_undefer = [Host.credentials_count, Host.open_service_count, Host.vulnerability_count] - get_joinedloads = [Host.hostnames] + get_joinedloads = [Host.hostnames, Host.services] @route('//services/') def service_list(self, workspace_name, host_id): diff --git a/server/api/modules/services.py b/server/api/modules/services.py index 6b54f3a5e4d..bbe2655c46f 100644 --- a/server/api/modules/services.py +++ b/server/api/modules/services.py @@ -36,7 +36,6 @@ class ServiceSchema(AutoSchema): status = fields.String(default='open') parent = fields.Integer(attribute='host_id') # parent is not required for updates host_id = fields.Integer(attribute='host_id', dump_only=True) - summary = fields.Method('get_summary') vulns = fields.Integer(attribute='vulnerability_count', dump_only=True) credentials = fields.Integer(attribute='credentials_count', dump_only=True) metadata = SelfNestedField(MetadataSchema()) @@ -46,9 +45,6 @@ def load_ports(self, value): # TODO migration: handle empty list and not numeric value return str(value.pop()) - def get_summary(self, obj): - return "(%s/%s) %s" % (obj.port, obj.protocol, obj.name) - @post_load def post_load_parent(self, data): """Gets the host_id from parent attribute. Pops it and tries to diff --git a/server/models.py b/server/models.py index 76ff5c11604..973c7f16651 100644 --- a/server/models.py +++ b/server/models.py @@ -349,6 +349,10 @@ class Service(Metadata): def parent(self): return self.host + @property + def summary(self): + return "(%s/%s) %s" % (self.port, self.protocol, self.name) + class VulnerabilityABC(Metadata): # revisar plugin nexpose, netspark para terminar de definir uniques. asegurar que se carguen bien diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index caf10306653..8fe0982c7eb 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -465,6 +465,7 @@ def test_update_host(self, test_client, session): u'owned': False, u'owner': host.creator.username, u'services': 0, + u'service_summaries': [], u'vulns': 0} @@ -585,3 +586,24 @@ def test_create_host_cant_assign_command_from_another_workspace(self, test_clien assert res.status_code == 400 assert res.json == {u'message': u'Command not found.'} assert len(command.command_objects) == 0 + + def test_service_summaries(self, test_client, session, service_factory): + service_factory.create(name='http', protocol='tcp', port=80, + host=self.first_object, + workspace=self.workspace) + service_factory.create(name='https', protocol='tcp', port=443, + host=self.first_object, + workspace=self.workspace) + service_factory.create(name='dns', protocol='udp', port=53, + host=self.first_object, + workspace=self.workspace) + session.commit() + res = test_client.get(self.url(self.first_object)) + assert res.status_code == 200 + service_summaries = res.json['service_summaries'] + service_summaries.sort() # TODO migration: quit this and define sort + assert service_summaries == [ + '(443/tcp) https', + '(53/udp) dns', + '(80/tcp) http', + ] From 36d266bd63e43e876602462c7b733c6f8dff105c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 18 Jan 2018 13:59:18 -0300 Subject: [PATCH 0806/1506] Add optional services column in host list view --- server/www/scripts/hosts/controllers/hosts.js | 7 ++--- server/www/scripts/hosts/partials/list.html | 27 ++++++++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/server/www/scripts/hosts/controllers/hosts.js b/server/www/scripts/hosts/controllers/hosts.js index af53a83b71a..3421857330b 100644 --- a/server/www/scripts/hosts/controllers/hosts.js +++ b/server/www/scripts/hosts/controllers/hosts.js @@ -19,10 +19,11 @@ angular.module('faradayApp') "name": true, "description": false, "hostnames": false, + "services": false, "mac": false, - "services": true, - "vulns": true, - "credentials": true, + "service_count": true, + "vuln_count": true, + "credential_count": true, "os": true, "owned": true, "create_time": true, diff --git a/server/www/scripts/hosts/partials/list.html b/server/www/scripts/hosts/partials/list.html index ffb5fbb7997..a4c2b7c6b06 100644 --- a/server/www/scripts/hosts/partials/list.html +++ b/server/www/scripts/hosts/partials/list.html @@ -82,21 +82,25 @@

    Add columns

    Hostnames

    		+ Services + + MAC + Open Services - + + Vulns - + + Credentials - + Owned @@ -137,12 +141,17 @@

    Add columns

    +     		+ + {{summary}}
    +     +     		{{host.mac}} + {{host.credentials}} From f067f9ba206035bfdbb99e7332c06d040c4266b5 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 18 Jan 2018 14:49:51 -0300 Subject: [PATCH 0807/1506] Remove prints and use logger.debug --- server/websocket_factories.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/server/websocket_factories.py b/server/websocket_factories.py index d955207afe6..3a41659b311 100644 --- a/server/websocket_factories.py +++ b/server/websocket_factories.py @@ -1,4 +1,6 @@ import json +import logging + import Cookie from collections import defaultdict from Queue import Queue, Empty @@ -11,7 +13,7 @@ WebSocketServerProtocol ) - +logger =logging.getLogger(__name__) changes_queue = Queue() @@ -20,11 +22,8 @@ class BroadcastServerProtocol(WebSocketServerProtocol): def onConnect(self, request): protocol, headers = None, {} # see if there already is a cookie set .. - print request + logger.debug('Websocket request {0}'.format(request)) if 'cookie' in request.headers: - print 'cookie' - print (str(request.headers['cookie'])) - print 'cookie' try: cookie = Cookie.SimpleCookie() cookie.load(str(request.headers['cookie'])) @@ -85,13 +84,13 @@ def tick(self): reactor.callLater(0.5, self.tick) def join_workspace(self, client, workspace): - print('Join workspace {0}'.format(workspace)) + logger.debug('Join workspace {0}'.format(workspace)) if client not in self.workspace_clients[workspace]: - print("registered client {}".format(client.peer)) + logger.debug("registered client {}".format(client.peer)) self.workspace_clients[workspace].append(client) def leave_workspace(self, client, workspace_name): - print('Leave workspace {0}'.format(workspace_name)) + logger.debug('Leave workspace {0}'.format(workspace_name)) self.workspace_clients[workspace_name].remove(client) def unregister(self, client_to_unregister): @@ -101,13 +100,13 @@ def unregister(self, client_to_unregister): for workspace_name, clients in self.workspace_clients.items(): for client in clients: if client == client_to_unregister: - print("unregistered client from workspace {0}".format(workspace_name)) + logger.debug("unregistered client from workspace {0}".format(workspace_name)) self.leave_workspace(client, workspace_name) return def broadcast(self, msg): - print("broadcasting prepared message '{}' ..".format(msg)) + logger.debug("broadcasting prepared message '{}' ..".format(msg)) prepared_msg = json.loads(self.prepareMessage(msg).payload) for client in self.workspace_clients[prepared_msg['workspace']]: reactor.callFromThread(client.sendPreparedMessage, self.prepareMessage(msg)) - print("prepared message sent to {}".format(client.peer)) + logger.debug("prepared message sent to {}".format(client.peer)) From 7be87e2a75c70c91b423dd65b53876ca9923dddf Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 18 Jan 2018 14:54:07 -0300 Subject: [PATCH 0808/1506] Fix info disclosure. Warn used that the port is for websockets --- server/websocket_factories.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/websocket_factories.py b/server/websocket_factories.py index 3a41659b311..fb4d2169e0a 100644 --- a/server/websocket_factories.py +++ b/server/websocket_factories.py @@ -50,6 +50,9 @@ def connectionLost(self, reason): WebSocketServerProtocol.connectionLost(self, reason) self.factory.unregister(self) + def sendServerStatus(self, redirectUrl=None, redirectAfter=0): + self.sendHtml('This is a websocket port.') + class WorkspaceServerFactory(WebSocketServerFactory): """ From 1537ad5747be3262aaa592f8ed02a34ae25fae7e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 18 Jan 2018 14:57:22 -0300 Subject: [PATCH 0809/1506] Remove print and use logger instead --- server/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/web.py b/server/web.py index 4bae93b239b..d286a088909 100644 --- a/server/web.py +++ b/server/web.py @@ -84,7 +84,7 @@ def __build_api_resource(self): return WSGIResource(reactor, reactor.getThreadPool(), app) def __build_websockets_resource(self): - print(u"wss://{0}:9000".format(self.__bind_address)) + logger.info(u"Websocket listening at wss://{0}:9000".format(self.__bind_address)) factory = WorkspaceServerFactory(u"ws://{0}:9000".format(self.__bind_address)) factory.protocol = BroadcastServerProtocol return factory From 41aacf559d39dc449409a5ce9d7bb31487a3bc18 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 18 Jan 2018 15:08:26 -0300 Subject: [PATCH 0810/1506] Disable directory listing --- server/web.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/server/web.py b/server/web.py index d286a088909..8f0698906e1 100644 --- a/server/web.py +++ b/server/web.py @@ -7,7 +7,7 @@ import functools import twisted.web -from twisted.web.resource import Resource +from twisted.web.resource import Resource, ForbiddenResource # Ugly hack to make "flask shell" work. It works because when running via flask # shell, __file__ will be server/web.py instead of faraday-server.py @@ -22,7 +22,6 @@ from twisted.web.util import Redirect from twisted.web.wsgi import WSGIResource from autobahn.twisted.websocket import ( - WebSocketServerFactory, listenWS ) import server.config @@ -38,6 +37,11 @@ logger = server.utils.logger.get_logger(__name__) +class FileWithoutDirectoryListing(File): + + def directoryListing(self): + return ForbiddenResource() + class WebServer(object): HOME = '' UI_URL_PATH = '_ui' @@ -78,7 +82,7 @@ def __build_web_redirect(self): return Redirect(WebServer.UI_URL_PATH) def __build_web_resource(self): - return File(WebServer.WEB_UI_LOCAL_PATH) + return FileWithoutDirectoryListing(WebServer.WEB_UI_LOCAL_PATH) def __build_api_resource(self): return WSGIResource(reactor, reactor.getThreadPool(), app) From d934474e0533db3972b98bd02a13f480687b141f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 18 Jan 2018 15:09:31 -0300 Subject: [PATCH 0811/1506] Add missing dep psycopg2 --- requirements_server.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_server.txt b/requirements_server.txt index d121bc269a0..b06a97db300 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -6,6 +6,7 @@ Flask-Security==3.0.0 flask==0.12.2 marshmallow Pillow==4.2.1 +psycopg2==2.7.3.2 pyasn1-modules>=0.0.11 pyopenssl==17.2.0 python-slugify==1.2.4 From 0abfed37836094e2068db913716b46a92c2e3c90 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 18 Jan 2018 15:11:27 -0300 Subject: [PATCH 0812/1506] Remove old sh script --- test_cases/run_testcases.sh | 7 ------- 1 file changed, 7 deletions(-) delete mode 100755 test_cases/run_testcases.sh diff --git a/test_cases/run_testcases.sh b/test_cases/run_testcases.sh deleted file mode 100755 index aa52e9c719f..00000000000 --- a/test_cases/run_testcases.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env sh -cd .. -# nosetests2 --no-byte-compile --with-coverage --cover-html --cover-html-dir=cover --cover-package=auth --cover-package=bin --cover-package=config --cover-package=exporters --cover-package=external --cover-package=gui --cover-package=managers --cover-package=model --cover-package=persistence --cover-package=plugins --cover-package=shell --cover-package=utils test_cases/model_controller.py -# nosetests2 --no-byte-compile --with-coverage --cover-html --cover-html-dir=cover --cover-package=auth --cover-package=bin --cover-package=config --cover-package=exporters --cover-package=external --cover-package=gui --cover-package=managers --cover-package=model --cover-package=persistence --cover-package=plugins --cover-package=shell --cover-package=utils test_cases/model_controller.py -# nosetests2 --with-coverage --cover-html --cover-html-dir=cover --cover-package=model test_cases/*.py -nosetests --ignore-files='.*dont_run_rest_controller_apis.*' --no-byte-compile -v `find test_cases -name '*.py' | grep -v dont_run` - From 0007ea325aa7393ef3dac3c09e1a153da64abd04 Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 18 Jan 2018 15:33:04 -0300 Subject: [PATCH 0813/1506] Fix PEP8 --- utils/dependencies.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utils/dependencies.py b/utils/dependencies.py index 0d43f0ab149..8d80fd42ad9 100644 --- a/utils/dependencies.py +++ b/utils/dependencies.py @@ -11,9 +11,10 @@ def check_dependencies(requirements_file='requirements.txt'): dependencies_file = open(requirements_file, 'r') + filtered_deps = [x for x in dependencies_file.readlines() if not + x.startswith('git+')] - requirements = list(pkg_resources.parse_requirements([x for x in - dependencies_file.readlines() if not x.startswith('git+')])) + requirements = list(pkg_resources.parse_requirements(filtered_deps)) installed = [] missing = [] From 082d99a679ef62e4e38ff0a8f698f56c08e48e6d Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 18 Jan 2018 16:05:16 -0300 Subject: [PATCH 0814/1506] Fix error message when conflicts occur in SR --- .../www/scripts/statusReport/controllers/statusReport.js | 4 ++-- server/www/scripts/vulns/providers/vuln.js | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 261538e6d7e..353a0e9865c 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -895,9 +895,9 @@ angular.module('faradayApp') $scope.insert = function(vuln) { vulnsManager.createVuln($scope.workspace, vuln).then(function() { loadVulns(); - }, function(message) { + }, function(data) { var msg = "The vulnerability couldn't be created"; - if(message == "409") { + if(data.status == 409) { msg += " because a vulnerability with the same parameters already exists in this Workspace"; } showMessage(msg); diff --git a/server/www/scripts/vulns/providers/vuln.js b/server/www/scripts/vulns/providers/vuln.js index 23380ceaace..449dbcec275 100644 --- a/server/www/scripts/vulns/providers/vuln.js +++ b/server/www/scripts/vulns/providers/vuln.js @@ -191,11 +191,11 @@ angular.module('faradayApp') self._save(resp, false) .then(function(data) { deferred.resolve(self); - }, function(data, status, headers, config) { - deferred.reject(status); + }, function(data) { + deferred.reject(data); }); - }, function() { - deferred.reject(); + }, function(data) { + deferred.reject(data); }); return deferred.promise; From 46448246999343d6a51abd39658aa84d9151fa6e Mon Sep 17 00:00:00 2001 From: micabot Date: Thu, 18 Jan 2018 16:15:58 -0300 Subject: [PATCH 0815/1506] Remove service name requirement on creation --- server/www/scripts/services/partials/modalNew.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/services/partials/modalNew.html b/server/www/scripts/services/partials/modalNew.html index db60e26a68a..721ce7f0ee7 100644 --- a/server/www/scripts/services/partials/modalNew.html +++ b/server/www/scripts/services/partials/modalNew.html @@ -12,10 +12,10 @@

    New service

    -
    +
    - +
    From cb04ea345e0e98d24c406d4db9214fb99998baa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 18 Jan 2018 16:42:04 -0300 Subject: [PATCH 0816/1506] Only show open services in services_summaries --- server/api/modules/hosts.py | 10 +++++++--- test_cases/test_api_hosts.py | 12 +++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index a5e437e8907..8fb8a63d63e 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -52,9 +52,8 @@ class HostSchema(AutoSchema): fields.List(fields.String)) metadata = SelfNestedField(MetadataSchema()) type = fields.Function(lambda obj: 'Host', dump_only=True) - service_summaries = PrimaryKeyRelatedField('summary', attribute='services', - many=True, - dump_only=True) + service_summaries = fields.Method('get_service_summaries', + dump_only=True) class Meta: model = Host @@ -64,6 +63,11 @@ class Meta: 'hostnames', 'type', 'service_summaries' ) + def get_service_summaries(self, obj): + return [service.summary + for service in obj.services + if service.status == 'open'] + class ServiceFilter(Filter): """Filter hosts by service name""" diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 8fe0982c7eb..61770ee47b8 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -589,13 +589,19 @@ def test_create_host_cant_assign_command_from_another_workspace(self, test_clien def test_service_summaries(self, test_client, session, service_factory): service_factory.create(name='http', protocol='tcp', port=80, - host=self.first_object, + host=self.first_object, status='open', workspace=self.workspace) service_factory.create(name='https', protocol='tcp', port=443, - host=self.first_object, + host=self.first_object, status='open', workspace=self.workspace) service_factory.create(name='dns', protocol='udp', port=53, - host=self.first_object, + host=self.first_object, status='open', + workspace=self.workspace) + service_factory.create(name='smtp', protocol='tcp', port=25, + host=self.first_object, status='filtered', + workspace=self.workspace) + service_factory.create(name='other', protocol='udp', port=1234, + host=self.first_object, status='closed', workspace=self.workspace) session.commit() res = test_client.get(self.url(self.first_object)) From e22cbf5a5a5b8452230efa5f17d058da8853a921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 18 Jan 2018 16:57:04 -0300 Subject: [PATCH 0817/1506] Sort service_summaries in hosts API First by protocol, then by port number. The sort is defined globally in the relation Host.services --- RELEASE.md | 2 +- server/models.py | 9 ++++++++- test_cases/test_api_hosts.py | 9 ++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 69e35cea461..b560a3f58d8 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -19,7 +19,7 @@ TBA: * Remove vuln template CWE fields, join it with references * Allow to search hosts by hostname * Allow the user to specify the desired fields of the host list table -* Add optional hostnames, MAC and description fields to the host list +* Add optional hostnames, services, MAC and description fields to the host list * Workspace names can be changed from the Web UI * Changed the scope field of a workspace from a free text input to a list of targets diff --git a/server/models.py b/server/models.py index 973c7f16651..5332e38bfca 100644 --- a/server/models.py +++ b/server/models.py @@ -189,6 +189,11 @@ class Host(Metadata): mac = Column(Text, nullable=True) net_segment = Column(Text, nullable=True) + services = relationship( + 'Service', + order_by='Service.protocol,Service.port', + ) + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship( @@ -328,7 +333,9 @@ class Service(Metadata): banner = Column(Text, nullable=True) host_id = Column(Integer, ForeignKey('host.id'), index=True, nullable=False) - host = relationship('Host', backref='services', foreign_keys=[host_id]) + host = relationship( + 'Host', + foreign_keys=[host_id]) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship( diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 61770ee47b8..f44a719822f 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -594,12 +594,15 @@ def test_service_summaries(self, test_client, session, service_factory): service_factory.create(name='https', protocol='tcp', port=443, host=self.first_object, status='open', workspace=self.workspace) - service_factory.create(name='dns', protocol='udp', port=53, + service_factory.create(name='dns', protocol='udp', port=5353, host=self.first_object, status='open', workspace=self.workspace) service_factory.create(name='smtp', protocol='tcp', port=25, host=self.first_object, status='filtered', workspace=self.workspace) + service_factory.create(name='dns', protocol='udp', port=53, + host=self.first_object, status='open', + workspace=self.workspace) service_factory.create(name='other', protocol='udp', port=1234, host=self.first_object, status='closed', workspace=self.workspace) @@ -607,9 +610,9 @@ def test_service_summaries(self, test_client, session, service_factory): res = test_client.get(self.url(self.first_object)) assert res.status_code == 200 service_summaries = res.json['service_summaries'] - service_summaries.sort() # TODO migration: quit this and define sort assert service_summaries == [ + '(80/tcp) http', '(443/tcp) https', '(53/udp) dns', - '(80/tcp) http', + '(5353/udp) dns', ] From 7e81338607769ebfdc993daaecb1767b20b5d729 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 18 Jan 2018 17:00:07 -0300 Subject: [PATCH 0818/1506] Remove server headers from responses --- server/web.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/server/web.py b/server/web.py index 8f0698906e1..5e0cac598c6 100644 --- a/server/web.py +++ b/server/web.py @@ -37,11 +37,30 @@ logger = server.utils.logger.get_logger(__name__) -class FileWithoutDirectoryListing(File): +class CleanHttpHeadersResource(Resource, object): + def render(self, request): + request.responseHeaders.removeHeader('Server') + return super(CleanHttpHeadersResource, self).render(request) + + +class FileWithoutDirectoryListing(File, CleanHttpHeadersResource): def directoryListing(self): return ForbiddenResource() + +class FaradayWSGIResource(WSGIResource, object): + def render(self, request): + request.responseHeaders.removeHeader('Server') + return super(FaradayWSGIResource, self).render(request) + + +class FaradayRedirectResource(Redirect, object): + def render(self, request): + request.responseHeaders.removeHeader('Server') + return super(FaradayWSGIResource, self).render(request) + + class WebServer(object): HOME = '' UI_URL_PATH = '_ui' @@ -71,7 +90,7 @@ def __load_ssl_certs(self): return ssl.DefaultOpenSSLContextFactory(*certs) def __build_server_tree(self): - self.__root_resource = Resource() + self.__root_resource = CleanHttpHeadersResource() self.__root_resource.putChild(WebServer.HOME, self.__build_web_redirect()) self.__root_resource.putChild( WebServer.UI_URL_PATH, self.__build_web_resource()) @@ -79,13 +98,13 @@ def __build_server_tree(self): WebServer.API_URL_PATH, self.__build_api_resource()) def __build_web_redirect(self): - return Redirect(WebServer.UI_URL_PATH) + return FaradayRedirectResource(WebServer.UI_URL_PATH) def __build_web_resource(self): return FileWithoutDirectoryListing(WebServer.WEB_UI_LOCAL_PATH) def __build_api_resource(self): - return WSGIResource(reactor, reactor.getThreadPool(), app) + return FaradayWSGIResource(reactor, reactor.getThreadPool(), app) def __build_websockets_resource(self): logger.info(u"Websocket listening at wss://{0}:9000".format(self.__bind_address)) From a652741c81863f54c66bbf687f394eb55236a973 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 18 Jan 2018 17:12:45 -0300 Subject: [PATCH 0819/1506] Remove unused pickled dict --- utils/pickled_dict.py | 80 ------------------------------------------- 1 file changed, 80 deletions(-) delete mode 100644 utils/pickled_dict.py diff --git a/utils/pickled_dict.py b/utils/pickled_dict.py deleted file mode 100644 index 9f158f8d9ed..00000000000 --- a/utils/pickled_dict.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python - -''' -Faraday Penetration Test IDE -Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) -See the file 'doc/LICENSE' for the license information - -''' -''' -File: pickled_dict.py -Author: Daniel J. Foguelman -Description: A persist-to-disk picklebased dictionary with all the normal features. -''' - -import cPickle as pickle -import IPython -import os -import threading -import unittest - -class PickleBackedDict(dict): - def __init__(self, path, filename = None): - self.path = os.path.join(path, filename) if not filename is None else path - self.lock = threading.Lock() - if os.path.exists(self.path): - with open(self.path, 'rb') as f: - self.dict = pickle.load(f) - else: - self.dict = {} - - def cleanUp(self): - with self.lock: - with open(self.path, 'wb', 0) as writer: - self.dict = {} - pickle.dump(self.dict, writer) - - - def __setitem__(self, key, value): - with self.lock: - with open(self.path, 'wb', 0) as writer: - self.dict.__setitem__(key, value) - pickle.dump(self.dict, writer) - - def __getitem__(self, key): - return self.dict.__getitem__(key) - - def __repr__(self): - return self.dict.__repr__() - - def __str__(self): - return self.dict.__str__() - -class TestPickledDict(unittest.TestCase): - def setUp(self): - pass - - def tearDown(self): - """docstring for tearDown""" - pass - - def test_time_insert_and_retrieve(self): - from time import time - d_file = os.tmpfile() - d = PickleBackedDict(path = d_file.name) - - - it = time() * 1000 - for i in range(10): - d[i] = range(50) - et = time() * 1000 - - self.assertTrue( et - it < 2500, "Inserting a millon records takes more than a 2.5sec") - - it = time() * 1000 - a = d[3] - et = time() * 1000 - self.assertTrue( et - it < 500, "reading is a heavy task") - -if __name__ == '__main__': - unittest.main() From ecf43e180d88ae2924377954b47b2d59042d0eb8 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 18 Jan 2018 17:14:48 -0300 Subject: [PATCH 0820/1506] Remove unused update --- faraday.py | 20 ----- updates/__init__.py | 6 -- updates/updater.py | 175 -------------------------------------------- 3 files changed, 201 deletions(-) delete mode 100644 updates/__init__.py delete mode 100644 updates/updater.py diff --git a/faraday.py b/faraday.py index 84ed7e21143..b73d9bbaf47 100755 --- a/faraday.py +++ b/faraday.py @@ -460,25 +460,6 @@ def printBanner(): logger.info("Starting Faraday IDE.") -def update(): - """Updates Faraday IDE. - - Deletes every .pyc file and does a git pull to the official repository. - - """ - if args.update: - CONF = getInstanceConfiguration() - - if not is_authenticated(CONF.getServerURI(), CONF.getDBSessionCookies()): - logger.warning("Credentials needed to update.") - doLoginLoop() - - from updates.updater import Updater - Updater().doUpdates() - logger.info("Update process finished with no errors") - logger.info("Faraday will start now.") - - def checkUpdates(): import requests uri = getInstanceConfiguration().getUpdatesUri() @@ -644,7 +625,6 @@ def main(): check_faraday_version() - update() checkUpdates() startFaraday() diff --git a/updates/__init__.py b/updates/__init__.py deleted file mode 100644 index 004c49be6c6..00000000000 --- a/updates/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -''' -Faraday Penetration Test IDE -Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) -See the file 'doc/LICENSE' for the license information - -''' diff --git a/updates/updater.py b/updates/updater.py deleted file mode 100644 index 4fcda0e4950..00000000000 --- a/updates/updater.py +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding: utf-8 -*- -''' -Faraday Penetration Test IDE -Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) -See the file 'doc/LICENSE' for the license information - -''' -import subprocess -import couchdbkit -import model.workspace -from utils.logs import getLogger -from config.globals import * -logger = getLogger('Updater') - -from config.configuration import getInstanceConfiguration -CONF = getInstanceConfiguration() -from utils.user_input import query_yes_no -import sys -import os -import shutil -from managers.all import ViewsManager -from persistence.server.models import create_workspace - -class Updater(object): - def doUpdates(self): - logger.info('Removing old pyc files') - subprocess.call(['find', '.', '-name', '*.pyc', '-delete']) - logger.info('Pulling latest Github Master copy') - if query_yes_no('Proceed?', 'yes'): - subprocess.call(['git', 'pull']) - - try: - import pip - logger.info('Installing missing dependencies in pip') - pip.main(['install', '-r', CONST_REQUIREMENTS_FILE, '--user']) - except ImportError: - logger.error("Checking missing dependencies in pip") - pass - - - # logger.info('Upgrading DBs to latest version') - # DB().run() - - logger.info('Upgrading DBs to latest version') - CouchViews().run() - -class Update(object): - pass - -class CouchViews(Update): - def run(self): - source_server = CONF.getServerURI() - if not source_server: - logger.info("""No DB configuration found. - To upgrade your DB please configure a valid CouchDB URI in: - ~/.faraday/config/user.xml configuration file.""") - return - - serv = couchdbkit.Server(source_server) - - logger.info('We are about to upload CouchdbViews in Server [%s]' % source_server) - # if not query_yes_no("Faraday won't behave correctly with older versions, proceed?", 'no'): - # return - - dbs = filter(lambda x: not x.startswith("_") and 'backup' not in x and x not in CONST_BLACKDBS, serv.all_dbs()) - logger.info('Dbs to upgrade: %s' % (', '.join(dbs))) - - logger.info('Preparing updates on Couchdbs') - processed = 0 - views_uploader = ViewsManager() - for db_name in dbs: - db_source = couchdbkit.Database("/". - join((source_server, db_name))) - views_uploader.addViews(db_source, force = True) - - -class DB(Update): - def __init__(self): - pass - - def update_db(self, db_name): - if 'backup' in db_name: - logger.info('Database [%s] is a backup, ignoring' % db_name) - return - - source_server = CONF.getServerURI() - # Levanto los servidores - db_source = couchdbkit.Database("/".join((source_server, db_name))) - if db_source.doc_exist(db_name): - logger.info('DB: [%s] Already had suffer migration' % db_name) - return - - # Primero replico para no cagarla - logger.info('Creating db backup: %s' % ('%s-backup' % db_name)) - db_source.server.replicate(db_name, '%s-backup' % db_name, create_target = True) - db_bkp = couchdbkit.Database("/".join((source_server, - '%s-backup' % db_name))) - import time - while db_source.info()['doc_count'] < db_bkp.info()['doc_count']: - time.sleep(1) - - # Crear documento 'workspace' - logger.info('Creating workspace document') - - create_workspace(db_name, - 'Migrated Workspace', - int(time.time() * 1000), - int(time.time() * 1000), - "") - types = {} - - logger.info('Updating modelobject documents') - for document in db_source.all_docs(include_docs=True): - # Alter parent id: - doc = document['doc'] - if not('type' in doc): - continue - if doc['type'] == 'CommandRunInformation': - # Should set the workspace here! - continue - elif doc['type'] == 'Workspace': - # Already modified - continue - else: - # Modify the parent ID - parent = doc['parent'] - if parent == 'None' or parent == '': - parent = None - else: - l_parent = doc['_id'].split('.')[:-1] - parent = '.'.join(l_parent) - doc['parent'] = parent - if doc['owned'] == '' or doc['owned'] is None: - doc['owned'] == False - else: - doc['owned'] = eval(doc['owned']) - - document['doc'] = doc - db_source.save_doc(doc, force_update = True) - - types[doc['type']] = types.get(doc['type'], 0) + 1 - - logger.info("Transformed %s objects" % str(types)) - - - def run(self): - source_server = CONF.getServerURI() - if not source_server: - logger.info("""No DB configuration found. - To upgrade your DB please configure a valid CouchDB URI in: - ~/.faraday/config/user.xml configuration file.""") - return - - serv = couchdbkit.Server(source_server) - - logger.info('We are about to upgrade dbs in Server [%s]' % source_server) - dbs = filter(lambda x: not x.startswith("_") and 'backup' not in x and x not in CONST_BLACKDBS, serv.all_dbs()) - logger.info('Dbs to upgrade: %s' % (', '.join(dbs))) - - if not query_yes_no('Proceed?', 'no'): - return - - logger.info('Preparing updates on Couchdbs') - processed = 0 - logger.info('About to upgrade %d dbs' % len(dbs)) - for db_name in dbs: - logger.info('Updating db %s' % db_name) - try: - self.update_db(db_name) - processed = processed + 1 - except Exception as e: - logger.error(e) - logger.info('Updated DB [%s]. %d remaining' % (db_name, len(dbs) - processed)) - logger.info("Update process finish, be kind to review the process.\nBackuped databases won't be accesible") From 2bc7c719d1c2caee4ee3f8a5960dfdb36fc54a6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 18 Jan 2018 17:20:32 -0300 Subject: [PATCH 0821/1506] Fix bug updating a host's hostnames in the API A missing session.commit() broke the API --- server/api/modules/hosts.py | 3 +++ test_cases/models/test_host.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 8fb8a63d63e..7e06a564100 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -120,6 +120,9 @@ def _update_object(self, obj, data): else: obj.set_hostnames(hostnames) + # A commit is required here, otherwise it breaks (i'm not sure why) + db.session.commit() + return super(HostsView, self)._update_object(obj, data) def _filter_query(self, query): diff --git a/test_cases/models/test_host.py b/test_cases/models/test_host.py index dd3535abb8d..aba8e7d38c0 100644 --- a/test_cases/models/test_host.py +++ b/test_cases/models/test_host.py @@ -95,3 +95,22 @@ def test_all(self, host, session): session.commit() assert set(hn.name for hn in host.hostnames) == {'b', 'c'} + + def test_change_one(self, host, session): + hn = Hostname(workspace=host.workspace, + host=host, + name='x') + session.add(hn) + session.commit() + host.set_hostnames(['y']) + + assert len(session.deleted) == 1 + assert session.deleted.pop() is hn + + assert len(session.new) == 1 + new = session.new.pop() + assert new.name == 'y' + + session.commit() + assert len(host.hostnames) == 1 + assert host.hostnames[0].name == 'y' From 198959d07af1a00aaacd6ea55f6d64589d964c64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 18 Jan 2018 17:50:49 -0300 Subject: [PATCH 0822/1506] Disable OK button when nothing selected in bulk update in status report --- server/www/scripts/commons/partials/editOptions.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/commons/partials/editOptions.html b/server/www/scripts/commons/partials/editOptions.html index 57ae9ca258e..c78d4097ea0 100644 --- a/server/www/scripts/commons/partials/editOptions.html +++ b/server/www/scripts/commons/partials/editOptions.html @@ -19,5 +19,5 @@
    {{msg}}
    - +
    From d78614f8420b33bb68213f68b96eccc9b81fe225 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Thu, 18 Jan 2018 18:16:21 -0300 Subject: [PATCH 0823/1506] Added ID column to status report --- .../statusReport/controllers/statusReport.js | 15 ++++++++++++--- .../partials/ui-grid/columns/idcolumn.html | 5 +++++ 2 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 server/www/scripts/statusReport/partials/ui-grid/columns/idcolumn.html diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 353a0e9865c..0faae8c8145 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -151,6 +151,7 @@ angular.module('faradayApp') } $scope.columns = { + "_id": true, "date": true, "name": true, "severity": true, @@ -233,6 +234,14 @@ angular.module('faradayApp') '
    '+ '
    '; + $scope.gridOptions.columnDefs.push({ name : '_id', + displayName : "id", + cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/idcolumn.html', + headerCellTemplate: header, + width: '50', + sort: getColumnSort('_id'), + visible: $scope.columns["_id"] + }); $scope.gridOptions.columnDefs.push({ name : 'date', displayName : "date", cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/datecolumn.html', @@ -930,13 +939,13 @@ angular.module('faradayApp') // Add the total amount of vulnerabilities as an option for pagination // if it is larger than our biggest page size if ($scope.gridOptions.totalItems > paginationOptions.defaultPageSizes[paginationOptions.defaultPageSizes.length - 1]) { - + $scope.gridOptions.paginationPageSizes = paginationOptions.defaultPageSizes.concat([$scope.gridOptions.totalItems]); - + // sadly, this will load the vuln list again because it fires a paginationChanged event if ($scope.gridOptions.paginationPageSize > $scope.gridOptions.totalItems) $scope.gridOptions.paginationPageSize = $scope.gridOptions.totalItems; - + // New vuln and MAX items per page setted => reload page size. if ($scope.gridOptions.paginationPageSize === $scope.gridOptions.totalItems - 1) $scope.gridOptions.paginationPageSize = $scope.gridOptions.totalItems; diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/idcolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/idcolumn.html new file mode 100644 index 00000000000..5b975a4c1b3 --- /dev/null +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/idcolumn.html @@ -0,0 +1,5 @@ +
    +
    {{COL_FIELD CUSTOM_FILTERS}} +
    +
    + From f6809a255e5b3fbeb4cbb1a89033484edfd4e210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 18 Jan 2018 18:44:52 -0300 Subject: [PATCH 0824/1506] Fix error creating vuln templates The id field wasn't read only and the web was sending an empty string as its value --- server/api/modules/vulnerability_template.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/api/modules/vulnerability_template.py b/server/api/modules/vulnerability_template.py index 2eb289cba49..f5f7583ac66 100644 --- a/server/api/modules/vulnerability_template.py +++ b/server/api/modules/vulnerability_template.py @@ -29,6 +29,7 @@ class VulnerabilityTemplateSchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') + id = fields.Integer(dump_only=True, attribute='id') _rev = fields.String(default='', dump_only=True) cwe = fields.String(dump_only=True, default='') # deprecated field, the legacy data is added to refs on import exploitation = SeverityField(attribute='severity', required=True) From e19294448a2255305e96771307d0d92222ae2b2e Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Thu, 18 Jan 2018 19:43:34 -0300 Subject: [PATCH 0825/1506] Filtered _id in group by status report. --- server/www/scripts/statusReport/partials/statusReport.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/statusReport/partials/statusReport.html b/server/www/scripts/statusReport/partials/statusReport.html index 6aaf6e152a8..08725979377 100644 --- a/server/www/scripts/statusReport/partials/statusReport.html +++ b/server/www/scripts/statusReport/partials/statusReport.html @@ -87,7 +87,7 @@

    Status report for {{ workspace

    From abe7734c26254c6000ee4278ccb4538b1c7a33b2 Mon Sep 17 00:00:00 2001 From: Winna_Z Date: Fri, 19 Jan 2018 10:35:17 -0300 Subject: [PATCH 0826/1506] Fixed severity nams important than went to info by dafault and should have gone to high --- plugins/repo/netsparker/plugin.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/repo/netsparker/plugin.py b/plugins/repo/netsparker/plugin.py index 2915d6d3f0c..9c45d86d9aa 100644 --- a/plugins/repo/netsparker/plugin.py +++ b/plugins/repo/netsparker/plugin.py @@ -91,6 +91,11 @@ class Item(object): @param item_node A item_node taken from an netsparker xml tree """ + def re_map_severity(self, severity): + if severity == "Important": + return "high" + return severity + def __init__(self, item_node): self.node = item_node self.url = self.get_text_from_subnode("url") @@ -108,7 +113,7 @@ def __init__(self, item_node): self.port = host.group(11) self.name = self.get_text_from_subnode("type") - self.severity = self.get_text_from_subnode("severity") + self.severity = self.re_map_severity(self.get_text_from_subnode("severity")) self.certainty = self.get_text_from_subnode("certainty") self.method = self.get_text_from_subnode("vulnerableparametertype") self.param = self.get_text_from_subnode("vulnerableparameter") From 898c6da49bf2ea8e8a7d1d0b0d7ad09fb61ba40f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 19 Jan 2018 14:16:02 -0300 Subject: [PATCH 0827/1506] Add vuln templates CSV info in RELEASE.md Commit not signed due to problems with my yubikey :( --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index b560a3f58d8..1d607e6ac28 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -22,6 +22,7 @@ TBA: * Add optional hostnames, services, MAC and description fields to the host list * Workspace names can be changed from the Web UI * Changed the scope field of a workspace from a free text input to a list of targets +* Exploitation and severity fields only allow certain values. CWE CVEs were fixed to be valid. A script to convert custom CSVs was added. November 17, 2017: --- From da0e3ab0c9576c4c156857663824e58ee3ee83db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 19 Jan 2018 16:15:31 -0300 Subject: [PATCH 0828/1506] Allow searching hosts by OS and improve service summaries Show service version in addition to name, protocol and port number --- RELEASE.md | 2 +- server/api/modules/hosts.py | 2 ++ server/models.py | 7 ++++++- test_cases/test_api_hosts.py | 8 +++++++- 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 1d607e6ac28..482d2a602a9 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -17,7 +17,7 @@ TBA: * Communication object was removed and replaced with Comment * Show credentials count in summarized report on the dashboard * Remove vuln template CWE fields, join it with references -* Allow to search hosts by hostname +* Allow to search hosts by hostname, os and service name * Allow the user to specify the desired fields of the host list table * Add optional hostnames, services, MAC and description fields to the host list * Workspace names can be changed from the Web UI diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 7e06a564100..dd550b17821 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -133,9 +133,11 @@ def _filter_query(self, query): match_ip = Host.ip.ilike(like_term) match_service_name = Host.services.any( Service.name.ilike(like_term)) + match_os = Host.os.ilike(like_term) match_hostname = Host.hostnames.any(Hostname.name.ilike(like_term)) query = query.filter(match_ip | match_service_name | + match_os | match_hostname) return query diff --git a/server/models.py b/server/models.py index 5332e38bfca..aa2fcf85cee 100644 --- a/server/models.py +++ b/server/models.py @@ -358,7 +358,12 @@ def parent(self): @property def summary(self): - return "(%s/%s) %s" % (self.port, self.protocol, self.name) + if self.version and self.version.lower() != "unknown": + version = " (" + self.version + ")" + else: + version = "" + return "(%s/%s) %s%s" % (self.port, self.protocol, self.name, + version or "") class VulnerabilityABC(Metadata): diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index f44a719822f..f8004470d55 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -590,28 +590,34 @@ def test_create_host_cant_assign_command_from_another_workspace(self, test_clien def test_service_summaries(self, test_client, session, service_factory): service_factory.create(name='http', protocol='tcp', port=80, host=self.first_object, status='open', + version='nginx', workspace=self.workspace) service_factory.create(name='https', protocol='tcp', port=443, host=self.first_object, status='open', + version=None, workspace=self.workspace) service_factory.create(name='dns', protocol='udp', port=5353, host=self.first_object, status='open', + version=None, workspace=self.workspace) service_factory.create(name='smtp', protocol='tcp', port=25, host=self.first_object, status='filtered', + version=None, workspace=self.workspace) service_factory.create(name='dns', protocol='udp', port=53, host=self.first_object, status='open', + version=None, workspace=self.workspace) service_factory.create(name='other', protocol='udp', port=1234, host=self.first_object, status='closed', + version=None, workspace=self.workspace) session.commit() res = test_client.get(self.url(self.first_object)) assert res.status_code == 200 service_summaries = res.json['service_summaries'] assert service_summaries == [ - '(80/tcp) http', + '(80/tcp) http (nginx)', '(443/tcp) https', '(53/udp) dns', '(5353/udp) dns', From 16e871e8ec8cc3a9deba67697a55f3806ac32cdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 19 Jan 2018 16:29:11 -0300 Subject: [PATCH 0829/1506] Change column order in hosts list Swap services and hostnames, move description to the right (because it isn't so important) --- server/www/scripts/hosts/partials/list.html | 32 ++++++++++----------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/server/www/scripts/hosts/partials/list.html b/server/www/scripts/hosts/partials/list.html index a4c2b7c6b06..4a20e1a7a3b 100644 --- a/server/www/scripts/hosts/partials/list.html +++ b/server/www/scripts/hosts/partials/list.html @@ -70,22 +70,18 @@

    Add columns

    IP -
    		- Description - - OS - Hostnames - - Services + Hostnames + + MAC @@ -106,6 +102,10 @@

    Add columns

    Owned     		+ Description + + Creation time @@ -127,25 +127,22 @@

    Add columns

    {{host.name}} -     		- {{host.description}} - - - {{hostname}}
    -     -     		{{summary}}
    		+ + {{hostname}}
    +     +     		{{host.mac}} + {{host.description}} +
    + ID + + IP @@ -123,6 +127,9 @@

    Add columns

    		 + {{host.id}} + {{host.name}} From ac03ee66213296b1a54b2cdb79ffdc1bd6c96b00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 21 Feb 2018 16:49:42 -0300 Subject: [PATCH 0915/1506] Change "name" for "ip" in host list and improve remembering fields Now it will be easier to add fields in the future (before it would require the cookie to be deleted to see that the field is available) --- server/www/scripts/hosts/controllers/hosts.js | 34 +++++++++---------- server/www/scripts/hosts/partials/list.html | 12 +++---- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/server/www/scripts/hosts/controllers/hosts.js b/server/www/scripts/hosts/controllers/hosts.js index b54f6f0db2a..f016354bc4d 100644 --- a/server/www/scripts/hosts/controllers/hosts.js +++ b/server/www/scripts/hosts/controllers/hosts.js @@ -12,24 +12,24 @@ angular.module('faradayApp') // hosts list $scope.hosts = []; $scope.totalHosts = 0; + $scope.columns = { + "id": false, + "ip": true, + "description": false, + "hostnames": false, + "services": false, + "mac": false, + "service_count": true, + "vuln_count": true, + "credential_count": true, + "os": true, + "owned": true, + "create_time": true, + "last_modified": true, + } if($cookies.get('HColumns')) { - $scope.columns = JSON.parse($cookies.get('HColumns')) - }else{ - $scope.columns = { - "id": false, - "name": true, - "description": false, - "hostnames": false, - "services": false, - "mac": false, - "service_count": true, - "vuln_count": true, - "credential_count": true, - "os": true, - "owned": true, - "create_time": true, - "last_modified": true, - }; + preferences = JSON.parse($cookies.get('HColumns')) + angular.extend($scope.columns, preferences); } // current workspace $scope.workspace = $routeParams.wsId; diff --git a/server/www/scripts/hosts/partials/list.html b/server/www/scripts/hosts/partials/list.html index 63477b585fe..12bb5b3bac5 100644 --- a/server/www/scripts/hosts/partials/list.html +++ b/server/www/scripts/hosts/partials/list.html @@ -70,9 +70,9 @@

    Add columns

    ID -     		- IP - + + IP + OS @@ -130,9 +130,9 @@

    Add columns

    {{host.id}} -     		- {{host.name}} - + + {{host.ip}} + From 20d9c30c51c69e2591be9651a729bd292a3e9410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 21 Feb 2018 17:13:09 -0300 Subject: [PATCH 0916/1506] Make parent_type required in create_vuln fplugin --- bin/create_vuln.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bin/create_vuln.py b/bin/create_vuln.py index 9b04128947d..4f0b7a34e5b 100644 --- a/bin/create_vuln.py +++ b/bin/create_vuln.py @@ -14,18 +14,16 @@ def main(workspace='', args=None, parser=None): + parser.add_argument('parent_type', + choices=['Host', 'Service']) parser.add_argument('parent', help='Parent ID') parser.add_argument('name', help='Vulnerability Name') - parser.add_argument('--reference', help='Vulnerability reference', default='') # Fixme + parser.add_argument('--reference', help='Vulnerability reference', default='') # Fixme parser.add_argument('--severity', help='Vulnerability severity', choices=['critical', 'high', 'med', 'low', 'info', 'unclassified'], default='unclassified') - parser.add_argument('--parent_type', - help='Vulnerability severity', - choices=['Host', 'Service'], - default='unclassified') parser.add_argument('--resolution', help='Resolution', default='') parser.add_argument('--confirmed', help='Is the vulnerability confirmed', From 1ba8a19cdfa1e8dbf22625f213c5f94e18e8b2a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 21 Feb 2018 17:20:17 -0300 Subject: [PATCH 0917/1506] Improve CantCommunicateWithServerError error messages Add a proper check so it shows the JSON with the errors if the response status code is 400 --- persistence/server/server_io_exceptions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/persistence/server/server_io_exceptions.py b/persistence/server/server_io_exceptions.py index 6c432294cb0..95c116d3f32 100644 --- a/persistence/server/server_io_exceptions.py +++ b/persistence/server/server_io_exceptions.py @@ -28,7 +28,10 @@ def __init__(self, function, server_url, payload, response=None): self.response = response def __str__(self): - response_text = self.response and self.response.text or '' + if self.response is None: + response_text = '' + else: + response_text = self.response.text return ("Couldn't get a valid response from the server when requesting " "to URL {0} and function {1}. Response was {2}".format(self.server_url, self.function, response_text)) From c9a91f8dbe80af3add63341ca56256f0ed17e43f Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 21 Feb 2018 18:40:38 -0300 Subject: [PATCH 0918/1506] Now GTK load all vulns amount of hosts and services correctly, and show the objects OK --- gui/gtk/dialogs.py | 6 +++--- gui/gtk/mainwidgets.py | 4 ++-- persistence/server/models.py | 19 +++++++++++-------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/gui/gtk/dialogs.py b/gui/gtk/dialogs.py index ae938bf7bd3..ef592b191a5 100644 --- a/gui/gtk/dialogs.py +++ b/gui/gtk/dialogs.py @@ -879,11 +879,11 @@ def create_model(self, host): # only the ID and the name are needed, but i still need to 'fill' # the other columns with dummy info - display_str = host.getName() + " (" + str(host.getVulnAmount()) + ")" + display_str = host.getName() + " (" + str(host.getVulnsAmount()) + ")" owned_status = ("Yes" if host.isOwned() else "No") host_position = model.append(None, [str(host.getID()), host.getName(), host.getOS(), owned_status, - str(host.getVulnAmount()), "", + str(host.getVulnsAmount()), "", "", "", "", "", "", "", display_str]) @@ -898,7 +898,7 @@ def lst_to_str(lst): def add_service_to_host_in_model(service, model): """Append a service to an host in the given model. Return None. Modifies the model""" - display_str = service.getName() + " (" + str(len(service.getVulns())) + ")" + display_str = service.getName() + " (" + str(service.getVulnsAmount()) + ")" model.append(host_position, [str(service.getID()), service.getName(), service.getDescription(), diff --git a/gui/gtk/mainwidgets.py b/gui/gtk/mainwidgets.py index 8a2ad4487f5..8aeadeae2f2 100644 --- a/gui/gtk/mainwidgets.py +++ b/gui/gtk/mainwidgets.py @@ -252,7 +252,7 @@ def _is_vuln_of_host(self, vuln_id, host_id): def _add_single_host_to_model(self, host): """Add a single host to the model. Return None.""" - vuln_count = host.getVulnAmount() + vuln_count = host.getVulnsAmount() os_icon, os_str = self.__decide_icon(host.getOS()) display_str = str(host) if str(host.id) not in map(lambda host_data: host_data[0], self.model): @@ -410,7 +410,7 @@ def remove_object(self, obj_id, obj_type): # name is updated with the ammount of vulns host = self.get_single_host_function(obj_id) if host: - self._modify_vuln_amount_of_single_host_in_model(host.getID(), host.getVulnAmount()) + self._modify_vuln_amount_of_single_host_in_model(host.getID(), host.getVulnsAmount()) def update_object(self, obj): """Update the obj in the model, if found there""" diff --git a/persistence/server/models.py b/persistence/server/models.py index f3993892b60..d593fc95425 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -871,26 +871,29 @@ def __str__(self): def getOS(self): return self.os - def getVulnAmount(self): + def getVulnsAmount(self): return self.vuln_amount def getDefaultGateway(self): return self.default_gateway def getVulns(self): - return get_all_vulns(self._workspace_name, target=self._server_id) + """ + Get all vulns of this host. + """ + return get_all_vulns(self._workspace_name, target=self.ip) def getServices(self): """ - Get all services of that host + Get all services of this host. """ return get_services(self._workspace_name, hostid=self._server_id) - def getService(self): + def getService(self, service_id): """ - Get a specific service id + Get a specific service id of this host. """ - return get_service(self._workspace_name, hostid=self._server_id) + return get_service(self._workspace_name, hostid=self._server_id, service_id=service_id) class Service(ModelBase): """A simple Service class. Should implement all the methods of the @@ -962,8 +965,8 @@ def getProtocol(self): def isOwned(self): return self.owned - def getVulns(self): - return get_all_vulns(self._workspace_name, serviceid=self._server_id) + def getVulnsAmount(self): + return self.vuln_amount class Vuln(ModelBase): From fb538835b013990eefcec165682b57f5dcf3ca7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 21 Feb 2018 19:19:16 -0300 Subject: [PATCH 0919/1506] Fix go to vulns from service list --- server/www/scripts/services/partials/list.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/services/partials/list.html b/server/www/scripts/services/partials/list.html index cee8d22861f..4b8954589f8 100644 --- a/server/www/scripts/services/partials/list.html +++ b/server/www/scripts/services/partials/list.html @@ -104,7 +104,7 @@

    No services found for {{host.ip}}

    		 - +
    - - - - - + + + + + From 1648bb5fe8106c82f09ff631dbaa5b2a762c8489 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 9 Mar 2018 12:00:46 -0300 Subject: [PATCH 0983/1506] Make some vuln fields non blankable nor nullable Also make field description of vuln schema read only (it was collapsing with desc) --- server/api/base.py | 20 +++++++++++++++- server/api/modules/vulns.py | 5 ++-- server/models.py | 8 +++++-- test_cases/test_api_vulnerability.py | 36 ++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 5 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 215a2cdb9d5..1d7dcefd6d8 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -9,6 +9,7 @@ from sqlalchemy import func from marshmallow import Schema from marshmallow.compat import with_metaclass +from marshmallow.validate import Length from marshmallow_sqlalchemy import ModelConverter from marshmallow_sqlalchemy.schema import ModelSchemaMeta, ModelSchemaOpts from webargs.flaskparser import FlaskParser, parser, abort @@ -643,6 +644,23 @@ class ReadWriteWorkspacedView(CreateWorkspacedMixin, pass +class CustomModelConverter(ModelConverter): + """ + Model converter that automatically sets minimum length + validators to not blankable fields + """ + def _add_column_kwargs(self, kwargs, column): + super(CustomModelConverter, self)._add_column_kwargs(kwargs, column) + if not column.info.get('allow_blank', True): + kwargs['validate'].append(Length(min=1)) + + +class CustomModelSchemaOpts(ModelSchemaOpts): + def __init__(self, *args, **kwargs): + super(CustomModelSchemaOpts, self).__init__(*args, **kwargs) + self.model_converter = CustomModelConverter + + class AutoSchema(with_metaclass(ModelSchemaMeta, Schema)): """ A Marshmallow schema that does field introspection based on @@ -650,7 +668,7 @@ class AutoSchema(with_metaclass(ModelSchemaMeta, Schema)): Unlike the marshmallow_sqlalchemy ModelSchema, it doesn't change the serialization and deserialization proccess. """ - OPTIONS_CLASS = ModelSchemaOpts + OPTIONS_CLASS = CustomModelSchemaOpts class FilterAlchemyModelConverter(ModelConverter): diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 4dcc89dab7e..005d28b59f8 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -12,7 +12,7 @@ from flask import Blueprint from flask_classful import route from marshmallow import Schema, fields, post_load, ValidationError -from marshmallow.validate import OneOf +from marshmallow.validate import OneOf, Length from sqlalchemy.orm import aliased, joinedload, selectin_polymorphic, undefer from sqlalchemy.orm.exc import NoResultFound @@ -96,7 +96,8 @@ class VulnerabilitySchema(AutoSchema): owned = fields.Boolean(dump_only=True, default=False) owner = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator') impact = SelfNestedField(ImpactSchema()) - desc = fields.String(attribute='description') + desc = fields.String(attribute='description', validate=Length(min=1)) + description = fields.String(dump_only=True) policyviolations = fields.List(fields.String, attribute='policy_violations') refs = fields.List(fields.String(), attribute='references') diff --git a/server/models.py b/server/models.py index cef757cdb00..f641b68f3ad 100644 --- a/server/models.py +++ b/server/models.py @@ -3,6 +3,7 @@ # See the file 'doc/LICENSE' for the license information import operator from datetime import datetime +from functools import partial from sqlalchemy import ( Boolean, @@ -44,6 +45,9 @@ ) from server.utils.database import BooleanToIntColumn +NonBlankColumn = partial(Column, nullable=False, + info={'allow_blank': False}) + OBJECT_TYPES = [ 'vulnerability', 'host', @@ -387,9 +391,9 @@ class VulnerabilityABC(Metadata): id = Column(Integer, primary_key=True) data = Column(Text, nullable=True) - description = Column(Text, nullable=False) + description = NonBlankColumn(Text) ease_of_resolution = Column(Enum(*EASE_OF_RESOLUTIONS, name='vulnerability_ease_of_resolution'), nullable=True) - name = Column(Text, nullable=False) + name = NonBlankColumn(Text, nullable=False) resolution = Column(Text, nullable=True) severity = Column(Enum(*SEVERITIES, name='vulnerability_severity'), nullable=False) risk = Column(Float(3, 1), nullable=True) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index ae31f9eaed5..761b1868085 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -252,6 +252,42 @@ def test_create_vuln(self, host_with_hostnames, test_client, session): assert res.json['description'] == 'helloworld' assert res.json['severity'] == 'low' + def test_create_cannot_create_vuln_with_empty_name( + self, host, session, test_client): + # I'm using this to test the NonBlankColumn which works for + # all models. Think twice before removing this test + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='', + vuln_type='Vulnerability', + parent_id=host.id, + parent_type='Host', + refs=[], + policyviolations=[], + description='aaa', + severity='low', + ) + res = test_client.post(self.url(), data=raw_data) + assert res.status_code == 400 + assert 'Shorter than minimum length 1' in res.data + + def test_create_cannot_create_vuln_with_empty_description( + self, host, session, test_client): + session.commit() # flush host_with_hostnames + raw_data = self._create_post_data_vulnerability( + name='New vulns', + vuln_type='Vulnerability', + parent_id=host.id, + parent_type='Host', + refs=[], + policyviolations=[], + description='', + severity='low', + ) + res = test_client.post(self.url(), data=raw_data) + assert res.status_code == 400 + assert 'Shorter than minimum length 1' in res.data + def test_create_vuln_with_attachments(self, host_with_hostnames, test_client, session): session.commit() # flush host_with_hostnames attachment = NamedTemporaryFile() From d3e8daacad1b957572c4ad95dba895fc687a84c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 9 Mar 2018 12:13:45 -0300 Subject: [PATCH 0984/1506] Fix tests broken on merge with last commit --- test_cases/test_api_vulnerability.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 90d3e12e0be..e0854eca856 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -265,7 +265,7 @@ def test_create_cannot_create_vuln_with_empty_name( # I'm using this to test the NonBlankColumn which works for # all models. Think twice before removing this test session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='', vuln_type='Vulnerability', parent_id=host.id, @@ -282,7 +282,7 @@ def test_create_cannot_create_vuln_with_empty_name( def test_create_cannot_create_vuln_with_empty_description( self, host, session, test_client): session.commit() # flush host_with_hostnames - raw_data = self._create_post_data_vulnerability( + raw_data = _create_post_data_vulnerability( name='New vulns', vuln_type='Vulnerability', parent_id=host.id, @@ -1505,4 +1505,4 @@ def send_api_request(raw_data): data=raw_data) assert res.status_code in [201, 400, 409] - send_api_request() \ No newline at end of file + send_api_request() From d6d0cb91681f896f4be8a92ed43a24254376299e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 9 Mar 2018 12:27:34 -0300 Subject: [PATCH 0985/1506] Don't run hypothesis tests by default --- Jenkinsfile | 6 +++--- test_cases/conftest.py | 8 ++++++++ test_cases/test_api_hosts.py | 4 +++- test_cases/test_api_vulnerability.py | 2 ++ 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 14aabd7352a..32e9a90e1ff 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ node (label: "master"){ try { sh """ source ${ENV_PATH}/bin/activate - cd $WORKSPACE && pytest -v --junitxml=$WORKSPACE/xunit.xml || : + cd $WORKSPACE && pytest -v --with-hypothesis --junitxml=$WORKSPACE/xunit.xml || : deactivate """ step([$class: 'CoberturaPublisher', autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: '**/coverage.xml', failNoReports: false, failUnhealthy: false, failUnstable: false, maxNumberOfBuilds: 0, onlyStable: false, sourceEncoding: 'ASCII', zoomCoverageChart: false]) @@ -70,7 +70,7 @@ node (label: "master"){ withCredentials([string(credentialsId: 'postgresql_connection_string', variable: 'CONN_STRING')]) { sh """ source ${ENV_PATH}/bin/activate - cd $WORKSPACE && pytest -v --junitxml=$WORKSPACE/xunit-postgres.xml --connection-string "$CONN_STRING" || : + cd $WORKSPACE && pytest -v --with-hypothesis --junitxml=$WORKSPACE/xunit-postgres.xml --connection-string "$CONN_STRING" || : deactivate """ step([$class: 'CoberturaPublisher', autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: '**/coverage.xml', failNoReports: false, failUnhealthy: false, failUnstable: false, maxNumberOfBuilds: 0, onlyStable: false, sourceEncoding: 'ASCII', zoomCoverageChart: false]) @@ -116,4 +116,4 @@ def notifyBuild(String buildStatus = 'STARTED', String extraMessage = '') { // Send notifications slackSend (color: colorCode, message: summary) -} \ No newline at end of file +} diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 2638340ddb1..0fe332c4b2a 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -65,6 +65,14 @@ def pytest_addoption(parser): "sqlite if not specified:") parser.addoption('--ignore-nplusone', action='store_true', help="Globally ignore nplusone errors") + parser.addoption("--with-hypothesis", action="store_true", + dest="use_hypothesis", default=False, + help="Run property based tests") + + +def pytest_configure(config): + if not config.option.use_hypothesis: + config.option.markexpr = 'not hypothesis' @pytest.fixture(scope='session') diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 0908daf3567..68abed8ed3f 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -662,7 +662,9 @@ def host_json(): @pytest.mark.usefixtures('logged_user') +@pytest.mark.hypothesis def test_hypothesis(host_with_hostnames, test_client, session): + assert 0, "hypothesis test run" session.commit() HostData = host_json() @@ -674,4 +676,4 @@ def send_api_request(raw_data): data=raw_data) assert res.status_code in [201, 400, 409] - send_api_request() \ No newline at end of file + send_api_request() diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index e0854eca856..0427a191f1d 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1493,7 +1493,9 @@ def vulnerability_json(parent_id, parent_type): @pytest.mark.usefixtures('logged_user') +@pytest.mark.hypothesis def test_hypothesis(host_with_hostnames, test_client, session): + assert 0, "hypothesis test run" session.commit() VulnerabilityData = vulnerability_json(host_with_hostnames.id, 'Host') From 5b1c3b32c1fd48e5506de3f25ccc2767a9aa05f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 9 Mar 2018 12:28:47 -0300 Subject: [PATCH 0986/1506] Fix hypothesis tests They were intended to fail to test jenkins --- test_cases/test_api_hosts.py | 1 - test_cases/test_api_vulnerability.py | 1 - 2 files changed, 2 deletions(-) diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 68abed8ed3f..0e251bb8fbd 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -664,7 +664,6 @@ def host_json(): @pytest.mark.usefixtures('logged_user') @pytest.mark.hypothesis def test_hypothesis(host_with_hostnames, test_client, session): - assert 0, "hypothesis test run" session.commit() HostData = host_json() diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 0427a191f1d..9a6a3ec99cc 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1495,7 +1495,6 @@ def vulnerability_json(parent_id, parent_type): @pytest.mark.usefixtures('logged_user') @pytest.mark.hypothesis def test_hypothesis(host_with_hostnames, test_client, session): - assert 0, "hypothesis test run" session.commit() VulnerabilityData = vulnerability_json(host_with_hostnames.id, 'Host') From 4e2a77701ef8adf854b340dde37f4be96ef1f814 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 9 Mar 2018 14:25:36 -0300 Subject: [PATCH 0987/1506] Add objects to session after adding cascade. * After adding cascade tests were broken. It was needed to add those objs to session. We tried save-update on cascade but it did't have any effects --- server/models.py | 2 +- test_cases/conftest.py | 1 - test_cases/models/test_vulnerability.py | 5 ++++ test_cases/test_api_vulnerability.py | 29 ++++++++++++++-------- test_cases/test_api_workspace.py | 15 ++++++----- test_cases/test_api_workspaced_base.py | 4 ++- test_cases/test_managers_mapper_manager.py | 7 +++++- 7 files changed, 43 insertions(+), 20 deletions(-) diff --git a/server/models.py b/server/models.py index f9b2f683235..413804a130a 100644 --- a/server/models.py +++ b/server/models.py @@ -816,7 +816,7 @@ def service_id(cls): @declared_attr def service(cls): - return relationship('Service', backref=backref("vulnerabilities_web", cascade="all, delete-orphan"), cascade='all') + return relationship('Service', backref=backref("vulnerabilities_web", cascade="all, delete-orphan")) @property def parent(self): diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 2638340ddb1..d727bdf469a 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -5,7 +5,6 @@ import pytest from factory import Factory from flask.testing import FlaskClient -from nplusone.core import signals from sqlalchemy import event from pytest_factoryboy import register diff --git a/test_cases/models/test_vulnerability.py b/test_cases/models/test_vulnerability.py index 11e62a9133a..c0d2b99cbb2 100644 --- a/test_cases/models/test_vulnerability.py +++ b/test_cases/models/test_vulnerability.py @@ -81,6 +81,7 @@ def test_empty_references(self): def test_add_references(self, session): self.childs().add('CVE-2017-1234') + session.add(self.vuln) assert session.new session.commit() assert self.model.query.count() == 1 @@ -109,6 +110,7 @@ def test_add_existing_from_other_workspace(self, session, child, self.childs().add(child.name) session.commit() self.childs(self.vuln_different_ws).add(child.name) + session.add(self.vuln_different_ws) session.commit() assert self.model.query.count() == 2 assert len(self.childs(self.vuln_different_ws)) == 1 @@ -150,6 +152,7 @@ def test_removes_orphan(self): def test_direct_assignation(self, session, previous_childs, new_childs): for ref in previous_childs: self.childs().add(ref) + session.add(self.vuln) session.commit() setattr(self.vuln, self.field_name, new_childs) session.commit() @@ -193,6 +196,7 @@ def test_no_creator_command(self, vulnerability, session): def test_creator_command(self, workspace, vulnerability, empty_command_factory, session): command = empty_command_factory.create(workspace=workspace) + session.add(vulnerability) session.flush() assert vulnerability.id is not None assert command.id is not None @@ -217,6 +221,7 @@ def test_creator_command(self, workspace, vulnerability, def test_different_object_type(self, vulnerability, workspace, empty_command_factory, session): command = empty_command_factory.create(workspace=workspace) + session.add(vulnerability) session.flush() invalid_co = CommandObject.create(vulnerability, command) invalid_co.object_type = 'host' diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 31f851e6872..70346eb0fd6 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -173,6 +173,7 @@ def test_handles_vuln_with_no_creator(self, session): # This can happen when a user is deleted but its objects persist vuln = self.factory.create(workspace=workspace, creator=None) + session.add(vuln) session.commit() res = test_client.get(self.url(vuln)) assert res.status_code == 200 @@ -184,6 +185,7 @@ def test_shows_policy_violations(self, workspace, test_client, session, 5, workspace=workspace) for pv in pvs: self.first_object.policy_violation_instances.add(pv) + session.add(self.first_object) session.commit() res = test_client.get(self.url(self.first_object)) assert res.status_code == 200 @@ -555,16 +557,17 @@ def test_filter_by_severity(self, test_client, session, ): expected_ids = set() - vulnerability_factory.create_batch( + vulns = vulnerability_factory.create_batch( 5, workspace=second_workspace, severity='high') - vulnerability_web_factory.create_batch( + vulns += vulnerability_web_factory.create_batch( 5, workspace=second_workspace, severity='high') medium_vulns = vulnerability_factory.create_batch( 5, workspace=second_workspace, severity='medium') medium_vulns_web = vulnerability_web_factory.create_batch( 5, workspace=second_workspace, severity='medium') - + session.add_all(vulns) + session.add_all(medium_vulns + medium_vulns_web) session.commit() expected_ids.update(vuln.id for vuln in medium_vulns) expected_ids.update(vuln.id for vuln in medium_vulns_web) @@ -675,13 +678,14 @@ def test_filter_by_target(self, test_client, session, host_factory, @pytest.mark.usefixtures('mock_envelope_list') def test_sort_by_method(self, session, test_client, second_workspace, vulnerability_factory, vulnerability_web_factory): - vulnerability_factory.create_batch( + vulns = vulnerability_factory.create_batch( 10, workspace=second_workspace ) - vulnerability_web_factory.create_batch( + vulns += vulnerability_web_factory.create_batch( 10, workspace=second_workspace, method=None ) - + session.add_all(vulns) + session.commit() for method in ('afjbeidcgh'): vulnerability_web_factory.create(workspace=second_workspace, method=method) @@ -918,12 +922,13 @@ def test_count_confirmed(self, test_client, session): ]) def test_count_severity_map(self, test_client, second_workspace, session): - self.factory.create_batch(4, severity='informational', + vulns = self.factory.create_batch(4, severity='informational', workspace=second_workspace) - self.factory.create_batch(3, severity='medium', + vulns += self.factory.create_batch(3, severity='medium', workspace=second_workspace) - self.factory.create_batch(2, severity='low', + vulns += self.factory.create_batch(2, severity='low', workspace=second_workspace) + session.add_all(vulns) session.commit() res = test_client.get(self.url(workspace=second_workspace) + @@ -1120,6 +1125,7 @@ def test_update_with_parent_of_other_workspace( self, parent_type, parent_factory, test_client, session, second_workspace, credential_factory): parent = parent_factory.create(workspace=second_workspace) + session.add(parent) session.commit() assert parent.workspace_id != self.workspace.id data = self._create_put_data( @@ -1328,6 +1334,8 @@ def test_type_filter(workspace, session, filter_ = VulnerabilityFilterSet().filters['type'] std_vulns = vulnerability_factory.create_batch(10, workspace=workspace) web_vulns = vulnerability_web_factory.create_batch(10, workspace=workspace) + session.add_all(std_vulns) + session.add_all(web_vulns) session.commit() std_filter = filter_.filter(VulnerabilityGeneric.query, @@ -1355,9 +1363,10 @@ def test_creator_filter(workspace, session, workspace=workspace)[:5] command = empty_command_factory.create(workspace=workspace, tool="metasploit") - session.commit() vulns = std_vulns + web_vulns + session.add_all(vulns) + session.commit() session.flush() for vuln in vulns: command_object_factory.create(command=command, diff --git a/test_cases/test_api_workspace.py b/test_cases/test_api_workspace.py index 9998f6d6db4..6fdcd6f12af 100644 --- a/test_cases/test_api_workspace.py +++ b/test_cases/test_api_workspace.py @@ -31,10 +31,11 @@ def test_vuln_count(self, test_client, session, querystring): - vulnerability_factory.create_batch(8, workspace=self.first_object, + vulns = vulnerability_factory.create_batch(8, workspace=self.first_object, confirmed=False) - vulnerability_factory.create_batch(5, workspace=self.first_object, + vulns += vulnerability_factory.create_batch(5, workspace=self.first_object, confirmed=True) + session.add_all(vulns) session.commit() res = test_client.get(self.url(self.first_object) + querystring) assert res.status_code == 200 @@ -50,10 +51,11 @@ def test_vuln_count_confirmed(self, test_client, session, querystring): - vulnerability_factory.create_batch(8, workspace=self.first_object, + vulns = vulnerability_factory.create_batch(8, workspace=self.first_object, confirmed=False) - vulnerability_factory.create_batch(5, workspace=self.first_object, + vulns += vulnerability_factory.create_batch(5, workspace=self.first_object, confirmed=True) + session.add_all(vulns) session.commit() res = test_client.get(self.url(self.first_object) + querystring) assert res.status_code == 200 @@ -116,8 +118,9 @@ def test_create_stat_is_zero(self, test_client, stat_name): def test_update_stats(self, workspace, session, test_client, vulnerability_factory, vulnerability_web_factory): - vulnerability_factory.create_batch(10, workspace=workspace) - vulnerability_web_factory.create_batch(5, workspace=workspace) + vulns = vulnerability_factory.create_batch(10, workspace=workspace) + vulns += vulnerability_web_factory.create_batch(5, workspace=workspace) + session.add_all(vulns) session.commit() raw_data = {'name': 'something', 'description': ''} res = test_client.put('/v2/ws/{}/'.format(workspace.name), diff --git a/test_cases/test_api_workspaced_base.py b/test_cases/test_api_workspaced_base.py index b0b301ee426..b483cc154ae 100644 --- a/test_cases/test_api_workspaced_base.py +++ b/test_cases/test_api_workspaced_base.py @@ -27,6 +27,7 @@ def load_workspace_with_objects(self, database, session, workspace): self.objects = self.factory.create_batch( OBJECT_COUNT, workspace=workspace) self.first_object = self.objects[0] + session.add_all(self.objects) session.commit() assert workspace.id is not None self.workspace = workspace @@ -65,7 +66,8 @@ def _envelope_list(self, objects, pagination_metadata=None): def test_list_retrieves_all_items_from_workspace(self, test_client, second_workspace, session): - self.factory.create(workspace=second_workspace) + obj = self.factory.create(workspace=second_workspace) + session.add(obj) session.commit() res = test_client.get(self.url()) assert res.status_code == 200 diff --git a/test_cases/test_managers_mapper_manager.py b/test_cases/test_managers_mapper_manager.py index 2510eb31913..c018ac3e62e 100644 --- a/test_cases/test_managers_mapper_manager.py +++ b/test_cases/test_managers_mapper_manager.py @@ -762,10 +762,12 @@ def test_update_without_command(self, obj_class, many_test_data, monkeypatch, se for test_data in many_test_data: relational_model = test_data['factory'].create() + session.add(relational_model) session.commit() raw_data = test_data['data'] if test_data['parent']: parent = test_data['parent']['parent_factory'].create() + session.add(parent) session.commit() test_data['data']['parent'] = parent.id @@ -802,9 +804,9 @@ def mock_server_put(test_data, put_url, update=False, expected_response=201, **p def test_update_with_command(self, obj_class, many_test_data, monkeypatch, session): if obj_class in [Command]: return - session.commit() workspace = WorkspaceFactory.create(name='test') command = CommandFactory.create(workspace=workspace) + session.add(command) session.commit() mapper_manager = MapperManager() mapper_manager.createMappers(workspace.name) @@ -813,6 +815,7 @@ def test_update_with_command(self, obj_class, many_test_data, monkeypatch, sessi raw_data = test_data['data'] if test_data['parent']: parent = test_data['parent']['parent_factory'].create() + session.add(parent) session.commit() test_data['data']['parent'] = parent.id test_data['data']['parent_type'] = test_data['parent']['parent_type'] @@ -820,6 +823,7 @@ def test_update_with_command(self, obj_class, many_test_data, monkeypatch, sessi if obj_class in [Vuln, Credential]: test_data['expected_payload']['parent_type'] = test_data['parent']['parent_type'] relational_model = test_data['factory'].create() + session.add(relational_model) session.commit() def mock_server_put(put_url, update=False, expected_response=201, **params): assert put_url == '{0}/ws/test/{1}/{2}/?command_id={3}'.format( @@ -845,6 +849,7 @@ def mock_server_put(put_url, update=False, expected_response=201, **params): def test_find_obj_by_id(self, obj_class, many_test_data, session, monkeypatch): for test_data in many_test_data: persisted_obj = test_data['factory'].create() + session.add(persisted_obj) session.commit() mapper_manager = MapperManager() mapper_manager.createMappers(persisted_obj.workspace.name) From 3504a64cf6b9a53d6662f5eee653defdb358da5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 9 Mar 2018 14:54:10 -0300 Subject: [PATCH 0988/1506] Add logic for blankable text columns --- server/api/base.py | 5 +++++ server/models.py | 5 ++++- server/schemas.py | 19 +++++++++++++++++++ test_cases/test_api_vulnerability.py | 12 ++++++++++++ 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/server/api/base.py b/server/api/base.py index 1d7dcefd6d8..404d555142c 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -15,6 +15,7 @@ from webargs.flaskparser import FlaskParser, parser, abort from webargs.core import ValidationError from server.models import Workspace, db, Command, CommandObject +from server.schemas import NullToBlankString import server.utils.logger logger = server.utils.logger.get_logger(__name__) @@ -670,6 +671,10 @@ class AutoSchema(with_metaclass(ModelSchemaMeta, Schema)): """ OPTIONS_CLASS = CustomModelSchemaOpts + # Use NullToBlankString instead of fields.String by default on text fields + TYPE_MAPPING = Schema.TYPE_MAPPING.copy() + TYPE_MAPPING[str] = NullToBlankString + class FilterAlchemyModelConverter(ModelConverter): """Use this to make all fields of a model not required. diff --git a/server/models.py b/server/models.py index f641b68f3ad..4eff74e72f7 100644 --- a/server/models.py +++ b/server/models.py @@ -47,6 +47,9 @@ NonBlankColumn = partial(Column, nullable=False, info={'allow_blank': False}) +BlankColumn = partial(Column, nullable=False, + info={'allow_blank': True}, + default='') OBJECT_TYPES = [ 'vulnerability', @@ -390,7 +393,7 @@ class VulnerabilityABC(Metadata): __abstract__ = True id = Column(Integer, primary_key=True) - data = Column(Text, nullable=True) + data = BlankColumn(Text) description = NonBlankColumn(Text) ease_of_resolution = Column(Enum(*EASE_OF_RESOLUTIONS, name='vulnerability_ease_of_resolution'), nullable=True) name = NonBlankColumn(Text, nullable=False) diff --git a/server/schemas.py b/server/schemas.py index 21ba4ea0550..6578558232f 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -133,6 +133,25 @@ def _deserialize(self, value, attr, data): return ret +class NullToBlankString(fields.String): + """ + Custom field that converts null into an empty value. Created for + compatibility with the web ui. + """ + + def __init__(self, *args, **kwargs): + super(NullToBlankString, self).__init__(*args, **kwargs) + # Always make the field nullable because it is translated + self.allow_none = True + + def _deserialize(self, value, attr, data): + print value, attr, data + if value is None and not self.nullable: + value = '' + return super(NullToBlankString, self)._deserialize( + value, attr, data) + + class MetadataSchema(Schema): command_id = fields.Function(lambda x: None, dump_only=True) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 9a6a3ec99cc..0884e8bd9b6 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -22,8 +22,10 @@ from server.api.modules.vulns import ( TypeFilter, VulnerabilityFilterSet, + VulnerabilitySchema, VulnerabilityView ) +from server.schemas import NullToBlankString from test_cases import factories from test_api_workspaced_base import ( ReadOnlyAPITests @@ -1507,3 +1509,13 @@ def send_api_request(raw_data): assert res.status_code in [201, 400, 409] send_api_request() + + +def test_model_converter(): + """Test that string fields are translated to NullToBlankString + fields""" + # Generic test. Think twice if you want to delete this + field = VulnerabilitySchema().fields['data'] + print field + assert isinstance(field, NullToBlankString) + assert field.allow_none From dfbd86560a024679c38692bf11f266b0f0e404eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 9 Mar 2018 15:50:44 -0300 Subject: [PATCH 0989/1506] Make text fields not nullable, first pass --- server/api/modules/hosts.py | 5 ++-- server/models.py | 46 ++++++++++++++++++------------------ server/schemas.py | 18 ++++++++------ test_cases/test_api_hosts.py | 8 +++---- 4 files changed, 41 insertions(+), 36 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index e13c1221242..f157bc0f433 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -20,6 +20,7 @@ from server.schemas import ( MetadataSchema, MutableField, + NullToBlankString, PrimaryKeyRelatedField, SelfNestedField ) @@ -35,8 +36,8 @@ class HostSchema(AutoSchema): _rev = fields.String(default='') ip = fields.String(default='') description = fields.String(required=True) # Explicitly set required=True - default_gateway = fields.String(attribute="default_gateway_ip", - required=False, allow_none=True) + default_gateway = NullToBlankString( + attribute="default_gateway_ip", required=False) name = fields.String(dump_only=True, attribute='ip', default='') os = fields.String(default='') owned = fields.Boolean(default=False) diff --git a/server/models.py b/server/models.py index 4eff74e72f7..686b8a6c5ec 100644 --- a/server/models.py +++ b/server/models.py @@ -164,9 +164,9 @@ def creator(cls): class SourceCode(Metadata): __tablename__ = 'source_code' id = Column(Integer, primary_key=True) - filename = Column(Text, nullable=False) - function = Column(Text, nullable=True) - module = Column(Text, nullable=True) + filename = NonBlankColumn(Text) + function = BlankColumn(Text) + module = BlankColumn(Text) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship('Workspace', backref='source_codes') @@ -183,17 +183,17 @@ def parent(self): class Host(Metadata): __tablename__ = 'host' id = Column(Integer, primary_key=True) - ip = Column(Text, nullable=False) # IP v4 or v6 - description = Column(Text, nullable=True) - os = Column(Text, nullable=True) + ip = NonBlankColumn(Text) # IP v4 or v6 + description = BlankColumn(Text) + os = BlankColumn(Text) owned = Column(Boolean, nullable=False, default=False) - default_gateway_ip = Column(Text, nullable=True) - default_gateway_mac = Column(Text, nullable=True) + default_gateway_ip = BlankColumn(Text) + default_gateway_mac = BlankColumn(Text) - mac = Column(Text, nullable=True) - net_segment = Column(Text, nullable=True) + mac = BlankColumn(Text) + net_segment = BlankColumn(Text) services = relationship( 'Service', @@ -397,7 +397,7 @@ class VulnerabilityABC(Metadata): description = NonBlankColumn(Text) ease_of_resolution = Column(Enum(*EASE_OF_RESOLUTIONS, name='vulnerability_ease_of_resolution'), nullable=True) name = NonBlankColumn(Text, nullable=False) - resolution = Column(Text, nullable=True) + resolution = BlankColumn(Text) severity = Column(Enum(*SEVERITIES, name='vulnerability_severity'), nullable=False) risk = Column(Float(3, 1), nullable=True) @@ -631,14 +631,14 @@ class Command(Metadata): __tablename__ = 'command' id = Column(Integer, primary_key=True) - command = Column(Text(), nullable=False) - tool = Column(Text(), nullable=False) + command = NonBlankColumn(Text) + tool = NonBlankColumn(Text) start_date = Column(DateTime, nullable=False) end_date = Column(DateTime, nullable=True) ip = Column(String(250), nullable=False) # where the command was executed hostname = Column(String(250), nullable=False) # where the command was executed - params = Column(Text(), nullable=True) - user = Column(String(250), nullable=True) # os username where the command was executed + params = BlankColumn(Text) + user = BlankColumn(String(250)) # os username where the command was executed import_source = Column(Enum(*IMPORT_SOURCE, name='import_source_enum')) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) @@ -806,14 +806,14 @@ def parent(self): class VulnerabilityWeb(VulnerabilityGeneric): __tablename__ = None - method = Column(Text, nullable=True) - parameters = Column(Text, nullable=True) - parameter_name = Column(Text, nullable=True) - path = Column(Text, nullable=True) - query_string = Column(Text, nullable=True) - request = Column(Text, nullable=True) - response = Column(Text, nullable=True) - website = Column(Text, nullable=True) + method = BlankColumn(Text) + parameters = BlankColumn(Text) + parameter_name = BlankColumn(Text) + path = BlankColumn(Text) + query_string = BlankColumn(Text) + request = BlankColumn(Text) + response = BlankColumn(Text) + website = BlankColumn(Text) @declared_attr def service_id(cls): diff --git a/server/schemas.py b/server/schemas.py index 6578558232f..28d3e79ac15 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -143,13 +143,17 @@ def __init__(self, *args, **kwargs): super(NullToBlankString, self).__init__(*args, **kwargs) # Always make the field nullable because it is translated self.allow_none = True - - def _deserialize(self, value, attr, data): - print value, attr, data - if value is None and not self.nullable: - value = '' - return super(NullToBlankString, self)._deserialize( - value, attr, data) + self.default = '' + + def deserialize(self, value, attr=None, data=None): + # Validate required fields, deserialize, then validate + # deserialized value + self._validate_missing(value) + if getattr(self, 'allow_none', False) is True and value is None: + return '' + output = self._deserialize(value, attr, data) + self._validate(output) + return output class MetadataSchema(Schema): diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 0e251bb8fbd..1a28f1ed5a4 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -109,7 +109,7 @@ def test_create_a_host_succeeds(self, test_client): host = Host.query.get(host_id) assert host.ip == "127.0.0.1" assert host.description == "aaaaa" - assert host.os is None + assert host.os == '' assert host.workspace == self.workspace def test_create_a_host_fails_with_missing_desc(self, test_client): @@ -451,12 +451,12 @@ def test_update_host(self, test_client, session): u'type': 'Host', u'_rev': u'', u'credentials': 0, - u'default_gateway': None, + u'default_gateway': '', u'description': u'', u'hostnames': [], u'id': host.id, u'ip': u'10.31.112.21', - u'mac': None, + u'mac': '', u'metadata': { u'command_id': None, u'create_time': int(time.mktime(updated_host.create_date.timetuple())) * 1000, @@ -551,7 +551,7 @@ def test_create_a_host_twice_returns_conflict(self, test_client): host = Host.query.get(host_id) assert host.ip == "127.0.0.1" assert host.description == "aaaaa" - assert host.os is None + assert host.os == '' assert host.workspace == self.workspace res = test_client.post(self.url(), data={ "ip": "127.0.0.1", From fe3cfd00236f3da1ee355e82f9277fc07882fb8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 9 Mar 2018 16:10:00 -0300 Subject: [PATCH 0990/1506] Make text fields not nullable, second pass --- server/models.py | 92 ++++++++++++++++++++++++------------------------ 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/server/models.py b/server/models.py index 686b8a6c5ec..9a7bcf356bc 100644 --- a/server/models.py +++ b/server/models.py @@ -297,7 +297,7 @@ def set_children_objects(instance, value, parent_field, child_field='id', class Hostname(Metadata): __tablename__ = 'hostname' id = Column(Integer, primary_key=True) - name = Column(Text, nullable=False) + name = NonBlankColumn(Text) host_id = Column(Integer, ForeignKey('host.id'), index=True, nullable=False) host = relationship('Host', backref='hostnames') @@ -327,16 +327,16 @@ class Service(Metadata): ] __tablename__ = 'service' id = Column(Integer, primary_key=True) - name = Column(Text, nullable=True) - description = Column(Text, nullable=True) + name = BlankColumn(Text) + description = BlankColumn(Text) port = Column(Integer, nullable=False) owned = Column(Boolean, nullable=False, default=False) - protocol = Column(Text, nullable=False) + protocol = NonBlankColumn(Text) status = Column(Enum(*STATUSES, name='service_statuses'), nullable=False) - version = Column(Text, nullable=True) + version = BlankColumn(Text) - banner = Column(Text, nullable=True) + banner = BlankColumn(Text) host_id = Column(Integer, ForeignKey('host.id'), index=True, nullable=False) host = relationship( @@ -840,9 +840,9 @@ def hostnames(self): class VulnerabilityCode(VulnerabilityGeneric): __tablename__ = None - code = Column(Text, nullable=True) - start_line = Column(Integer, nullable=True) - end_line = Column(Integer, nullable=True) + code = BlankColumn(Text) + start_line = BlankColumn(Text) + end_line = BlankColumn(Text) source_code_id = Column(Integer, ForeignKey(SourceCode.id), index=True) source_code = relationship( @@ -867,7 +867,7 @@ def parent(self): class ReferenceTemplate(Metadata): __tablename__ = 'reference_template' id = Column(Integer, primary_key=True) - name = Column(Text, nullable=False) + name = NonBlankColumn(Text) __table_args__ = ( UniqueConstraint('name', name='uix_reference_template_name'), @@ -881,7 +881,7 @@ def __init__(self, name=None, **kwargs): class Reference(Metadata): __tablename__ = 'reference' id = Column(Integer, primary_key=True) - name = Column(Text, nullable=False) + name = NonBlankColumn(Text) workspace_id = Column( Integer, @@ -959,7 +959,7 @@ class PolicyViolationTemplateVulnerabilityAssociation(db.Model): class PolicyViolationTemplate(Metadata): __tablename__ = 'policy_violation_template' id = Column(Integer, primary_key=True) - name = Column(Text, nullable=False) + name = NonBlankColumn(Text) __table_args__ = ( UniqueConstraint( @@ -975,7 +975,7 @@ def __init__(self, name=None, **kwargs): class PolicyViolation(Metadata): __tablename__ = 'policy_violation' id = Column(Integer, primary_key=True) - name = Column(Text, nullable=False) + name = NonBlankColumn(Text) workspace_id = Column( Integer, @@ -1010,10 +1010,10 @@ def parent(self): class Credential(Metadata): __tablename__ = 'credential' id = Column(Integer, primary_key=True) - username = Column(String(250), nullable=False) - password = Column(Text(), nullable=False) - description = Column(Text(), nullable=True) - name = Column(String(250), nullable=True) + username = BlankColumn(Text) + password = BlankColumn(Text) + description = BlankColumn(Text) + name = BlankColumn(Text) host_id = Column(Integer, ForeignKey(Host.id), index=True, nullable=True) host = relationship('Host', backref='credentials', foreign_keys=[host_id]) @@ -1080,7 +1080,7 @@ class Workspace(Metadata): __tablename__ = 'workspace' id = Column(Integer, primary_key=True) customer = Column(String(250), nullable=True) # TBI - description = Column(Text(), nullable=True) + description = BlankColumn(Text) active = Column(Boolean(), nullable=False, default=True) # TBI end_date = Column(DateTime(), nullable=True) name = Column(String(250), nullable=False, unique=True) @@ -1147,7 +1147,7 @@ def set_scope(self, new_scope): class Scope(Metadata): __tablename__ = 'scope' id = Column(Integer, primary_key=True) - name = Column(Text(), nullable=False) + name = NonBlankColumn(Text) workspace_id = Column( Integer, @@ -1238,20 +1238,20 @@ class File(Metadata): __tablename__ = 'file' id = Column(Integer, autoincrement=True, primary_key=True) - name = Column(Text) - filename = Column(Text, nullable=False) - description = Column(Text) + name = BlankColumn(Text) # TODO migration: check why blank is allowed + filename = NonBlankColumn(Text) + description = BlankColumn(Text) content = Column(UploadedFileField(upload_type=FaradayUploadedFile), nullable=False) # plain attached file object_id = Column(Integer, nullable=False) - object_type = Column(Text, nullable=False) + object_type = NonBlankColumn(Text) # TODO migration: add enum class UserAvatar(Metadata): __tablename_ = 'user_avatar' id = Column(Integer, autoincrement=True, primary_key=True) - name = Column(Text, unique=True) + name = BlankColumn(Text, unique=True) # photo field will automatically generate thumbnail # if the file is a valid image photo = Column(UploadedFileField(upload_type=FaradayUploadedFile)) @@ -1263,14 +1263,14 @@ class MethodologyTemplate(Metadata): # TODO: reset template_id in methodologies when deleting meth template __tablename__ = 'methodology_template' id = Column(Integer, primary_key=True) - name = Column(Text, nullable=False) + name = NonBlankColumn(Text) class Methodology(Metadata): # TODO: add unique constraint -> name, workspace __tablename__ = 'methodology' id = Column(Integer, primary_key=True) - name = Column(Text, nullable=False) + name = NonBlankColumn(Text) template = relationship('MethodologyTemplate', backref='methodologies') template_id = Column( @@ -1292,8 +1292,8 @@ class TaskABC(Metadata): __abstract__ = True id = Column(Integer, primary_key=True) - name = Column(Text, nullable=False) - description = Column(Text, nullable=False) + name = NonBlankColumn(Text) + description = BlankColumn(Text) class TaskTemplate(TaskABC): @@ -1369,12 +1369,12 @@ def parent(self): class License(Metadata): __tablename__ = 'license' id = Column(Integer, primary_key=True) - product = Column(Text, nullable=False) + product = NonBlankColumn(Text) start_date = Column(DateTime, nullable=False) end_date = Column(DateTime, nullable=False) - type = Column(Text, nullable=True) - notes = Column(Text, nullable=True) + type = BlankColumn(Text) + notes = BlankColumn(Text) __table_args__ = ( UniqueConstraint('product', 'start_date', 'end_date', name='uix_license_product_start_end_dates'), @@ -1384,8 +1384,8 @@ class License(Metadata): class Tag(Metadata): __tablename__ = 'tag' id = Column(Integer, primary_key=True) - name = Column(Text, nullable=False, unique=True) - slug = Column(Text, nullable=False, unique=True) + name = NonBlankColumn(Text, unique=True) + slug = NonBlankColumn(Text, unique=True) class TagObject(db.Model): @@ -1393,7 +1393,7 @@ class TagObject(db.Model): id = Column(Integer, primary_key=True) object_id = Column(Integer, nullable=False) - object_type = Column(Text, nullable=False) + object_type = NonBlankColumn(Text) # TODO migration: add enum tag = relationship('Tag', backref='tagged_objects') tag_id = Column(Integer, ForeignKey('tag.id'), index=True) @@ -1403,7 +1403,7 @@ class Comment(Metadata): __tablename__ = 'comment' id = Column(Integer, primary_key=True) - text = Column(Text, nullable=False) + text = BlankColumn(Text) reply_to_id = Column(Integer, ForeignKey('comment.id')) reply_to = relationship( @@ -1417,7 +1417,7 @@ class Comment(Metadata): workspace = relationship('Workspace', foreign_keys=[workspace_id]) object_id = Column(Integer, nullable=False) - object_type = Column(Text, nullable=False) + object_type = NonBlankColumn(Text) # TODO migration: add enum @property def parent(self): @@ -1434,17 +1434,17 @@ class ExecutiveReport(Metadata): id = Column(Integer, primary_key=True) grouped = Column(Boolean, nullable=False, default=False) - name = Column(Text, nullable=False, index=True) + name = NonBlankColumn(Text, index=True) status = Column(Enum(*STATUSES, name='executive_report_statuses'), nullable=True) - template_name = Column(Text, nullable=False) - - conclusions = Column(Text, nullable=True) - enterprise = Column(Text, nullable=True) - objectives = Column(Text, nullable=True) - recommendations = Column(Text, nullable=True) - scope = Column(Text, nullable=True) - summary = Column(Text, nullable=True) - title = Column(Text, nullable=True) + template_name = NonBlankColumn(Text) + + conclusions = BlankColumn(Text) + enterprise = BlankColumn(Text) + objectives = BlankColumn(Text) + recommendations = BlankColumn(Text) + scope = BlankColumn(Text) + summary = BlankColumn(Text) + title = BlankColumn(Text) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship('Workspace', foreign_keys=[workspace_id]) From 03f596c5f317b1d0c4d7a1e1211ebaaccd1ff7e5 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 9 Mar 2018 16:48:08 -0300 Subject: [PATCH 0991/1506] Fix a bug when a host had blank ip --- server/events.py | 5 ++++- test_cases/test_api_hosts.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/server/events.py b/server/events.py index 558ab700f2e..86cbad782e2 100644 --- a/server/events.py +++ b/server/events.py @@ -28,7 +28,10 @@ def new_object_event(mapper, connection, instance): def delete_object_event(mapper, connection, instance): - name = getattr(instance, 'ip', None) or instance.name + try: + name = instance.ip + except AttributeError: + name = instance.name msg = { 'id': instance.id, 'action': 'DELETE', diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 0908daf3567..8fe3c942f8e 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -629,6 +629,18 @@ def test_service_summaries(self, test_client, session, service_factory): '(5353/udp) dns', ] + def test_delete_host_with_blank_ip(self, session, test_client): + """ + Bug found while deleting data from workspaces. + If we don't allow blank in name we should delete this test. + """ + host = self.factory.create(ip='') + session.add(host) + session.commit() + + res = test_client.delete(self.url(host, workspace=host.workspace)) + assert res.status_code == 204 + def host_json(): return st.fixed_dictionaries( From 1b5b6bac0cb9c2d83c5fbad300b3b603286b1b21 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 9 Mar 2018 16:48:27 -0300 Subject: [PATCH 0992/1506] Add more cascade deletes --- server/models.py | 85 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 59 insertions(+), 26 deletions(-) diff --git a/server/models.py b/server/models.py index 413804a130a..405516089db 100644 --- a/server/models.py +++ b/server/models.py @@ -196,10 +196,10 @@ class Host(Metadata): workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship( - 'Workspace', - backref='hosts', - foreign_keys=[workspace_id] - ) + 'Workspace', + foreign_keys=[workspace_id], + backref=backref("hosts", cascade="all, delete-orphan") + ) open_service_count = _make_generic_count_property( 'host', 'service', where=text("service.status = 'open'")) @@ -334,14 +334,16 @@ class Service(Metadata): host_id = Column(Integer, ForeignKey('host.id'), index=True, nullable=False) host = relationship( 'Host', - foreign_keys=[host_id]) + foreign_keys=[host_id], + cascade='all' + ) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship( - 'Workspace', - backref='services', - foreign_keys=[workspace_id] - ) + 'Workspace', + backref=backref('services', cascade="all, delete-orphan"), + foreign_keys=[workspace_id] + ) vulnerability_count = _make_generic_count_property('service', 'vulnerability') @@ -529,7 +531,11 @@ class CommandObject(db.Model): command_id = Column(Integer, ForeignKey('command.id'), index=True) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) - workspace = relationship('Workspace', foreign_keys=[workspace_id]) + workspace = relationship( + 'Workspace', + foreign_keys=[workspace_id], + backref = backref('command_objects', cascade="all, delete-orphan") + ) create_date = Column(DateTime, default=datetime.utcnow) @@ -635,7 +641,11 @@ class Command(Metadata): import_source = Column(Enum(*IMPORT_SOURCE, name='import_source_enum')) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) - workspace = relationship('Workspace', foreign_keys=[workspace_id]) + workspace = relationship( + 'Workspace', + foreign_keys=[workspace_id], + backref=backref('commands', cascade="all, delete-orphan") + ) sum_created_vulnerabilities = _make_created_objects_sum('vulnerability') @@ -912,7 +922,9 @@ class ReferenceVulnerabilityAssociation(db.Model): reference_id = Column(Integer, ForeignKey('reference.id'), primary_key=True) reference = relationship("Reference", - backref="reference_associations", + backref=backref( + "reference_associations", + cascade="all, delete-orphan"), foreign_keys=[reference_id]) vulnerability = relationship("Vulnerability", backref=backref("reference_vulnerability_associations", @@ -1027,10 +1039,10 @@ class Credential(Metadata): workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship( - 'Workspace', - backref='credentials', - foreign_keys=[workspace_id], - ) + 'Workspace', + backref=backref('credentials', cascade="all, delete-orphan"), + foreign_keys=[workspace_id], + ) __table_args__ = ( CheckConstraint('(host_id IS NULL AND service_id IS NOT NULL) OR ' @@ -1155,10 +1167,11 @@ class Scope(Metadata): index=True, nullable=False ) - workspace = relationship('Workspace', - backref=backref('scope', lazy="joined"), - foreign_keys=[workspace_id], - ) + workspace = relationship( + 'Workspace', + backref=backref('scope', lazy="joined", cascade="all, delete-orphan"), + foreign_keys=[workspace_id], + ) __table_args__ = ( UniqueConstraint('name', 'workspace_id', @@ -1272,7 +1285,10 @@ class Methodology(Metadata): id = Column(Integer, primary_key=True) name = Column(Text, nullable=False) - template = relationship('MethodologyTemplate', backref='methodologies') + template = relationship( + 'MethodologyTemplate', + backref=backref('methodologies', cascade="all, delete-orphan") + ) template_id = Column( Integer, ForeignKey('methodology_template.id'), @@ -1280,7 +1296,10 @@ class Methodology(Metadata): nullable=True, ) - workspace = relationship('Workspace', backref='methodologies') + workspace = relationship( + 'Workspace', + backref=backref('methodologies', cascade="all, delete-orphan") + ) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) @property @@ -1344,7 +1363,10 @@ class Task(TaskABC): index=True, nullable=False, ) - methodology = relationship('Methodology', backref='tasks') + methodology = relationship( + 'Methodology', + backref=backref('tasks', cascade="all, delete-orphan") + ) template_id = Column( Integer, @@ -1354,7 +1376,10 @@ class Task(TaskABC): ) template = relationship('TaskTemplate', backref='tasks') - workspace = relationship('Workspace', backref='tasks') + workspace = relationship( + 'Workspace', + backref=backref('tasks', cascade="all, delete-orphan") + ) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) # __table_args__ = ( @@ -1414,7 +1439,11 @@ class Comment(Metadata): workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) - workspace = relationship('Workspace', foreign_keys=[workspace_id]) + workspace = relationship( + 'Workspace', + foreign_keys=[workspace_id], + backref=backref('comments', cascade="all, delete-orphan"), + ) object_id = Column(Integer, nullable=False) object_type = Column(Text, nullable=False) @@ -1447,7 +1476,11 @@ class ExecutiveReport(Metadata): title = Column(Text, nullable=True) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) - workspace = relationship('Workspace', foreign_keys=[workspace_id]) + workspace = relationship( + 'Workspace', + backref=backref('reports', cascade="all, delete-orphan"), + foreign_keys=[workspace_id] + ) @property def parent(self): From ad4dfa6fe9ffef911fa25f2b16c18df29326f2b6 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 9 Mar 2018 17:25:38 -0300 Subject: [PATCH 0993/1506] Add a test to verify that command history is preserved when objects are deleted --- test_cases/test_api_commands.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index b108457d740..c9a3781901a 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -350,3 +350,31 @@ def test_update_command(self, test_client, session): assert res.status_code == 200 updated_command = self.model.query.get(command.id) assert updated_command.end_date == datetime.datetime.fromtimestamp(1511387720.048548) + datetime.timedelta(seconds=120) + + def test_delete_objects_preserve_history(self, session, test_client): + + command = EmptyCommandFactory(command='test', tool='test', workspace=self.workspace) + host = HostFactory.create(workspace=self.workspace) + session.add(host) + session.commit() + CommandObjectFactory.create( + command=command, + object_type='host', + object_id=host.id, + workspace=self.workspace + ) + session.commit() + + res = test_client.get(u'/v2/ws/{0}/hosts/{1}/'.format(host.workspace.name, host.id)) + assert res.status_code == 200 + + res = test_client.delete(u'/v2/ws/{0}/hosts/{1}/'.format(host.workspace.name, host.id)) + assert res.status_code == 204 + + res = test_client.get(self.url(workspace=command.workspace) + 'activity_feed/') + assert res.status_code == 200 + command_history = filter(lambda hist: hist['_id'] == command.id, res.json) + assert len(command_history) + command_history = command_history[0] + assert command_history['hosts_count'] == 1 + assert command_history['tool'] == 'test' \ No newline at end of file From 7d875e72f50fdbcdfd0dce615b333e9eb38e39ed Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 9 Mar 2018 17:37:42 -0300 Subject: [PATCH 0994/1506] delete generic relations: Tag and Comment for deleted object --- server/events.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/server/events.py b/server/events.py index 86cbad782e2..9fc88909140 100644 --- a/server/events.py +++ b/server/events.py @@ -1,14 +1,17 @@ import sys +import logging import inspect from models import ( Host, Service, -) + TagObject, Comment) from sqlalchemy import event - +from server.models import db from server.websocket_factories import changes_queue +logger = logging.getLogger(__name__) + def new_object_event(mapper, connection, instance): # Since we don't have jet a model for workspace we @@ -39,9 +42,18 @@ def delete_object_event(mapper, connection, instance): 'name': name, 'workspace': instance.workspace.name } + db.session.query(TagObject).filter_by( + object_id=instance.id, + object_type=msg['type'].lower(), + ).delete() + db.session.query(Comment).filter_by( + object_id = instance.id, + object_type = msg['type'].lower(), + ).delete() changes_queue.put(msg) + def update_object_event(mapper, connection, instance): delta = instance.update_date - instance.create_date if delta.seconds < 30: From 32c46b71dee5d8bfb54e1e7c93e98bb5f73871ba Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 12 Mar 2018 15:41:45 -0300 Subject: [PATCH 0995/1506] Add a test to make sure references and policy violations are not deleted --- test_cases/test_api_vulnerability.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 70346eb0fd6..406e5f8b5a5 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -35,7 +35,8 @@ Reference, PolicyViolation, CommandObject) from test_cases.factories import ServiceFactory, CommandFactory, \ CommandObjectFactory, HostFactory, EmptyCommandFactory, \ - UserFactory, VulnerabilityWebFactory, VulnerabilityFactory + UserFactory, VulnerabilityWebFactory, VulnerabilityFactory, \ + ReferenceFactory, PolicyViolationFactory CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) @@ -1327,6 +1328,27 @@ def test_invalid_host_id_error_message(self, test_client): assert res.status_code == 400 assert res.json == {u'messages': {u'_schema': [u'Parent id not found: 358302']}} + def test_after_deleting_vuln_ref_and_policies_remains(self, session, test_client): + vuln = VulnerabilityFactory.create(workspace=self.workspace) + ref1 = ReferenceFactory.create(workspace=self.workspace) + pv1 = PolicyViolationFactory.create(workspace=self.workspace) + vuln.reference_instances.add(ref1) + vuln.policy_violation_instances.add(pv1) + session.add(vuln) + session.commit() + + assert Reference.query.count() == 1 + assert PolicyViolation.query.count() == 1 + assert Vulnerability.query.count() == 6 + + res = test_client.delete(self.url(vuln)) + + assert res.status_code == 204 + + assert Reference.query.count() == 1 + assert PolicyViolation.query.count() == 1 + assert Vulnerability.query.count() == 5 + def test_type_filter(workspace, session, vulnerability_factory, From efbd9dc646a7455ff8c6c50719fdc067b67ec33e Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Mon, 12 Mar 2018 17:40:19 -0300 Subject: [PATCH 0996/1506] Add CVSS to Nessus plugin --- plugins/repo/nessus/dotnessus_v2.py | 1 + plugins/repo/nessus/plugin.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/plugins/repo/nessus/dotnessus_v2.py b/plugins/repo/nessus/dotnessus_v2.py index 532df56b295..b6d68b4cb07 100644 --- a/plugins/repo/nessus/dotnessus_v2.py +++ b/plugins/repo/nessus/dotnessus_v2.py @@ -32,6 +32,7 @@ 'cve', 'bid', 'xref', + 'cvss_base_score' ] # HOST_(START|END) date format diff --git a/plugins/repo/nessus/plugin.py b/plugins/repo/nessus/plugin.py index a7d68136cbc..73459491e4e 100644 --- a/plugins/repo/nessus/plugin.py +++ b/plugins/repo/nessus/plugin.py @@ -138,6 +138,8 @@ def parseOutputString(self, output, debug=False): ref = [] if v.get('cve'): ref.append(", ".join(v.get('cve'))) + if v.get('cvss_base_score'): + ref.append("CVSS: " + ", ".join(v.get('cvss_base_score'))) if v.get('bid'): ref.append(", ".join(v.get('bid'))) if v.get('xref'): From 22b7fba74e50ae7c443532c71ddbb40e2d099bde Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Mon, 12 Mar 2018 21:22:58 -0300 Subject: [PATCH 0997/1506] First version Webinspect plugin --- plugins/repo/webinspect/plugin.py | 172 ++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 plugins/repo/webinspect/plugin.py diff --git a/plugins/repo/webinspect/plugin.py b/plugins/repo/webinspect/plugin.py new file mode 100644 index 00000000000..32a9028f777 --- /dev/null +++ b/plugins/repo/webinspect/plugin.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Faraday Penetration Test IDE +Copyright (C) 2018 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information +""" + +from __future__ import with_statement +from plugins import core +from plugins.plugin_utils import get_vulnweb_url_fields +from model import api +import re + +try: + import xml.etree.ElementTree as ET +except ImportError: + import xml.etree.ElementTree as ET + + +def cleanhtml(raw_html): + cleanr = re.compile('<.*?>') + cleantext = re.sub(cleanr, '', raw_html) + return cleantext + + +class WebInspectParser(): + + def __init__(self, output): + self.xml = ET.fromstring(output) + self.issues = self.xml.findall("Issues/Issue") + + def parse_severity(self, severity): + + severity_dict = { + "0": "info", + "1": "low", + "2": "med", + "3": "high", + "4": "critical"} + + result = severity_dict.get(severity) + if not result: + return "info" + else: + return result + + def return_text(self, tag,element): + try: + text = element.find(tag).text.encode("ascii", errors="backslashreplace") + return text + except: + return "" + + def parse(self): + + map_objects_fields = { + "Name": ["Vuln", "name"], + "URL": ["Vuln", "website"], + "Scheme": ["Service", "name"], + "Host": ["Host", "name"], + "Port": ["Service", "port"], + "AttackMethod": ["Vuln", "method"], + "VulnerableSession": ["Vuln", "request"], + "VulnerabilityID": ["Vuln", "reference"], + "RawResponse": ["Vuln", "response"], + "Summary": ["Vuln", "description"], + "Implication": ["Vuln", "data"], + "Fix": ["Vuln", "resolution"], + "Reference Info": ["Vuln", "reference"], + "Severity": ["Vuln", "severity"] + } + + result = [] + for issue in self.issues: + + obj = { + "Host" : {}, + "Service" : {}, + "Interface" : {}, + "Vuln": { + "reference" : []} + } + + for tag, obj_property in map_objects_fields.iteritems(): + + value = self.return_text(tag,issue) + + if value != None: + + faraday_obj_name = obj_property[0] + faraday_field = obj_property[1] + if faraday_field == "reference": + obj[faraday_obj_name].get("reference").append(value) + else: + obj[faraday_obj_name].update({faraday_field:value}) + + # This for loads Summary, Implication, Fix and Reference + for section in issue.findall("ReportSection"): + + try: + field = section.find("Name").text.encode("ascii", errors="backslashreplace") + value = section.find("SectionText").text.encode("ascii", errors="backslashreplace") + + faraday_obj_name = map_objects_fields.get(field)[0] + faraday_field = map_objects_fields.get(field)[1] + except: + continue + + if faraday_field == "reference" and value != "": + obj[faraday_obj_name].get("reference").append(cleanhtml(value)) + else: + obj[faraday_obj_name].update({faraday_field:value}) + + result.append(obj) + return result + + +class WebInspectPlugin(core.PluginBase): + """ + This plugin handles WebInspect reports. + """ + + def __init__(self): + core.PluginBase.__init__(self) + self.id = "Webinspect" + self.name = "Webinspect" + self.plugin_version = "0.0.1" + self.version = "1.0.0" + + def parseOutputString(self, output, debug=False): + + parser = WebInspectParser(output) + vulns = parser.parse() + + for vuln in vulns: + + host_id = self.createAndAddHost( + vuln.get("Host").get("name")) + + interface_id = self.createAndAddInterface( + host_id, vuln.get("Host").get("name")) + + service_id = self.createAndAddServiceToInterface( + host_id, interface_id, + vuln.get("Service").get("name"), + protocol=vuln.get("Service").get("name"), + ports=[vuln.get("Service").get("port")]) + + self.createAndAddVulnWebToService( + host_id, service_id, + vuln.get("Vuln").get("name"), + website=get_vulnweb_url_fields(vuln.get("Vuln").get("website")).get("website"), + path=get_vulnweb_url_fields(vuln.get("Vuln").get("website")).get("path"), + query=get_vulnweb_url_fields(vuln.get("Vuln").get("website")).get("query"), + method=vuln.get("Vuln").get("method"), + request=vuln.get("Vuln").get("request"), + ref=filter(None ,vuln.get("Vuln").get("reference")), + response=vuln.get("Vuln").get("response"), + desc=cleanhtml(vuln.get("Vuln").get("description")), + resolution=cleanhtml(vuln.get("Vuln").get("resolution")), + severity=parser.parse_severity(vuln.get("Vuln").get("severity")) + ) + + return True + + def processCommandString(self, username, current_path, command_string): + return None + +def createPlugin(): + return WebInspectPlugin() From 4c77654818986323895e006a8e9becb33fded139 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 13 Mar 2018 16:11:30 -0300 Subject: [PATCH 0998/1506] Make the polling argument a boolean --- manage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manage.py b/manage.py index dc2384be387..7f3cbedf402 100755 --- a/manage.py +++ b/manage.py @@ -34,15 +34,15 @@ def check_faraday_server(url): @click.command() @click.option('--debug/--no-debug', default=False) @click.option('--workspace', default=None) -@click.option('--disable-polling', default=False) -def process_reports(debug, workspace, disable_polling): +@click.option('--polling/--no-polling', default=True) +def process_reports(debug, workspace, polling): setUpLogger(debug) configuration = _conf() url = '{0}/_api/v2/info'.format(configuration.getServerURI() if FARADAY_UP else SERVER_URL) with app.app_context(): try: check_faraday_server(url) - import_external_reports(workspace, disable_polling) + import_external_reports(workspace, polling) except OperationalError as ex: print('{0}'.format(ex)) print('Please verify database is running or configuration on server.ini!') From 8f566e699663462b76452bfb2453140c82fab566 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 13 Mar 2018 16:11:58 -0300 Subject: [PATCH 0999/1506] Add logging to report manager. avoid doing two iterations --- managers/reports_managers.py | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/managers/reports_managers.py b/managers/reports_managers.py index 08a642f9e6d..28a320d22e3 100755 --- a/managers/reports_managers.py +++ b/managers/reports_managers.py @@ -10,7 +10,6 @@ import os import re import time -import logging import traceback from threading import Thread @@ -25,7 +24,6 @@ from config.configuration import getInstanceConfiguration CONF = getInstanceConfiguration() -logger = logging.getLogger(__name__) class ReportProcessor(): @@ -129,28 +127,26 @@ def syncReports(self): Synchronize report directory using the DataManager and Plugins online We first make sure that all shared reports were added to the repo """ - filenames = [] - for root, dirs, files in os.walk(self._report_path, False): # skip processed and unprocessed directories if root == self._report_path: for name in files: - filenames.append(os.path.join(root, name)) - - for filename in filenames: - name = os.path.basename(filename) - # If plugin not is detected... move to unprocessed - # PluginCommiter will rename the file to processed or unprocessed - # when the plugin finishes - if self.processor.processReport(filename) is False: - logger.info('Plugin not detected. Moving {0} to unprocessed'.format(filename)) - os.rename( - filename, - os.path.join(self._report_upath, name)) - else: - os.rename( - filename, - os.path.join(self._report_ppath, name)) + filename = os.path.join(root, name) + name = os.path.basename(filename) + # If plugin not is detected... move to unprocessed + # PluginCommiter will rename the file to processed or unprocessed + # when the plugin finishes + if self.processor.processReport(filename) is False: + getLogger(self).info('Plugin not detected. Moving {0} to unprocessed'.format(filename)) + os.rename( + filename, + os.path.join(self._report_upath, name)) + else: + getLogger(self).info( + 'Detected valid report {0}'.format(filename)) + os.rename( + filename, + os.path.join(self._report_ppath, name)) self.onlinePlugins() From 2b010b8f8104947830221a7b043ec6e78f89eb8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 13 Mar 2018 16:13:39 -0300 Subject: [PATCH 1000/1506] Fix importer for not-nullable text columns --- server/importer.py | 46 ++++++++++++++++++++++++++++++++++++++++ server/utils/database.py | 2 +- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index c13ca94036e..d4b57e2f84a 100644 --- a/server/importer.py +++ b/server/importer.py @@ -21,6 +21,7 @@ OrderedDict ) from slugify import slugify +from sqlalchemy import Text, String from binascii import unhexlify from IPy import IP @@ -117,6 +118,51 @@ ] +# Really ugly hack to avoid setting to null non-nullable text columns +for model in ( + Command, + CommandObject, + Comment, + Credential, + ExecutiveReport, + Host, + Hostname, + License, + Methodology, + MethodologyTemplate, + PolicyViolation, + Reference, + Service, + Scope, + Tag, + TagObject, + Task, + TaskTemplate, + User, + Vulnerability, + VulnerabilityWeb, + VulnerabilityTemplate, + Workspace, + File, + ): + old_setattr = model.__setattr__ + + def __setattr__(self, key, value): + assert self.__table__ is not None + try: + column = self.__table__.columns[key] + except KeyError: + pass + else: + if (isinstance(column.type, (Text, String)) + and not column.nullable + and value is None): + value = '' + return old_setattr(self, key, value) + + model.__setattr__ = __setattr__ + + def get_object_from_couchdb(couchdb_id, workspace): doc_url = 'http://{username}:{password}@{hostname}:{port}/{workspace_name}/{doc_id}'.format( username=server.config.couchdb.user, diff --git a/server/utils/database.py b/server/utils/database.py index 47433cb1856..4d3affed79f 100644 --- a/server/utils/database.py +++ b/server/utils/database.py @@ -202,4 +202,4 @@ def _integer_to_boolean_postgresql(element, compiler, **kw): @compiler.compiles(BooleanToIntColumn, 'sqlite') def _integer_to_boolean_sqlite(element, compiler, **kw): - return element.expression_str \ No newline at end of file + return element.expression_str From 3672eb9f6f7a81ae83518b057b0de2d541314ee0 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 13 Mar 2018 16:17:05 -0300 Subject: [PATCH 1001/1506] Add status_code to VulnerabilityWeb --- server/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/models.py b/server/models.py index cef757cdb00..1db47baf6df 100644 --- a/server/models.py +++ b/server/models.py @@ -807,6 +807,7 @@ class VulnerabilityWeb(VulnerabilityGeneric): request = Column(Text, nullable=True) response = Column(Text, nullable=True) website = Column(Text, nullable=True) + status_code = Column(Integer, nullable=True) @declared_attr def service_id(cls): From d452f140413ec4dbacdadb3914ebefacacdd9c37 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 13 Mar 2018 16:19:41 -0300 Subject: [PATCH 1002/1506] Add support for status code on the backend --- server/api/modules/vulns.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 2080e64065e..ce8d6b191ef 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -303,7 +303,9 @@ class Meta(FilterSetMeta): "data", "severity", "confirmed", "name", "request", "response", "parameters", "params", "resolution", "ease_of_resolution", "description", "command_id", "target", "creator", "method", - "easeofresolution", "query_string", "parameter_name", "service_id") + "easeofresolution", "query_string", "parameter_name", "service_id", + "status_code" + ) strict_fields = ( "severity", "confirmed", "method", "status", "easeofresolution", From 6e9a09dc098d4ad45348751b204110045d917e1f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 13 Mar 2018 16:38:22 -0300 Subject: [PATCH 1003/1506] add status code to the api schema --- server/api/modules/vulns.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 77837ab57a0..f5a011d6172 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -243,6 +243,7 @@ class VulnerabilityWebSchema(VulnerabilitySchema): request = fields.String(default='') website = fields.String(default='') query = fields.String(attribute='query_string', default='') + status_code = fields.Integer(allow_none=True) class Meta: model = VulnerabilityWeb @@ -255,7 +256,7 @@ class Meta: 'desc', 'impact', 'confirmed', 'name', 'service', 'obj_id', 'type', 'policyviolations', 'request', '_attachments', 'params', - 'target', 'resolution', 'method', 'metadata') + 'target', 'resolution', 'method', 'metadata', 'status_code') # Use this override for filterset fields that filter by en exact match by From d2a9a48a08cee0c3b801edc422c5ddc6fab732ab Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 13 Mar 2018 16:38:41 -0300 Subject: [PATCH 1004/1506] Fix start_line and end_line types --- server/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/models.py b/server/models.py index e1701cc92a3..8d68e646230 100644 --- a/server/models.py +++ b/server/models.py @@ -842,8 +842,8 @@ def hostnames(self): class VulnerabilityCode(VulnerabilityGeneric): __tablename__ = None code = BlankColumn(Text) - start_line = BlankColumn(Text) - end_line = BlankColumn(Text) + start_line = Column(Integer, nullable=True) + end_line = Column(Integer, nullable=True) source_code_id = Column(Integer, ForeignKey(SourceCode.id), index=True) source_code = relationship( From e915e26b49a07a7417b81f832acc2022a486abd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 13 Mar 2018 17:16:51 -0300 Subject: [PATCH 1005/1506] Fix bugs when detecting conflicts with different vuln types --- server/api/base.py | 2 ++ server/utils/database.py | 41 ++++++++++++++++++++------ test_cases/test_api_vulnerability.py | 43 ++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 8 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 46ebc4f9ed0..8d0607fbc6e 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -427,6 +427,7 @@ def _perform_create(self, data, **kwargs): conflict_obj).data, } )) + raise return obj @@ -497,6 +498,7 @@ def _perform_create(self, data, workspace_name): conflict_obj).data, } )) + raise self._set_command_id(obj, True) return obj diff --git a/server/utils/database.py b/server/utils/database.py index 9b35535f2a5..4d2902b71e7 100644 --- a/server/utils/database.py +++ b/server/utils/database.py @@ -2,6 +2,7 @@ # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information +import operator from sqlalchemy import distinct, Boolean from sqlalchemy.sql import func, asc, desc from sqlalchemy.sql.expression import ClauseElement @@ -257,22 +258,46 @@ def get_conflict_object(session, obj, data, workspace=None): unique_fields = filter( lambda unique_field: not unique_field.endswith('_id'), unique_fields) - filter_data = {unique_field: data[unique_field] for unique_field in - unique_fields if unique_field in data} + + if get_object_type_for(obj) == 'vulnerability': + # This is a special key due to model inheritance + from server.models import VulnerabilityGeneric + klass = VulnerabilityGeneric + else: + klass = obj.__class__ + + table = klass.__table__ + assert (klass is not None and table is not None) + + filter_data = [] + for unique_field in unique_fields: + column = table.columns[unique_field] + try: + value = data[unique_field] + except KeyError: + if column.default is None: + # No default nor data value, ignore the field + continue + value = column.default.arg + filter_data.append(column == value) if 'workspace_id' in relations_fields: relations_fields.remove('workspace_id') - filter_data['workspace_id'] = workspace.id + filter_data.append(table.columns['workspace_id'] == workspace.id) for relations_field in relations_fields: if relations_field not in data and relations_field.strip('_id') in data: - filter_data[relations_field] = data[ - relations_field.strip('_id')].id + related_object = data[relations_field.strip('_id')] + assert related_object.id is not None + filter_data.append( + table.columns[relations_field] == related_object.id) else: relation_id = data.get(relations_field, None) if relation_id: - filter_data[relations_field] = relation_id + filter_data.append( + table.columns[relations_field] == relation_id) - conflict_obj = session.query(obj.__class__).filter_by( - **filter_data).first() + filter_data = reduce(operator.and_, filter_data) + conflict_obj = session.query(klass).filter( + filter_data).first() return conflict_obj diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index f643c972295..808ed3e5be3 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1227,6 +1227,49 @@ def test_create_webvuln_multiple_times_returns_conflict(self, host_with_hostname data=raw_data) assert res.status_code == 409 + def test_create_vuln_service_vuln_web_conflict( + self, service, vulnerability_factory, vulnerability_web_factory, + session, test_client, workspace): + service_vuln = vulnerability_factory.create( + service=service, host=None, workspace=workspace, + name="test conflict", description="test" + ) + session.commit() + old_count = VulnerabilityGeneric.query.count() + raw_data = _create_post_data_vulnerability( + name='test conflict', + description='test', + vuln_type='Vulnerability', + parent_id=service.id, + parent_type='Service', + refs=[], + policyviolations=[], + severity='low', + ) + raw_data['type'] = 'VulnerabilityWeb' + res = test_client.post(self.url(), data=raw_data) + assert res.status_code == 409 + assert VulnerabilityGeneric.query.count() == old_count + + def test_update_conflict(self, host, vulnerability_factory, session, + test_client): + vulnerability_factory.create( + workspace=self.workspace, host=host, service=None, + name="x", description="x") + target_vuln = vulnerability_factory.create( + workspace=self.workspace, host=host, service=None, + name="y", description="y") + session.commit() + raw_data = self._create_put_data( + 'x', + 'x', + 'open', + host.id, + 'Host', + ) + res = test_client.put(self.url(obj=target_vuln), data=raw_data) + assert res.status_code == 409 + def test_create_and_update_webvuln(self, host_with_hostnames, test_client, session): """ This reproduces a bug found. after creating an object with a From e95ea3587b0fe39c4c8e4753ba0378179d578ad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 13 Mar 2018 17:33:56 -0300 Subject: [PATCH 1006/1506] Improve unique contraint exception detection Check for pgcode when using postgresql, re-raise exception if no conflicting object found --- server/api/base.py | 19 ++++++++++++++++--- server/models.py | 31 ++++++++++++++++--------------- server/utils/database.py | 13 +++++++++++++ 3 files changed, 45 insertions(+), 18 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 8d0607fbc6e..40612fd1719 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -18,7 +18,10 @@ from server.models import Workspace, db, Command, CommandObject from server.schemas import NullToBlankString import server.utils.logger -from server.utils.database import get_conflict_object +from server.utils.database import ( + get_conflict_object, + is_unique_constraint_violation + ) logger = server.utils.logger.get_logger(__name__) @@ -417,6 +420,8 @@ def _perform_create(self, data, **kwargs): db.session.add(obj) db.session.commit() except sqlalchemy.exc.IntegrityError as ex: + if not is_unique_constraint_violation(ex): + raise db.session.rollback() conflict_obj = get_conflict_object(db.session, obj, data) if conflict_obj: @@ -427,7 +432,8 @@ def _perform_create(self, data, **kwargs): conflict_obj).data, } )) - raise + else: + raise return obj @@ -487,6 +493,8 @@ def _perform_create(self, data, workspace_name): db.session.add(obj) db.session.commit() except sqlalchemy.exc.IntegrityError as ex: + if not is_unique_constraint_violation(ex): + raise db.session.rollback() workspace = self._get_workspace(workspace_name) conflict_obj = get_conflict_object(db.session, obj, data, workspace) @@ -498,7 +506,8 @@ def _perform_create(self, data, workspace_name): conflict_obj).data, } )) - raise + else: + raise self._set_command_id(obj, True) return obj @@ -528,6 +537,8 @@ def _perform_update(self, object_id, obj, data, workspace_name=None): db.session.add(obj) db.session.commit() except sqlalchemy.exc.IntegrityError as ex: + if not is_unique_constraint_violation(ex): + raise db.session.rollback() workspace = None if workspace_name: @@ -541,6 +552,8 @@ def _perform_update(self, object_id, obj, data, workspace_name=None): conflict_obj).data, } )) + else: + raise return obj diff --git a/server/models.py b/server/models.py index 161112ed535..03637ba1cea 100644 --- a/server/models.py +++ b/server/models.py @@ -44,7 +44,10 @@ RoleMixin, UserMixin, ) -from server.utils.database import BooleanToIntColumn, get_object_type_for +from server.utils.database import ( + BooleanToIntColumn, + get_object_type_for, + is_unique_constraint_violation) NonBlankColumn = partial(Column, nullable=False, info={'allow_blank': False}) @@ -60,7 +63,6 @@ 'source_code', 'comment', ] -UNIQUE_VIOLATION = '23505' class SQLAlchemy(OriginalSQLAlchemy): @@ -451,20 +453,19 @@ def _create(self, value): try: yield self.creator(value, parent_instance) except IntegrityError as ex: - if ex.orig.pgcode == UNIQUE_VIOLATION: - # unique constraint failed at database - # other process/thread won us on the commit - # we need to fetch already created objs. - session.rollback() - conflict_obj_names = [obj.name for obj in conflict_objs if obj.name != value] - for conflict_obj_name in conflict_obj_names: - conclict_obj = session.query(Reference).filter_by(name=conflict_obj_name).first() - if not conclict_obj: - raise Exception('This should not happend. AssocProxy could not find a conflict obj.') - self.col.add(conclict_obj) - yield self.creator(value, parent_instance) - else: + if not is_unique_constraint_violation(ex): raise + # unique constraint failed at database + # other process/thread won us on the commit + # we need to fetch already created objs. + session.rollback() + conflict_obj_names = [obj.name for obj in conflict_objs if obj.name != value] + for conflict_obj_name in conflict_obj_names: + conclict_obj = session.query(Reference).filter_by(name=conflict_obj_name).first() + if not conclict_obj: + raise Exception('This should not happend. AssocProxy could not find a conflict obj.') + self.col.add(conclict_obj) + yield self.creator(value, parent_instance) def add(self, value): if value not in self: diff --git a/server/utils/database.py b/server/utils/database.py index 4d2902b71e7..68535002068 100644 --- a/server/utils/database.py +++ b/server/utils/database.py @@ -301,3 +301,16 @@ def get_conflict_object(session, obj, data, workspace=None): conflict_obj = session.query(klass).filter( filter_data).first() return conflict_obj + + +UNIQUE_VIOLATION = '23505' + + +def is_unique_constraint_violation(exception): + from server.models import db + if db.engine.dialect != 'postgresql': + # Not implemened for RDMS other than postgres, we can't live without + # this, it is just an extra check + return True + assert isinstance(exception.orig.pgcode, str) + return exception.orig.pgcode == UNIQUE_VIOLATION From f87bb16d7e9086374d05162b0402fc80afcd9575 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 13 Mar 2018 17:53:21 -0300 Subject: [PATCH 1007/1506] Allow vuln desc to be empty --- server/api/modules/vulns.py | 2 +- server/models.py | 2 +- test_cases/test_api_vulnerability.py | 20 +++++++++++++++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index f5a011d6172..860bb46735a 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -95,7 +95,7 @@ class VulnerabilitySchema(AutoSchema): owned = fields.Boolean(dump_only=True, default=False) owner = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator') impact = SelfNestedField(ImpactSchema()) - desc = fields.String(attribute='description', validate=Length(min=1)) + desc = fields.String(attribute='description') description = fields.String(dump_only=True) policyviolations = fields.List(fields.String, attribute='policy_violations') diff --git a/server/models.py b/server/models.py index 8d68e646230..3d098ce96f6 100644 --- a/server/models.py +++ b/server/models.py @@ -394,7 +394,7 @@ class VulnerabilityABC(Metadata): id = Column(Integer, primary_key=True) data = BlankColumn(Text) - description = NonBlankColumn(Text) + description = BlankColumn(Text) ease_of_resolution = Column(Enum(*EASE_OF_RESOLUTIONS, name='vulnerability_ease_of_resolution'), nullable=True) name = NonBlankColumn(Text, nullable=False) resolution = BlankColumn(Text) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 0884e8bd9b6..e0fe74f2b37 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -262,7 +262,7 @@ def test_create_vuln(self, host_with_hostnames, test_client, session): assert res.json['description'] == 'helloworld' assert res.json['severity'] == 'low' - def test_create_cannot_create_vuln_with_empty_name( + def test_create_cannot_create_vuln_with_empty_name_fails( self, host, session, test_client): # I'm using this to test the NonBlankColumn which works for # all models. Think twice before removing this test @@ -281,6 +281,24 @@ def test_create_cannot_create_vuln_with_empty_name( assert res.status_code == 400 assert 'Shorter than minimum length 1' in res.data + def test_create_cannot_create_vuln_with_empty_desc_succed( + self, host, session, test_client): + # I'm using this to test the NonBlankColumn which works for + # all models. Think twice before removing this test + session.commit() # flush host_with_hostnames + raw_data = _create_post_data_vulnerability( + name='Empty desc', + vuln_type='Vulnerability', + parent_id=host.id, + parent_type='Host', + refs=[], + policyviolations=[], + description='', + severity='low', + ) + res = test_client.post(self.url(), data=raw_data) + assert res.status_code == 201 + def test_create_cannot_create_vuln_with_empty_description( self, host, session, test_client): session.commit() # flush host_with_hostnames From a915891870412a06048fbdd6f1fe1eeb06fc32a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 13 Mar 2018 18:06:34 -0300 Subject: [PATCH 1008/1506] Remove invalid vuln test case --- test_cases/test_api_vulnerability.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index e0fe74f2b37..8312fef564f 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -281,7 +281,7 @@ def test_create_cannot_create_vuln_with_empty_name_fails( assert res.status_code == 400 assert 'Shorter than minimum length 1' in res.data - def test_create_cannot_create_vuln_with_empty_desc_succed( + def test_create_create_vuln_with_empty_desc_success( self, host, session, test_client): # I'm using this to test the NonBlankColumn which works for # all models. Think twice before removing this test @@ -299,23 +299,6 @@ def test_create_cannot_create_vuln_with_empty_desc_succed( res = test_client.post(self.url(), data=raw_data) assert res.status_code == 201 - def test_create_cannot_create_vuln_with_empty_description( - self, host, session, test_client): - session.commit() # flush host_with_hostnames - raw_data = _create_post_data_vulnerability( - name='New vulns', - vuln_type='Vulnerability', - parent_id=host.id, - parent_type='Host', - refs=[], - policyviolations=[], - description='', - severity='low', - ) - res = test_client.post(self.url(), data=raw_data) - assert res.status_code == 400 - assert 'Shorter than minimum length 1' in res.data - def test_create_vuln_with_attachments(self, host_with_hostnames, test_client, session): session.commit() # flush host_with_hostnames attachment = NamedTemporaryFile() From 44f7042a229b5c8c6054cf5218ab37fa509a325a Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 13 Mar 2018 18:20:43 -0300 Subject: [PATCH 1009/1506] Avoid None in request, response and panme serialization form plugins --- persistence/server/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/persistence/server/utils.py b/persistence/server/utils.py index 0fd1fbc883e..36197f98d73 100644 --- a/persistence/server/utils.py +++ b/persistence/server/utils.py @@ -93,11 +93,11 @@ def get_vuln_web_properties(vuln_web): vuln_web_dict = { 'method': vuln_web.getMethod(), 'params': vuln_web.getParams(), - 'request': vuln_web.getRequest(), - 'response': vuln_web.getResponse(), + 'request': vuln_web.getRequest() or '', + 'response': vuln_web.getResponse() or '', 'website': vuln_web.getWebsite(), 'path': vuln_web.getPath(), - 'pname': vuln_web.getPname(), + 'pname': vuln_web.getPname() or '', 'query': vuln_web.getQuery(), 'status': vuln_web.getStatus(), 'parent': vuln_web.getParent(), From 6d75a4def095e95ffeb26d42cb2d215c086c9538 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 14 Mar 2018 16:31:48 -0300 Subject: [PATCH 1010/1506] Update angular moment. convert from utc to local time --- server/schemas.py | 4 +- server/www/script/angular-moment.js | 359 +++++++++++------- .../dashboard/partials/activityFeed.html | 2 +- .../dashboard/partials/commands-list.html | 2 +- .../dashboard/partials/last-vulns.html | 2 +- .../partials/workspace-progress.html | 4 +- server/www/scripts/hosts/partials/list.html | 4 +- .../partials/ui-grid/columns/datecolumn.html | 2 +- .../www/scripts/workspaces/partials/list.html | 4 +- test_cases/test_api_vulnerability.py | 2 +- 10 files changed, 245 insertions(+), 140 deletions(-) diff --git a/server/schemas.py b/server/schemas.py index 28d3e79ac15..b9d4e3faea3 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -162,8 +162,8 @@ class MetadataSchema(Schema): creator = fields.Function(lambda x: '', dump_only=True) owner = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator') - create_time = JSTimestampField(attribute='create_date', dump_only=True) - update_time = JSTimestampField(attribute='update_date', dump_only=True) + update_time = fields.DateTime(attribute='update_date', dump_only=True) + create_time = fields.DateTime(attribute='create_date', dump_only=True) update_user = fields.String(default='', dump_only=True) update_action = fields.Integer(default=0, dump_only=True) diff --git a/server/www/script/angular-moment.js b/server/www/script/angular-moment.js index 100b88da71f..628983a0677 100644 --- a/server/www/script/angular-moment.js +++ b/server/www/script/angular-moment.js @@ -1,4 +1,4 @@ -/* angular-moment.js / v0.10.3 / (c) 2013, 2014, 2015 Uri Shaked / MIT Licence */ +/* angular-moment.js / v1.2.0 / (c) 2013, 2014, 2015, 2016, 2017 Uri Shaked / MIT Licence */ 'format amd'; /* global define */ @@ -6,8 +6,28 @@ (function () { 'use strict'; + function isUndefinedOrNull(val) { + return angular.isUndefined(val) || val === null; + } + + function requireMoment() { + try { + return require('moment'); // Using nw.js or browserify? + } catch (e) { + throw new Error('Please install moment via npm. Please reference to: https://github.com/urish/angular-moment'); // Add wiki/troubleshooting section? + } + } + function angularMoment(angular, moment) { + if(typeof moment === 'undefined') { + if(typeof require === 'function') { + moment = requireMoment(); + }else{ + throw new Error('Moment cannot be found by angular-moment! Please reference to: https://github.com/urish/angular-moment'); // Add wiki/troubleshooting section? + } + } + /** * @ngdoc overview * @name angularMoment @@ -15,7 +35,7 @@ * @description * angularMoment module provides moment.js functionality for angular.js apps. */ - return angular.module('angularMoment', []) + angular.module('angularMoment', []) /** * @ngdoc object @@ -29,13 +49,19 @@ * @ngdoc property * @name angularMoment.config.angularMomentConfig#preprocess * @propertyOf angularMoment.config:angularMomentConfig - * @returns {string} The default preprocessor to apply + * @returns {function} A preprocessor function that will be applied on all incoming dates * * @description - * Defines a default preprocessor to apply (e.g. 'unix', 'etc', ...). The default value is null, - * i.e. no preprocessor will be applied. + * Defines a preprocessor function to apply on all input dates (e.g. the input of `am-time-ago`, + * `amCalendar`, etc.). The function must return a `moment` object. + * + * @example + * // Causes angular-moment to always treat the input values as unix timestamps + * angularMomentConfig.preprocess = function(value) { + * return moment.unix(value); + * } */ - preprocess: null, // e.g. 'unix', 'utc', ... + preprocess: null, /** * @ngdoc property @@ -46,8 +72,10 @@ * @description * The default timezone (e.g. 'Europe/London'). Empty string by default (does not apply * any timezone shift). + * + * NOTE: This option requires moment-timezone >= 0.3.0. */ - timezone: '', + timezone: null, /** * @ngdoc property @@ -71,7 +99,7 @@ * @description * Specifies whether the filters included with angular-moment are stateful. * Stateful filters will automatically re-evaluate whenever you change the timezone - * or language settings, but may negatively impact performance. true by default. + * or locale settings, but may negatively impact performance. true by default. */ statefulFilters: true }) @@ -150,7 +178,9 @@ * @description * Specify the format of the date when displayed as full date. null by default. */ - fullDateFormat: null + fullDateFormat: null, + + fullDateThresholdUnit: 'day' }) /** @@ -160,21 +190,22 @@ * * @restrict A */ - .directive('amTimeAgo', ['$window', 'moment', 'amMoment', 'amTimeAgoConfig', 'angularMomentConfig', function ($window, moment, amMoment, amTimeAgoConfig, angularMomentConfig) { + .directive('amTimeAgo', ['$window', 'moment', 'amMoment', 'amTimeAgoConfig', function ($window, moment, amMoment, amTimeAgoConfig) { return function (scope, element, attr) { var activeTimeout = null; var currentValue; - var currentFormat = angularMomentConfig.format; var withoutSuffix = amTimeAgoConfig.withoutSuffix; var titleFormat = amTimeAgoConfig.titleFormat; var fullDateThreshold = amTimeAgoConfig.fullDateThreshold; var fullDateFormat = amTimeAgoConfig.fullDateFormat; + var fullDateThresholdUnit = amTimeAgoConfig.fullDateThresholdUnit; + var localDate = new Date().getTime(); - var preprocess = angularMomentConfig.preprocess; var modelName = attr.amTimeAgo; var currentFrom; var isTimeElement = ('TIME' === element[0].nodeName.toUpperCase()); + var setTitleTime = !element.attr('title'); function getNow() { var now; @@ -199,8 +230,8 @@ } function updateTime(momentInstance) { - var daysAgo = getNow().diff(momentInstance, 'day'); - var showFullDate = fullDateThreshold && daysAgo >= fullDateThreshold; + var timeAgo = getNow().diff(momentInstance, fullDateThresholdUnit); + var showFullDate = fullDateThreshold && timeAgo >= fullDateThreshold; if (showFullDate) { element.text(momentInstance.format(fullDateFormat)); @@ -208,8 +239,8 @@ element.text(momentInstance.from(getNow(), withoutSuffix)); } - if (titleFormat && !element.attr('title')) { - element.attr('title', momentInstance.local().format(titleFormat)); + if (titleFormat && setTitleTime) { + element.attr('title', momentInstance.format(titleFormat)); } if (!showFullDate) { @@ -238,14 +269,14 @@ function updateMoment() { cancelTimer(); if (currentValue) { - var momentValue = amMoment.preprocessDate(currentValue, preprocess, currentFormat); + var momentValue = amMoment.preprocessDate(currentValue); updateTime(momentValue); updateDateTimeAttr(momentValue.toISOString()); } } scope.$watch(modelName, function (value) { - if ((typeof value === 'undefined') || (value === null) || (value === '')) { + if (isUndefinedOrNull(value) || (value === '')) { cancelTimer(); if (currentValue) { element.text(''); @@ -261,7 +292,7 @@ if (angular.isDefined(attr.amFrom)) { scope.$watch(attr.amFrom, function (value) { - if ((typeof value === 'undefined') || (value === null) || (value === '')) { + if (isUndefinedOrNull(value) || (value === '')) { currentFrom = null; } else { currentFrom = moment(value); @@ -281,18 +312,6 @@ }); } - attr.$observe('amFormat', function (format) { - if (typeof format !== 'undefined') { - currentFormat = format; - updateMoment(); - } - }); - - attr.$observe('amPreprocess', function (newValue) { - preprocess = newValue; - updateMoment(); - }); - attr.$observe('amFullDateThreshold', function (newValue) { fullDateThreshold = newValue; updateMoment(); @@ -303,6 +322,11 @@ updateMoment(); }); + attr.$observe('amFullDateThresholdUnit', function (newValue) { + fullDateThresholdUnit = newValue; + updateMoment(); + }); + scope.$on('$destroy', function () { cancelTimer(); }); @@ -319,19 +343,7 @@ * @module angularMoment */ .service('amMoment', ['moment', '$rootScope', '$log', 'angularMomentConfig', function (moment, $rootScope, $log, angularMomentConfig) { - /** - * @ngdoc property - * @name angularMoment:amMoment#preprocessors - * @module angularMoment - * - * @description - * Defines the preprocessors for the preprocessDate method. By default, the following preprocessors - * are defined: utc, unix. - */ - this.preprocessors = { - utc: moment.utc, - unix: moment.unix - }; + var defaultTimezone = null; /** * @ngdoc function @@ -363,11 +375,19 @@ * Changes the default timezone for amCalendar, amDateFormat and amTimeAgo. Also broadcasts an * `amMoment:timezoneChanged` event on $rootScope. * + * Note: this method works only if moment-timezone > 0.3.0 is loaded + * * @param {string} timezone Timezone name (e.g. UTC) */ this.changeTimezone = function (timezone) { + if (moment.tz && moment.tz.setDefault) { + moment.tz.setDefault(timezone); + $rootScope.$broadcast('amMoment:timezoneChanged'); + } else { + $log.warn('angular-moment: changeTimezone() works only with moment-timezone.js v0.3.0 or greater.'); + } angularMomentConfig.timezone = timezone; - $rootScope.$broadcast('amMoment:timezoneChanged'); + defaultTimezone = timezone; }; /** @@ -377,62 +397,120 @@ * * @description * Preprocess a given value and convert it into a Moment instance appropriate for use in the - * am-time-ago directive and the filters. + * am-time-ago directive and the filters. The behavior of this function can be overriden by + * setting `angularMomentConfig.preprocess`. * * @param {*} value The value to be preprocessed - * @param {string} preprocess The name of the preprocessor the apply (e.g. utc, unix) - * @param {string=} format Specifies how to parse the value (see {@link http://momentjs.com/docs/#/parsing/string-format/}) - * @return {Moment} A value that can be parsed by the moment library + * @return {Moment} A `moment` object */ - this.preprocessDate = function (value, preprocess, format) { - if (angular.isUndefined(preprocess)) { - preprocess = angularMomentConfig.preprocess; - } - if (this.preprocessors[preprocess]) { - return this.preprocessors[preprocess](value, format); + this.preprocessDate = function (value) { + // Configure the default timezone if needed + if (defaultTimezone !== angularMomentConfig.timezone) { + this.changeTimezone(angularMomentConfig.timezone); } - if (preprocess) { - $log.warn('angular-moment: Ignoring unsupported value for preprocess: ' + preprocess); + + if (angularMomentConfig.preprocess) { + return angularMomentConfig.preprocess(value); } + if (!isNaN(parseFloat(value)) && isFinite(value)) { // Milliseconds since the epoch return moment(parseInt(value, 10)); } + // else just returns the value as-is. + return moment(value); + }; + }]) + + /** + * @ngdoc filter + * @name angularMoment.filter:amParse + * @module angularMoment + */ + .filter('amParse', ['moment', function (moment) { + return function (value, format) { return moment(value, format); }; + }]) + + /** + * @ngdoc filter + * @name angularMoment.filter:amFromUnix + * @module angularMoment + */ + .filter('amFromUnix', ['moment', function (moment) { + return function (value) { + return moment.unix(value); + }; + }]) + + /** + * @ngdoc filter + * @name angularMoment.filter:amUtc + * @module angularMoment + */ + .filter('amUtc', ['moment', function (moment) { + return function (value) { + return moment.utc(value); + }; + }]) + + /** + * @ngdoc filter + * @name angularMoment.filter:amUtcOffset + * @module angularMoment + * + * @description + * Adds a UTC offset to the given timezone object. The offset can be a number of minutes, or a string such as + * '+0300', '-0300' or 'Z'. + */ + .filter('amUtcOffset', ['amMoment', function (amMoment) { + function amUtcOffset(value, offset) { + return amMoment.preprocessDate(value).utcOffset(offset); + } + + return amUtcOffset; + }]) + + /** + * @ngdoc filter + * @name angularMoment.filter:amLocal + * @module angularMoment + */ + .filter('amLocal', ['moment', function (moment) { + return function (value) { + return moment.isMoment(value) ? value.local() : null; + }; + }]) + + /** + * @ngdoc filter + * @name angularMoment.filter:amTimezone + * @module angularMoment + * + * @description + * Apply a timezone onto a given moment object, e.g. 'America/Phoenix'). + * + * You need to include moment-timezone.js for timezone support. + */ + .filter('amTimezone', ['amMoment', 'angularMomentConfig', '$log', function (amMoment, angularMomentConfig, $log) { + function amTimezone(value, timezone) { + var aMoment = amMoment.preprocessDate(value); - /** - * @ngdoc function - * @name angularMoment.service.amMoment#applyTimezone - * @methodOf angularMoment.service.amMoment - * - * @description - * Apply a timezone onto a given moment object. It can be a named timezone (e.g. 'America/Phoenix') or an offset from UTC (e.g. '+0300') - * moment-timezone.js is needed when a named timezone is used, otherwise, it'll not apply any timezone shift. - * - * @param {Moment} aMoment a moment() instance to apply the timezone shift to - * @param {string=} timezone The timezone to apply. If none given, will apply the timezone - * configured in angularMomentConfig.timezone. It can be a named timezone (e.g. 'America/Phoenix') or an offset from UTC (e.g. '+0300') - * - * @returns {Moment} The given moment with the timezone shift applied - */ - this.applyTimezone = function (aMoment, timezone) { - timezone = timezone || angularMomentConfig.timezone; if (!timezone) { return aMoment; } - if (timezone.match(/^Z|[+-]\d\d:?\d\d$/i)) { - aMoment = aMoment.utcOffset(timezone); - } else if (aMoment.tz) { - aMoment = aMoment.tz(timezone); + if (aMoment.tz) { + return aMoment.tz(timezone); } else { - $log.warn('angular-moment: named timezone specified but moment.tz() is undefined. Did you forget to include moment-timezone.js?'); + $log.warn('angular-moment: named timezone specified but moment.tz() is undefined. Did you forget to include moment-timezone.js ?'); + return aMoment; } + } - return aMoment; - }; + return amTimezone; }]) /** @@ -441,18 +519,13 @@ * @module angularMoment */ .filter('amCalendar', ['moment', 'amMoment', 'angularMomentConfig', function (moment, amMoment, angularMomentConfig) { - function amCalendarFilter(value, preprocess, timezone) { - if (typeof value === 'undefined' || value === null) { - return ''; - } - - value = amMoment.preprocessDate(value, preprocess); - var date = moment(value); - if (!date.isValid()) { + function amCalendarFilter(value, referenceTime, formats) { + if (isUndefinedOrNull(value)) { return ''; } - return amMoment.applyTimezone(date, timezone).calendar(); + var date = amMoment.preprocessDate(value); + return date.isValid() ? date.calendar(referenceTime, formats) : ''; } // Since AngularJS 1.3, filters have to explicitly define being stateful @@ -468,29 +541,19 @@ * @module angularMoment */ .filter('amDifference', ['moment', 'amMoment', 'angularMomentConfig', function (moment, amMoment, angularMomentConfig) { - function amDifferenceFilter(value, otherValue, unit, usePrecision, preprocessValue, preprocessOtherValue) { - if (typeof value === 'undefined' || value === null) { + function amDifferenceFilter(value, otherValue, unit, usePrecision) { + if (isUndefinedOrNull(value)) { return ''; } - value = amMoment.preprocessDate(value, preprocessValue); - var date = moment(value); - if (!date.isValid()) { - return ''; - } + var date = amMoment.preprocessDate(value); + var date2 = !isUndefinedOrNull(otherValue) ? amMoment.preprocessDate(otherValue) : moment(); - var date2; - if (typeof otherValue === 'undefined' || otherValue === null) { - date2 = moment(); - } else { - otherValue = amMoment.preprocessDate(otherValue, preprocessOtherValue); - date2 = moment(otherValue); - if (!date2.isValid()) { - return ''; - } + if (!date.isValid() || !date2.isValid()) { + return ''; } - return amMoment.applyTimezone(date).diff(amMoment.applyTimezone(date2), unit, usePrecision); + return date.diff(date2, unit, usePrecision); } amDifferenceFilter.$stateful = angularMomentConfig.statefulFilters; @@ -505,19 +568,17 @@ * @function */ .filter('amDateFormat', ['moment', 'amMoment', 'angularMomentConfig', function (moment, amMoment, angularMomentConfig) { - function amDateFormatFilter(value, format, preprocess, timezone, inputFormat) { - var currentFormat = inputFormat || angularMomentConfig.format; - if (typeof value === 'undefined' || value === null) { + function amDateFormatFilter(value, format) { + if (isUndefinedOrNull(value)) { return ''; } - value = amMoment.preprocessDate(value, preprocess, currentFormat); - var date = moment(value); + var date = amMoment.preprocessDate(value); if (!date.isValid()) { return ''; } - return amMoment.applyTimezone(date, timezone).format(format); + return date.format(format); } amDateFormatFilter.$stateful = angularMomentConfig.statefulFilters; @@ -533,7 +594,7 @@ */ .filter('amDurationFormat', ['moment', 'angularMomentConfig', function (moment, angularMomentConfig) { function amDurationFormatFilter(value, format, suffix) { - if (typeof value === 'undefined' || value === null) { + if (isUndefinedOrNull(value)) { return ''; } @@ -552,25 +613,25 @@ * @function */ .filter('amTimeAgo', ['moment', 'amMoment', 'angularMomentConfig', function (moment, amMoment, angularMomentConfig) { - function amTimeAgoFilter(value, preprocess, suffix, from) { + function amTimeAgoFilter(value, suffix, from) { var date, dateFrom; - if (typeof value === 'undefined' || value === null) { + if (isUndefinedOrNull(value)) { return ''; } - value = amMoment.preprocessDate(value, preprocess); + value = amMoment.preprocessDate(value); date = moment(value); if (!date.isValid()) { return ''; } dateFrom = moment(from); - if (typeof from !== 'undefined' && dateFrom.isValid()) { - return amMoment.applyTimezone(date).from(dateFrom, suffix); + if (!isUndefinedOrNull(from) && dateFrom.isValid()) { + return date.from(dateFrom, suffix); } - return amMoment.applyTimezone(date).fromNow(suffix); + return date.fromNow(suffix); } amTimeAgoFilter.$stateful = angularMomentConfig.statefulFilters; @@ -587,7 +648,7 @@ .filter('amSubtract', ['moment', 'angularMomentConfig', function (moment, angularMomentConfig) { function amSubtractFilter(value, amount, type) { - if (typeof value === 'undefined' || value === null) { + if (isUndefinedOrNull(value)) { return ''; } @@ -608,7 +669,7 @@ .filter('amAdd', ['moment', 'angularMomentConfig', function (moment, angularMomentConfig) { function amAddFilter(value, amount, type) { - if (typeof value === 'undefined' || value === null) { + if (isUndefinedOrNull(value)) { return ''; } @@ -618,15 +679,59 @@ amAddFilter.$stateful = angularMomentConfig.statefulFilters; return amAddFilter; - }]); + }]) + + /** + * @ngdoc filter + * @name angularMoment.filter:amStartOf + * @module angularMoment + * @function + */ + .filter('amStartOf', ['moment', 'angularMomentConfig', function (moment, angularMomentConfig) { + function amStartOfFilter(value, type) { + + if (isUndefinedOrNull(value)) { + return ''; + } + + return moment(value).startOf(type); + } + + amStartOfFilter.$stateful = angularMomentConfig.statefulFilters; + + return amStartOfFilter; + }]) + + /** + * @ngdoc filter + * @name angularMoment.filter:amEndOf + * @module angularMoment + * @function + */ + .filter('amEndOf', ['moment', 'angularMomentConfig', function (moment, angularMomentConfig) { + function amEndOfFilter(value, type) { + + if (isUndefinedOrNull(value)) { + return ''; + } + + return moment(value).endOf(type); + } + + amEndOfFilter.$stateful = angularMomentConfig.statefulFilters; + + return amEndOfFilter; + }]); + + return 'angularMoment'; } + var isElectron = window && window.process && window.process.type; if (typeof define === 'function' && define.amd) { define(['angular', 'moment'], angularMoment); - } else if (typeof module !== 'undefined' && module && module.exports) { - angularMoment(angular, require('moment')); - module.exports = 'angularMoment'; + } else if (typeof module !== 'undefined' && module && module.exports && (typeof require === 'function') && !isElectron) { + module.exports = angularMoment(require('angular'), require('moment')); } else { - angularMoment(angular, (typeof global !== 'undefined' ? global : window).moment); + angularMoment(angular, (typeof global !== 'undefined' && typeof global.moment !== 'undefined' ? global : window).moment); } })(); diff --git a/server/www/scripts/dashboard/partials/activityFeed.html b/server/www/scripts/dashboard/partials/activityFeed.html index 4553ce93dbb..57c545a264b 100644 --- a/server/www/scripts/dashboard/partials/activityFeed.html +++ b/server/www/scripts/dashboard/partials/activityFeed.html @@ -33,7 +33,7 @@

    Activity Feed & {{cmd.vulnerabilities_count}} {{cmd.vulnerabilities_count == 1 ? 'vulnerability' : 'vulnerabilities'}} - {{cmd.criticalIssue}} {{cmd.criticalIssue == 1 ? 'is' : 'are'}} rated as Critical. - +

    diff --git a/server/www/scripts/dashboard/partials/commands-list.html b/server/www/scripts/dashboard/partials/commands-list.html index 38cd1b756e4..e4851e53bd3 100644 --- a/server/www/scripts/dashboard/partials/commands-list.html +++ b/server/www/scripts/dashboard/partials/commands-list.html @@ -26,7 +26,7 @@

    Commands History

    diff --git a/server/www/scripts/dashboard/partials/last-vulns.html b/server/www/scripts/dashboard/partials/last-vulns.html index d6b695925cd..2e3d13c1a49 100644 --- a/server/www/scripts/dashboard/partials/last-vulns.html +++ b/server/www/scripts/dashboard/partials/last-vulns.html @@ -32,7 +32,7 @@

    Last Vulnerabilities -

    +
    SeverityTargetNameWebDateSeverityTargetNameWebDate
    Import {{(cmd.tool == 'unknown' ? cmd.command : cmd.tool) + ': ' + cmd.params}} {{cmd.command + ' ' + cmd.params}} - +
    diff --git a/server/www/scripts/dashboard/partials/workspace-progress.html b/server/www/scripts/dashboard/partials/workspace-progress.html index bdd04508711..4535344058a 100644 --- a/server/www/scripts/dashboard/partials/workspace-progress.html +++ b/server/www/scripts/dashboard/partials/workspace-progress.html @@ -9,8 +9,8 @@

    {{wsProgress}}%
      -
    • Start date: {{wsStart | date:"MM/dd/yyyy"}}
      • -
    • End date: {{wsEnd | date:"MM/dd/yyyy"}}
      • +
    • Start date: {{wsStart | amUtc | amDateFormat:'MM/DD/YYYY'}}
      • +
    • End date: {{wsEnd | amUtc | amDateFormat:'MM/DD/YYYY'}}
    diff --git a/server/www/scripts/hosts/partials/list.html b/server/www/scripts/hosts/partials/list.html index 12bb5b3bac5..78228d7d795 100644 --- a/server/www/scripts/hosts/partials/list.html +++ b/server/www/scripts/hosts/partials/list.html @@ -167,10 +167,10 @@

    Add columns

    - + - + diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/datecolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/datecolumn.html index 5dbb32839aa..1a5ab831bca 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/datecolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/datecolumn.html @@ -2,4 +2,4 @@
    {{row.entity.metadata.create_time | date:"MM/dd/yyyy"}}
    -
    {{row.entity.metadata.create_time.split(" ")[0] | date:"MM/dd/yyyy"}}
    +
    {{row.entity.metadata.create_time | amUtc | amDateFormat:'MM/DD/YYYY'}}
    diff --git a/server/www/scripts/workspaces/partials/list.html b/server/www/scripts/workspaces/partials/list.html index 35507b3d754..cfa3ca7c679 100644 --- a/server/www/scripts/workspaces/partials/list.html +++ b/server/www/scripts/workspaces/partials/list.html @@ -60,8 +60,8 @@

    {{ws.name}} - - + + {{objects[ws.name]['total_vulns']}} {{objects[ws.name]['hosts']}} diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 6097628caf4..4794726d40c 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -17,7 +17,7 @@ import pytest import time -from hypothesis import given, assume, settings, strategies as st +from hypothesis import given, strategies as st from server.api.modules.vulns import ( TypeFilter, From 8c27ef577f70697d507e32e40bea05d0118b7a0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 14 Mar 2018 17:26:50 -0300 Subject: [PATCH 1011/1506] Fix hardcoded command duration --- server/api/modules/commandsrun.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/api/modules/commandsrun.py b/server/api/modules/commandsrun.py index a5282ea6341..e48861b130e 100644 --- a/server/api/modules/commandsrun.py +++ b/server/api/modules/commandsrun.py @@ -47,7 +47,7 @@ def post_load_set_end_date_with_duration(self, data): # there is a potential bug when updating, the start_date can be changed. duration = data.pop('duration', None) if duration: - data['end_date'] = data['start_date'] + datetime.timedelta(seconds=120) + data['end_date'] = data['start_date'] + datetime.timedelta(seconds=duration) class Meta: model = Command From 1546eb3fe35fb09abf0c4907e76f27533e1dbbe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 14 Mar 2018 17:35:29 -0300 Subject: [PATCH 1012/1506] Fix Command creation on gtk Correctly set the command field --- plugins/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/controller.py b/plugins/controller.py index 1b51799a522..8efd9cd8408 100644 --- a/plugins/controller.py +++ b/plugins/controller.py @@ -275,7 +275,7 @@ def processReport(self, plugin, filepath, ws_name=None): **{'workspace': ws_name, 'itime': time.time(), 'import_source': 'report', - 'command': 'Import %s:' % plugin, + 'command': plugin, 'params': filepath}) self._mapper_manager.createMappers(ws_name) cmd_info.setID(self._mapper_manager.save(cmd_info)) From 0e9261739bddd630f7c9862badb6732f47d98b15 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 14 Mar 2018 17:58:50 -0300 Subject: [PATCH 1013/1506] Convert create time from utc --- server/www/scripts/services/partials/list.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/services/partials/list.html b/server/www/scripts/services/partials/list.html index 4b8954589f8..808bb0070ca 100644 --- a/server/www/scripts/services/partials/list.html +++ b/server/www/scripts/services/partials/list.html @@ -217,7 +217,7 @@
    Description

    Created - +

    From 781eea0dab2a74c2360dd7af9ef7cbfeaf0c89f5 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 14 Mar 2018 18:00:18 -0300 Subject: [PATCH 1014/1506] Add hypothesis for license api --- test_cases/test_api_license.py | 37 +++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/test_cases/test_api_license.py b/test_cases/test_api_license.py index 15caef520b1..8b463dded6d 100644 --- a/test_cases/test_api_license.py +++ b/test_cases/test_api_license.py @@ -2,6 +2,8 @@ """Tests for many API endpoints that do not depend on workspace_name""" import pytest +from hypothesis import given, strategies as st + from test_cases import factories from test_api_non_workspaced_base import ReadWriteAPITests, API_PREFIX from server.models import ( @@ -41,4 +43,37 @@ def test_license_note_was_missing(self, test_client, session): session.commit() res = test_client.get(self.url(obj=lic)) assert res.status_code == 200 - assert res.json['notes'] == 'A great note. License' \ No newline at end of file + assert res.json['notes'] == 'A great note. License' + + +def license_json(): + return st.fixed_dictionaries( + { + "lictype": st.one_of(st.none(), st.text()), + "metadata": st.fixed_dictionaries({ + "update_time": st.floats(), + "update_user": st.one_of(st.none(), st.text()), + "update_action": st.integers(), + "creator": st.one_of(st.none(), st.text()), + "create_time": st.floats(), + "update_controller_action": st.one_of(st.none(), st.text()), + "owner": st.one_of(st.none(), st.text())}), + "notes": st.one_of(st.none(), st.text()), + "product": st.one_of(st.none(), st.text()), + "start": st.datetimes(), + "end": st.datetimes(), + "type": st.one_of(st.none(), st.text()) + }) + + +@pytest.mark.usefixtures('logged_user') +@pytest.mark.hypothesis +def test_hypothesis_license(test_client): + LicenseData = license_json() + + @given(LicenseData) + def send_api_request(raw_data): + res = test_client.post('_api/v2/licenses/', data=raw_data) + assert res.status_code in [201, 400, 409] + + send_api_request() \ No newline at end of file From ab05fff668a07dbef53022ab7696b3fc446ee553 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 14 Mar 2018 18:38:17 -0300 Subject: [PATCH 1015/1506] Update tests since time formats are now ISO8601 --- test_cases/test_api_hosts.py | 7 +++++-- test_cases/test_api_vulnerability.py | 5 +++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index 674c8872aef..ddeeccc65f1 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -1,5 +1,8 @@ import time import operator + +import pytz + try: import urlparse from urllib import urlencode @@ -460,12 +463,12 @@ def test_update_host(self, test_client, session): u'mac': '', u'metadata': { u'command_id': None, - u'create_time': int(time.mktime(updated_host.create_date.timetuple())) * 1000, + u'create_time': pytz.UTC.localize(updated_host.create_date).isoformat(), u'creator': u'', u'owner': host.creator.username, u'update_action': 0, u'update_controller_action': u'', - u'update_time': int(time.mktime(updated_host.update_date.timetuple())) * 1000, + u'update_time': pytz.UTC.localize(updated_host.update_date).isoformat(), u'update_user': u''}, u'name': u'10.31.112.21', u'os': u'Microsoft Windows Server 2008 R2 Standard Service Pack 1', diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 4794726d40c..aa16e38db78 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -5,6 +5,7 @@ import os from base64 import b64encode +import pytz from hypothesis import given from hypothesis.strategies import text, lists, integers, one_of, none @@ -1117,12 +1118,12 @@ def test_vulnerability_metadata(self, session, test_client, workspace): assert 'metadata' in from_json_vuln[0]['value'] expected_metadata = { u'command_id': command.id, - u'create_time': int(time.mktime(vuln.create_date.timetuple()) * 1000), + u'create_time': pytz.UTC.localize(vuln.create_date).isoformat(), u'creator': command.tool, u'owner': owner.username, u'update_action': 0, u'update_controller_action': u'', - u'update_time': int(time.mktime(vuln.update_date.timetuple()) * 1000), + u'update_time': pytz.UTC.localize(vuln.update_date).isoformat(), u'update_user': u'' } From 1918e520f7c71c2b000717c8b6a940532a12da6a Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 14 Mar 2018 19:13:33 -0300 Subject: [PATCH 1016/1506] Add cascade for vuln template. keep references --- server/models.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/server/models.py b/server/models.py index 0799e978aaa..943d4153c7d 100644 --- a/server/models.py +++ b/server/models.py @@ -532,7 +532,8 @@ class VulnerabilityTemplate(VulnerabilityABC): ) references = association_proxy( - 'reference_template_instances', 'name', + 'reference_template_instances', + 'name', proxy_factory=CustomAssociationSet, creator=_build_associationproxy_creator_non_workspaced('ReferenceTemplate') ) @@ -545,7 +546,8 @@ class VulnerabilityTemplate(VulnerabilityABC): ) policy_violations = association_proxy( - 'policy_violation_template_instances', 'name', + 'policy_violation_template_instances', + 'name', proxy_factory=CustomAssociationSet, creator=_build_associationproxy_creator_non_workspaced('PolicyViolationTemplate') ) @@ -975,8 +977,17 @@ class ReferenceTemplateVulnerabilityAssociation(db.Model): vulnerability_id = Column(Integer, ForeignKey('vulnerability_template.id'), primary_key=True) reference_id = Column(Integer, ForeignKey('reference_template.id'), primary_key=True) - reference = relationship("ReferenceTemplate", backref="reference_template_associations", foreign_keys=[reference_id]) - vulnerability = relationship("VulnerabilityTemplate", backref="reference_template_vulnerability_associations", foreign_keys=[vulnerability_id]) + reference = relationship( + "ReferenceTemplate", + foreign_keys=[reference_id], + backref=backref('reference_template_associations', cascade="all, delete-orphan") + ) + vulnerability = relationship( + "VulnerabilityTemplate", + backref=backref('reference_template_vulnerability_associations', + cascade="all, delete-orphan"), + foreign_keys=[vulnerability_id] + ) class PolicyViolationTemplateVulnerabilityAssociation(db.Model): From 70374be8f2397bc32ac64508a4dda617dbdaaee2 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 14 Mar 2018 19:45:58 -0300 Subject: [PATCH 1017/1506] Serialize dates --- test_cases/test_api_license.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test_cases/test_api_license.py b/test_cases/test_api_license.py index 8b463dded6d..5b7a2974daa 100644 --- a/test_cases/test_api_license.py +++ b/test_cases/test_api_license.py @@ -2,6 +2,7 @@ """Tests for many API endpoints that do not depend on workspace_name""" import pytest +import pytz from hypothesis import given, strategies as st from test_cases import factories @@ -73,6 +74,8 @@ def test_hypothesis_license(test_client): @given(LicenseData) def send_api_request(raw_data): + raw_data['start'] = pytz.UTC.localize(raw_data['start']).isoformat() + raw_data['end'] = pytz.UTC.localize(raw_data['end']).isoformat() res = test_client.post('_api/v2/licenses/', data=raw_data) assert res.status_code in [201, 400, 409] From d82b7234176d0c16441b8b918f52e294c786d4a2 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 15 Mar 2018 12:45:27 -0300 Subject: [PATCH 1018/1506] Update requirements to allow individual server install --- requirements_dev.txt | 1 + requirements_server.txt | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 10e96477513..0bfa4f577a7 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -3,3 +3,4 @@ pytest pytest-factoryboy responses hypothesis==3.48.0 +beautifulsoup4==4.6.0 \ No newline at end of file diff --git a/requirements_server.txt b/requirements_server.txt index fa7a0219c46..273050b796a 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -1,9 +1,12 @@ -bcrypt +autobahn==17.10.1 +bcrypt==3.1.4 couchdbkit==0.6.5 +colorama==0.3.9 Flask-SQLAlchemy==2.3.1 flask-classful==0.14 Flask-Security==3.0.0 flask==0.12.2 +IPy==0.83 marshmallow Pillow==4.2.1 psycopg2==2.7.3.2 @@ -25,3 +28,4 @@ filedepot==0.5.0 nplusone==0.8.1 deprecation==1.0.1 pgcli==1.8.2 +websocket-client==0.46.0 From b8d5f3677688a408928bb18ae9fb3a1f1d48749f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 15 Mar 2018 12:45:36 -0300 Subject: [PATCH 1019/1506] Fix broken test --- test_cases/test_api_license.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test_cases/test_api_license.py b/test_cases/test_api_license.py index 5b7a2974daa..e979d2f1eeb 100644 --- a/test_cases/test_api_license.py +++ b/test_cases/test_api_license.py @@ -69,14 +69,15 @@ def license_json(): @pytest.mark.usefixtures('logged_user') @pytest.mark.hypothesis -def test_hypothesis_license(test_client): +def test_hypothesis_license(test_client, session): + session.commit() LicenseData = license_json() @given(LicenseData) def send_api_request(raw_data): raw_data['start'] = pytz.UTC.localize(raw_data['start']).isoformat() raw_data['end'] = pytz.UTC.localize(raw_data['end']).isoformat() - res = test_client.post('_api/v2/licenses/', data=raw_data) + res = test_client.post('v2/licenses/', data=raw_data) assert res.status_code in [201, 400, 409] - send_api_request() \ No newline at end of file + send_api_request() From e5eaf898f4a6c606aa5ef2bc08cdf6b33dbd1727 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 15 Mar 2018 12:54:51 -0300 Subject: [PATCH 1020/1506] Add test to verify that reference is not deleted when a template was deleted --- test_cases/test_api_vulnerability_template.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test_cases/test_api_vulnerability_template.py b/test_cases/test_api_vulnerability_template.py index cd73d704e1a..1e37c3b784c 100644 --- a/test_cases/test_api_vulnerability_template.py +++ b/test_cases/test_api_vulnerability_template.py @@ -10,7 +10,7 @@ ) from server.models import ( VulnerabilityTemplate, -) + ReferenceTemplate) from test_cases.factories import VulnerabilityTemplateFactory, ReferenceTemplateFactory CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) @@ -155,4 +155,16 @@ def test_create_same_vuln_template_multipe_times_sohuld_raise_409(self, test_cli res = test_client.post(self.url(), data=raw_data) assert res.status_code == 201 res = test_client.post(self.url(), data=raw_data) - assert res.status_code == 409 \ No newline at end of file + assert res.status_code == 409 + + def test_when_a_template_with_ref_is_deleted_ref_remains_at_database( + self, session, test_client): + session.query(ReferenceTemplate).count() == 0 + template = VulnerabilityTemplateFactory.create() + ref1 = ReferenceTemplateFactory.create() + template.reference_template_instances.add(ref1) + session.commit() + res = test_client.delete(self.url(template)) + assert res.status_code == 204 + + assert session.query(ReferenceTemplate).count() == 1 \ No newline at end of file From 56eabaf0019892623c44093aeb591569d51a24e2 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 15 Mar 2018 12:55:09 -0300 Subject: [PATCH 1021/1506] Delete associated files of the obejct deleted --- server/events.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/server/events.py b/server/events.py index 9fc88909140..9a2e5d1579c 100644 --- a/server/events.py +++ b/server/events.py @@ -4,7 +4,10 @@ from models import ( Host, Service, - TagObject, Comment) + TagObject, + Comment, + File, +) from sqlalchemy import event from server.models import db @@ -47,8 +50,12 @@ def delete_object_event(mapper, connection, instance): object_type=msg['type'].lower(), ).delete() db.session.query(Comment).filter_by( - object_id = instance.id, - object_type = msg['type'].lower(), + object_id=instance.id, + object_type=msg['type'].lower(), + ).delete() + db.session.query(File).filter_by( + object_id=instance.id, + object_type=msg['type'].lower(), ).delete() changes_queue.put(msg) From 71d578b385e183631d9d6736756f56b4b0acfdd6 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 15 Mar 2018 14:16:46 -0300 Subject: [PATCH 1022/1506] remove duplicate buttons. add space between cancel and ok --- server/www/scripts/vulndb/partials/upload.html | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/server/www/scripts/vulndb/partials/upload.html b/server/www/scripts/vulndb/partials/upload.html index 48aab1e160b..92435fbe06e 100644 --- a/server/www/scripts/vulndb/partials/upload.html +++ b/server/www/scripts/vulndb/partials/upload.html @@ -4,10 +4,6 @@
    -
    - - -

    Import CSV

    @@ -21,7 +17,7 @@

    Import CSV

    -
    +
    From c4dafa18797c9e063ff0ec67c1d1b9397371850b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 15 Mar 2018 14:58:21 -0300 Subject: [PATCH 1023/1506] Disable ok button on import csv when no file was selected --- server/www/index.html | 1 + server/www/scripts/commons/directives/file.js | 26 +++++++++++++++++++ .../www/scripts/vulndb/partials/upload.html | 6 ++--- 3 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 server/www/scripts/commons/directives/file.js diff --git a/server/www/index.html b/server/www/index.html index 2fcfdecfc82..abcafbed6c3 100644 --- a/server/www/index.html +++ b/server/www/index.html @@ -96,6 +96,7 @@ + diff --git a/server/www/scripts/commons/directives/file.js b/server/www/scripts/commons/directives/file.js new file mode 100644 index 00000000000..4e358413683 --- /dev/null +++ b/server/www/scripts/commons/directives/file.js @@ -0,0 +1,26 @@ +// Faraday Penetration Test IDE +// Copyright (C) 2018 Infobyte LLC (http://www.infobytesec.com/) +// See the file 'doc/LICENSE' for the license information + +/** + * @see http://docs.angularjs.org/guide/concepts + * @see http://docs.angularjs.org/api/ng.directive:ngModel.NgModelController + * @see https://github.com/angular/angular.js/issues/528#issuecomment-7573166 + */ + + +angular.module('faradayApp') + .directive('validFile',function(){ + return { + require:'ngModel', + link:function(scope,el,attrs,ngModel){ + //change event is fired when file is selected + el.bind('change',function(){ + scope.$apply(function(){ + ngModel.$setViewValue(el.val()); + ngModel.$render(); + }); + }); + } + } +}); diff --git a/server/www/scripts/vulndb/partials/upload.html b/server/www/scripts/vulndb/partials/upload.html index 92435fbe06e..b6c7756eb81 100644 --- a/server/www/scripts/vulndb/partials/upload.html +++ b/server/www/scripts/vulndb/partials/upload.html @@ -2,7 +2,7 @@ - +

    Import CSV

    @@ -14,11 +14,11 @@

    Import CSV

    All of these are optional except for the name column.

    Also keep in mind there we ship with some useful CSV which you can find at $FARADAY/data/*.csv

    - +
    - +
    From bd3701fbf3a7eb86b1c89e885a52409ee6b411b4 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 15 Mar 2018 15:25:00 -0300 Subject: [PATCH 1024/1506] Add new field type to filter nul string (0x00) --- server/api/modules/licenses.py | 4 ++-- server/schemas.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/server/api/modules/licenses.py b/server/api/modules/licenses.py index 38965a48d4f..5017489c946 100644 --- a/server/api/modules/licenses.py +++ b/server/api/modules/licenses.py @@ -11,7 +11,7 @@ ReadWriteView, AutoSchema, ) -from server.schemas import StrictDateTimeField +from server.schemas import StrictDateTimeField, FilteredString license_api = Blueprint('license_api', __name__) @@ -20,7 +20,7 @@ class LicenseSchema(AutoSchema): _id = fields.Integer(dump_only=True, attribute='id') end = StrictDateTimeField(load_as_tz_aware=False, attribute='end_date') start = StrictDateTimeField(load_as_tz_aware=False, attribute='start_date') - lictype = fields.String(attribute='type') + lictype = FilteredString(attribute='type') class Meta: model = License diff --git a/server/schemas.py b/server/schemas.py index b9d4e3faea3..e5b0620c2ae 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -4,7 +4,14 @@ from marshmallow.exceptions import ValidationError from dateutil.tz import tzutc -from server.models import CommandObject, VulnerabilityABC +from server.models import VulnerabilityABC + + +class FilteredString(fields.String): + + def _serialize(self, value, attr, obj): + value = value.replace('\0', '') + return super(FilteredString, self)._serialize(value, attr, obj) class JSTimestampField(fields.Integer): From 0c213522476dcb6e779c6c73febb154efea9e692 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 15 Mar 2018 16:02:36 -0300 Subject: [PATCH 1025/1506] Filter notes and product --- server/api/modules/licenses.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/api/modules/licenses.py b/server/api/modules/licenses.py index 5017489c946..ae8089b4bd2 100644 --- a/server/api/modules/licenses.py +++ b/server/api/modules/licenses.py @@ -21,6 +21,8 @@ class LicenseSchema(AutoSchema): end = StrictDateTimeField(load_as_tz_aware=False, attribute='end_date') start = StrictDateTimeField(load_as_tz_aware=False, attribute='start_date') lictype = FilteredString(attribute='type') + notes = FilteredString(attribute='notes') + product = FilteredString(attribute='product') class Meta: model = License From 0829ca749a4e91b87581cc714d86ab47d00d10c3 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 15 Mar 2018 17:27:29 -0300 Subject: [PATCH 1026/1506] fix a bug when chacking database dialect --- server/utils/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/utils/database.py b/server/utils/database.py index 68535002068..cd60bac71cc 100644 --- a/server/utils/database.py +++ b/server/utils/database.py @@ -308,7 +308,7 @@ def get_conflict_object(session, obj, data, workspace=None): def is_unique_constraint_violation(exception): from server.models import db - if db.engine.dialect != 'postgresql': + if db.engine.dialect.name != 'postgresql': # Not implemened for RDMS other than postgres, we can't live without # this, it is just an extra check return True From ed474f6e51eb04bf0756d16b78efa4d44ac5dd34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 16 Mar 2018 12:36:53 -0300 Subject: [PATCH 1027/1506] Remove flask security role logic We are using flask principal directly instead --- server/app.py | 7 +++--- server/commands/reset_db.py | 3 ++- server/models.py | 45 +++++++++++++------------------------ test_cases/test_server.py | 2 +- 4 files changed, 22 insertions(+), 35 deletions(-) diff --git a/server/app.py b/server/app.py index 283bc28ec40..e3e27414ce4 100644 --- a/server/app.py +++ b/server/app.py @@ -190,9 +190,10 @@ def create_app(db_connection_string=None, testing=None): db.init_app(app) # Setup Flask-Security - app.user_datastore = SQLAlchemyUserDatastore(db, - server.models.User, - server.models.Role) + app.user_datastore = SQLAlchemyUserDatastore( + db, + user_model=server.models.User, + role_model=None) # We won't use flask security roles feature Security(app, app.user_datastore) # Make API endpoints require a login user by default. Based on # https://stackoverflow.com/questions/13428708/best-way-to-make-flask-logins-login-required-the-default diff --git a/server/commands/reset_db.py b/server/commands/reset_db.py index cc9cc3c56b4..773dc7ec927 100644 --- a/server/commands/reset_db.py +++ b/server/commands/reset_db.py @@ -3,7 +3,8 @@ def reset_db_all(): # It might be required to do a cascade delete to correctly the # vulnerability table - for table in ('vulnerability', 'vulnerability_template', 'comment'): + for table in ('vulnerability', 'vulnerability_template', 'comment', + 'faraday_user'): try: db.engine.execute('DROP TABLE {} CASCADE'.format(table)) except: diff --git a/server/models.py b/server/models.py index c6035d2952d..bb616727ac2 100644 --- a/server/models.py +++ b/server/models.py @@ -1227,22 +1227,11 @@ def get(workspace_name): return db.session.query(Workspace).filter_by(name=workspace_name).first() -class RolesUsers(db.Model): - __tablename__ = 'roles_users' - id = Column(Integer(), primary_key=True) - user_id = Column('user_id', Integer(), ForeignKey('faraday_user.id')) - role_id = Column('role_id', Integer(), ForeignKey('role.id')) - - -class Role(Metadata, RoleMixin): - __tablename__ = 'role' - id = Column(Integer(), primary_key=True) - name = Column(String(80), unique=True) - description = Column(String(255), nullable=True) - - class User(db.Model, UserMixin): + __tablename__ = 'faraday_user' + ROLES = ['admin', 'pentester', 'client'] + id = Column(Integer, primary_key=True) username = Column(String(255), unique=True, nullable=False) password = Column(String(255), nullable=True) @@ -1256,32 +1245,28 @@ class User(db.Model, UserMixin): login_count = Column(Integer) # flask-security active = Column(Boolean(), default=True, nullable=False) # TBI flask-security confirmed_at = Column(DateTime()) - roles = relationship('Role', secondary='roles_users', - backref=backref('users', lazy='dynamic')) + role = Column(Enum(*ROLES, name='user_roles'), + nullable=False, default='client') # TODO: add many to many relationship to add permission to workspace - @property - def role(self): - """ "admin", "pentester", "client" or None """ + def __init__(self, *args, **kwargs): + # added for compatibility with flask security try: - return next(role_name for role_name - in ['admin', 'pentester', 'client'] - if self.has_role(role_name)) - except StopIteration: - return None + kwargs.pop('roles') + except KeyError: + pass + super(User, self).__init__(*args, **kwargs) + + def __repr__(self): + return '<%sUser: %s>' % ('LDAP ' if self.is_ldap else '', + self.username) def get_security_payload(self): return { "username": self.username, - "role": self.role, - "roles": [role.name for role in self.roles], # Deprectated "name": self.email } - def __repr__(self): - return '<%sUser: %s>' % ('LDAP ' if self.is_ldap else '', - self.username) - class File(Metadata): __tablename__ = 'file' diff --git a/test_cases/test_server.py b/test_cases/test_server.py index f143cf0a169..b55da585d6c 100644 --- a/test_cases/test_server.py +++ b/test_cases/test_server.py @@ -5,7 +5,7 @@ sys.path.append(os.path.abspath(os.getcwd())) -from server.models import db, User, Role +from server.models import db def endpoint(): From 15c69b44e902bdb351580a5c4aeff157f4445950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 16 Mar 2018 14:36:12 -0300 Subject: [PATCH 1028/1506] Improve logged_user fixture Send identity_changed to flask principal --- test_cases/conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 77b8e2adede..8f3c2d5c4b9 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -7,6 +7,7 @@ import pytest from factory import Factory from flask.testing import FlaskClient +from flask_principal import Identity, identity_changed from sqlalchemy import event from pytest_factoryboy import register @@ -241,8 +242,11 @@ def login_as(test_client, user): with test_client.session_transaction() as sess: # Without this line the test breaks. Taken from # http://pythonhosted.org/Flask-Testing/#testing-with-sqlalchemy + assert user.id is not None db.session.add(user) sess['user_id'] = user.id + identity_changed.send(test_client.application, + identity=Identity(user.id)) @pytest.fixture From 4c4fade21de2efeba80cb3dd62a9136c0b2f3391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 16 Mar 2018 14:38:48 -0300 Subject: [PATCH 1029/1506] Change workspace.public default to False The field won't be used in community version, so we don't care about its value. For commercial version it will be better to explicitly set it to True to make it public --- server/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index bb616727ac2..96154ef15fe 100644 --- a/server/models.py +++ b/server/models.py @@ -1132,7 +1132,7 @@ class Workspace(Metadata): active = Column(Boolean(), nullable=False, default=True) # TBI end_date = Column(DateTime(), nullable=True) name = Column(String(250), nullable=False, unique=True) - public = Column(Boolean(), nullable=False, default=True) # TBI + public = Column(Boolean(), nullable=False, default=False) # TBI start_date = Column(DateTime(), nullable=True) credential_count = _make_generic_count_property('workspace', 'credential') From 26b9d7e672f23dd7bb5dab78545fac6c95184b47 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 16 Mar 2018 15:42:31 -0300 Subject: [PATCH 1030/1506] Fix tornado 5.0 bug with IOLoop.instance() --- apis/rest/api.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apis/rest/api.py b/apis/rest/api.py index af0e86c39cb..4b1b1c273d5 100644 --- a/apis/rest/api.py +++ b/apis/rest/api.py @@ -25,28 +25,31 @@ _plugin_controller_api = None _http_server = None - - +ioloop_instance = None def startServer(): global _http_server + global ioloop_instance if _http_server is not None: - IOLoop.instance().start() + ioloop_instance.start() def stopServer(): global _http_server + global ioloop_instance if _http_server is not None: - IOLoop.instance().stop() + ioloop_instance.stop() _http_server.stop() def startAPIs(plugin_controller, model_controller, hostname, port): global _rest_controllers global _http_server + global ioloop_instance _rest_controllers = [PluginControllerAPI(plugin_controller), ModelControllerAPI(model_controller)] app = Flask('APISController') + ioloop_instance = IOLoop.current() _http_server = HTTPServer(WSGIContainer(app)) while True: try: @@ -76,9 +79,6 @@ def startAPIs(plugin_controller, model_controller, hostname, port): logging.getLogger("tornado.access").propagate = False threading.Thread(target=startServer).start() -def stopAPIs(): - stopServer() - class RESTApi(object): """ Abstract class for REST Controllers From 0b686b894c1ecdf19ecc39e012dc7c79b79f1d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 19 Mar 2018 16:02:04 -0300 Subject: [PATCH 1031/1506] Add workspace permission model --- server/models.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/models.py b/server/models.py index 96154ef15fe..72eb97d0455 100644 --- a/server/models.py +++ b/server/models.py @@ -1219,6 +1219,18 @@ def parent(self): return +class WorkspacePermission(db.Model): + __tablename__ = "workspace_permission_association" + id = Column(Integer, primary_key=True) + workspace_id = Column( + Integer, ForeignKey('workspace.id'), nullable=False) + workspace = relationship('Workspace') + + user_id = Column(Integer, ForeignKey('faraday_user.id'), nullable=False) + user = relationship('User', + foreign_keys=[user_id]) + + def is_valid_workspace(workspace_name): return db.session.query(server.models.Workspace).filter_by(name=workspace_name).first() is not None From 7a2d8d6cd90d2d9f1e987d620fa1ea949137ade9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 19 Mar 2018 16:37:43 -0300 Subject: [PATCH 1032/1506] Add count tests structure --- test_cases/test_api_workspaced_base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test_cases/test_api_workspaced_base.py b/test_cases/test_api_workspaced_base.py index b483cc154ae..470b44f4295 100644 --- a/test_cases/test_api_workspaced_base.py +++ b/test_cases/test_api_workspaced_base.py @@ -172,6 +172,9 @@ def test_update_cant_change_id(self, test_client): assert res.json['id'] == expected_id +class CountTestsMixin: + pass + class DeleteTestsMixin: @@ -200,6 +203,7 @@ def create_many_objects(self, session, n): class ReadWriteTestsMixin(ListTestsMixin, RetrieveTestsMixin, CreateTestsMixin, + CountTestsMixin, UpdateTestsMixin, DeleteTestsMixin): pass From 3d1fccaac99f45d69bc6e9d3a1abb2ed32abf031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 19 Mar 2018 18:16:07 -0300 Subject: [PATCH 1033/1506] Fix test that broke with postgres --- test_cases/test_api_workspaced_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_cases/test_api_workspaced_base.py b/test_cases/test_api_workspaced_base.py index b483cc154ae..8f6b42d30f2 100644 --- a/test_cases/test_api_workspaced_base.py +++ b/test_cases/test_api_workspaced_base.py @@ -88,7 +88,7 @@ def test_retrieve_fails_object_of_other_workspcae(self, res = test_client.get(self.url(self.first_object, second_workspace)) assert res.status_code == 404 - @pytest.mark.parametrize('object_id', [123, -1, 'xxx', u'áá']) + @pytest.mark.parametrize('object_id', [12345, -1, 'xxx', u'áá']) def test_404_when_retrieving_unexistent_object(self, test_client, object_id): url = self.url(object_id) From b8eb02d2fdfc469b0b13ac808a49468a1e23006d Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 22 Mar 2018 14:50:02 -0300 Subject: [PATCH 1034/1506] Fix import object types --- server/importer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/importer.py b/server/importer.py index d4b57e2f84a..7f48ec44314 100644 --- a/server/importer.py +++ b/server/importer.py @@ -236,7 +236,7 @@ def create_tags(raw_tags, parent_id, parent_type): session, TagObject, object_id=parent_id, - object_type=parent_type, + object_type=parent_type.lower(), tag_id=tag.id, ) session.commit() @@ -688,7 +688,7 @@ def add_attachments(self, document, vulnerability, workspace): File, filename=attachment_name, object_id=vulnerability.id, - object_type=vulnerability.__class__.__name__) + object_type='vulnerability') file.content = attachment_file.read() attachment_file.close() @@ -721,7 +721,7 @@ def add_references(self, document, vulnerability, workspace): workspace=workspace ) session.flush() - if created and reference not in map(lambda ref: ref.name, references): + if created and reference not in map(lambda ref: ref.name, references): references.add(reference) return references @@ -772,7 +772,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation Comment, text='{0}\n{1}'.format(document.get('text', ''), document.get('description', '')), object_id=couchdb_relational_map[parent_document.get('_id')], - object_type=parent_document['type'], + object_type=parent_document['type'].lower(), workspace=workspace) yield comment @@ -921,7 +921,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation Comment, text=document.get('text'), object_id=workspace.id, - object_type='Workspace', + object_type='workspace', workspace=workspace) yield comment From 911443786374b06fca0d09edd74375ae62bb5c79 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 22 Mar 2018 14:54:27 -0300 Subject: [PATCH 1035/1506] Model updates --- server/models.py | 50 ++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/server/models.py b/server/models.py index c6035d2952d..99a0f1559a2 100644 --- a/server/models.py +++ b/server/models.py @@ -62,6 +62,7 @@ 'service', 'source_code', 'comment', + 'executive_report' ] @@ -367,17 +368,6 @@ class Service(Metadata): @property def parent(self): return self.host - - @property - def summary(self): - if self.version and self.version.lower() != "unknown": - version = " (" + self.version + ")" - else: - version = "" - return "(%s/%s) %s%s" % (self.port, self.protocol, self.name, - version or "") - - class VulnerabilityABC(Metadata): # revisar plugin nexpose, netspark para terminar de definir uniques. asegurar que se carguen bien EASE_OF_RESOLUTIONS = [ @@ -797,8 +787,12 @@ class VulnerabilityGeneric(VulnerabilityABC): __mapper_args__ = { 'polymorphic_on': type } - - + @property + def attachments(self): + return db.session.query(File).filter_by( + object_id=self.id, + object_type='vulnerability' + ) class Vulnerability(VulnerabilityGeneric): __tablename__ = None host_id = Column(Integer, ForeignKey(Host.id), index=True) @@ -1293,9 +1287,7 @@ class File(Metadata): content = Column(UploadedFileField(upload_type=FaradayUploadedFile), nullable=False) # plain attached file object_id = Column(Integer, nullable=False) - object_type = NonBlankColumn(Text) # TODO migration: add enum - - + object_type = Column(Enum(*OBJECT_TYPES, name='object_types'), nullable=False) class UserAvatar(Metadata): __tablename_ = 'user_avatar' @@ -1454,8 +1446,7 @@ class TagObject(db.Model): id = Column(Integer, primary_key=True) object_id = Column(Integer, nullable=False) - object_type = NonBlankColumn(Text) # TODO migration: add enum - + object_type = Column(Enum(*OBJECT_TYPES, name='object_types'), nullable=False) tag = relationship('Tag', backref='tagged_objects') tag_id = Column(Integer, ForeignKey('tag.id'), index=True) @@ -1482,8 +1473,7 @@ class Comment(Metadata): ) object_id = Column(Integer, nullable=False) - object_type = NonBlankColumn(Text) # TODO migration: add enum - + object_type = Column(Enum(*OBJECT_TYPES, name='object_types'), nullable=False) @property def parent(self): return @@ -1500,7 +1490,7 @@ class ExecutiveReport(Metadata): grouped = Column(Boolean, nullable=False, default=False) name = NonBlankColumn(Text, index=True) - status = Column(Enum(*STATUSES, name='executive_report_statuses'), nullable=True) + status = Column(Enum(*STATUSES, name='executive_report_statuses'), nullable=False, default='processing') template_name = NonBlankColumn(Text) conclusions = BlankColumn(Text) @@ -1510,19 +1500,29 @@ class ExecutiveReport(Metadata): scope = BlankColumn(Text) summary = BlankColumn(Text) title = BlankColumn(Text) - + confirmed = Column(Boolean, nullable=False, default=False) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship( 'Workspace', backref=backref('reports', cascade="all, delete-orphan"), foreign_keys=[workspace_id] ) - + tags = relationship( + "Tag", + secondary="tag_object", + primaryjoin="and_(TagObject.object_id==ExecutiveReport.id, " + "TagObject.object_type=='executive_report')", + collection_class=set, + ) @property def parent(self): return - - + @property + def attachments(self): + return db.session.query(File).filter_by( + object_id=self.id, + object_type='executive_report' + ) # This constraint uses Columns from different classes # Since it applies to the table vulnerability it should be adVulnerability.ded to the Vulnerability class # However, since it contains columns from children classes, this cannot be done From 20fdfe924afbcaed0b909d565e90ffff94514f55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 22 Mar 2018 15:26:12 -0300 Subject: [PATCH 1036/1506] Use BlankColumn and NonBlankColumn in String fields --- server/models.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server/models.py b/server/models.py index c6035d2952d..7deb5876a1e 100644 --- a/server/models.py +++ b/server/models.py @@ -660,8 +660,8 @@ class Command(Metadata): tool = NonBlankColumn(Text) start_date = Column(DateTime, nullable=False) end_date = Column(DateTime, nullable=True) - ip = Column(String(250), nullable=False) # where the command was executed - hostname = Column(String(250), nullable=False) # where the command was executed + ip = BlankColumn(String(250)) # where the command was executed + hostname = BlankColumn(String(250)) # where the command was executed params = BlankColumn(Text) user = BlankColumn(String(250)) # os username where the command was executed import_source = Column(Enum(*IMPORT_SOURCE, name='import_source_enum')) @@ -1127,11 +1127,11 @@ def _make_vuln_count_property(type_=None, only_confirmed=False, class Workspace(Metadata): __tablename__ = 'workspace' id = Column(Integer, primary_key=True) - customer = Column(String(250), nullable=True) # TBI + customer = BlankColumn(String(250)) # TBI description = BlankColumn(Text) active = Column(Boolean(), nullable=False, default=True) # TBI end_date = Column(DateTime(), nullable=True) - name = Column(String(250), nullable=False, unique=True) + name = NonBlankColumn(String(250), unique=True) public = Column(Boolean(), nullable=False, default=True) # TBI start_date = Column(DateTime(), nullable=True) @@ -1244,15 +1244,15 @@ class Role(Metadata, RoleMixin): class User(db.Model, UserMixin): __tablename__ = 'faraday_user' id = Column(Integer, primary_key=True) - username = Column(String(255), unique=True, nullable=False) + username = NonBlankColumn(String(255), unique=True) password = Column(String(255), nullable=True) email = Column(String(255), unique=True, nullable=True) # TBI - name = Column(String(255), nullable=True) # TBI + name = BlankColumn(String(255)) # TBI is_ldap = Column(Boolean(), nullable=False, default=False) last_login_at = Column(DateTime()) # flask-security current_login_at = Column(DateTime()) # flask-security - last_login_ip = Column(String(100)) # flask-security - current_login_ip = Column(String(100)) # flask-security + last_login_ip = BlankColumn(String(100)) # flask-security + current_login_ip = BlankColumn(String(100)) # flask-security login_count = Column(Integer) # flask-security active = Column(Boolean(), default=True, nullable=False) # TBI flask-security confirmed_at = Column(DateTime()) From 00b12668ac0affec80774e769e749c1e3e67150c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 22 Mar 2018 16:21:54 -0300 Subject: [PATCH 1037/1506] Fix conflict checking when there are no filters --- server/utils/database.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/utils/database.py b/server/utils/database.py index cd60bac71cc..ed9f0a08646 100644 --- a/server/utils/database.py +++ b/server/utils/database.py @@ -297,10 +297,11 @@ def get_conflict_object(session, obj, data, workspace=None): filter_data.append( table.columns[relations_field] == relation_id) - filter_data = reduce(operator.and_, filter_data) - conflict_obj = session.query(klass).filter( - filter_data).first() - return conflict_obj + if filter_data: + filter_data = reduce(operator.and_, filter_data) + return session.query(klass).filter(filter_data).first() + else: + return UNIQUE_VIOLATION = '23505' From f36afdea489931bd61d2a643ad3d786a9ac35b07 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 22 Mar 2018 17:14:19 -0300 Subject: [PATCH 1038/1506] Importer fixes after update object_type to enum. Improvement on progress bar update --- server/importer.py | 16 +++++++++------- server/models.py | 3 ++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/server/importer.py b/server/importer.py index 7f48ec44314..27cf155c43d 100644 --- a/server/importer.py +++ b/server/importer.py @@ -231,12 +231,13 @@ def create_tags(raw_tags, parent_id, parent_type): for tag_name in [x.strip() for x in raw_tags if x.strip()]: tag, tag_created = get_or_create(session, Tag, name=tag_name, slug=slugify(tag_name)) session.commit() - + parent_type = parent_type.lower() + parent_type = parent_type.replace('web', '') relation, relation_created = get_or_create( session, TagObject, object_id=parent_id, - object_type=parent_type.lower(), + object_type=parent_type, tag_id=tag.id, ) session.commit() @@ -1215,9 +1216,9 @@ def run(self): users_import = ImportCouchDBUsers() users_import.run() - logger.info('Importing workspaces. Using {0} threads'.format(multiprocessing.cpu_count())) + logger.info('Importing workspaces. Using {0} threads'.format(multiprocessing.cpu_count() * 2)) workspace_threads = [] - with tqdm(total=len(workspaces_list), + with tqdm(total=len(workspaces_list) * 18, unit='B', unit_scale=True, unit_divisor=1024) as pbar: for workspace_name in workspaces_list: logger.debug(u'Setting up workspace {}'.format(workspace_name)) @@ -1226,11 +1227,11 @@ def run(self): logger.error(u"Unauthorized access to CouchDB. Make sure faraday-server's"\ " configuration file has CouchDB admin's credentials set") sys.exit(1) - thread = threading.Thread(target=self.import_workspace_into_database, args=(workspace_name, )) + thread = threading.Thread(target=self.import_workspace_into_database, args=(workspace_name, pbar)) thread.daemon = True thread.start() workspace_threads.append(thread) - if len(workspace_threads) > multiprocessing.cpu_count(): + if len(workspace_threads) > multiprocessing.cpu_count() * 2: for thread in workspace_threads: thread.join() pbar.update(1) @@ -1306,7 +1307,7 @@ def verify_import_data(self, couchdb_relational_map, couchdb_relational_map_by_t with open(missing_objs_filename, 'w') as missing_objs_file: missing_objs_file.write(json.dumps(objs_diff)) - def import_workspace_into_database(self, workspace_name): + def import_workspace_into_database(self, workspace_name, pbar): with app.app_context(): faraday_importer = FaradayEntityImporter(workspace_name) @@ -1348,6 +1349,7 @@ def import_workspace_into_database(self, workspace_name): session.commit() couchdb_relational_map_by_type[couchdb_id].append({'type': obj_type, 'id': new_obj.id}) couchdb_relational_map[couchdb_id].append(new_obj.id) + pbar.update(1) update_command_tools(workspace, command_tool_map, couchdb_relational_map_by_type) session.commit() diff --git a/server/models.py b/server/models.py index 99a0f1559a2..ff0d67be6a4 100644 --- a/server/models.py +++ b/server/models.py @@ -62,7 +62,8 @@ 'service', 'source_code', 'comment', - 'executive_report' + 'executive_report', + 'workspace' ] From 865d6892de30d5570b2d57ef8cb94da35724fa53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 23 Mar 2018 15:51:38 -0300 Subject: [PATCH 1039/1506] Fix user deletion foreign key constraint Set creator_id to null when a user is deleted --- server/models.py | 5 ++++- test_cases/models/test_cascades.py | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 test_cases/models/test_cascades.py diff --git a/server/models.py b/server/models.py index 53f8eb37625..5e4795e6250 100644 --- a/server/models.py +++ b/server/models.py @@ -157,7 +157,10 @@ class Metadata(db.Model): @declared_attr def creator_id(cls): - return Column(Integer, ForeignKey('faraday_user.id'), nullable=True) + return Column( + Integer, + ForeignKey('faraday_user.id', ondelete="SET NULL"), + nullable=True) @declared_attr def creator(cls): diff --git a/test_cases/models/test_cascades.py b/test_cases/models/test_cascades.py new file mode 100644 index 00000000000..af89c040a90 --- /dev/null +++ b/test_cases/models/test_cascades.py @@ -0,0 +1,8 @@ + +def test_delete_user(workspace, session): + assert workspace.creator + session.commit() + user = workspace.creator + session.delete(user) + session.commit() + assert workspace.creator is None From b7d313d43c0626b5bdf40bc7c4768f81259de5f8 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 23 Mar 2018 16:55:49 -0300 Subject: [PATCH 1040/1506] Udate factory after adding enum. add deleted by mistake service summary --- server/models.py | 11 +++++++++++ test_cases/factories.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index 53f8eb37625..75e01a8f675 100644 --- a/server/models.py +++ b/server/models.py @@ -369,6 +369,17 @@ class Service(Metadata): @property def parent(self): return self.host + + @property + def summary(self): + if self.version and self.version.lower() != "unknown": + version = " (" + self.version + ")" + else: + version = "" + return "(%s/%s) %s%s" % (self.port, self.protocol, self.name, + version or "") + + class VulnerabilityABC(Metadata): # revisar plugin nexpose, netspark para terminar de definir uniques. asegurar que se carguen bien EASE_OF_RESOLUTIONS = [ diff --git a/test_cases/factories.py b/test_cases/factories.py index e28f92231ac..a5ab5842c82 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -367,7 +367,7 @@ class CommentFactory(WorkspaceObjectFactory): """ text = FuzzyText() object_id = FuzzyInteger(1) - object_type = FuzzyChoice(['Host', 'Service', 'Comment']) + object_type = FuzzyChoice(['host', 'service', 'comment']) class Meta: From e82d98cc738acf2102caac7f7ea5e82782ecb629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 27 Mar 2018 18:07:53 -0300 Subject: [PATCH 1041/1506] Add nix file for building a shell with the server available --- requirements.nix | 670 ++++++++++++++++++++++++++++++++++++++ requirements_override.nix | 19 ++ shell.nix | 24 ++ 3 files changed, 713 insertions(+) create mode 100644 requirements.nix create mode 100644 requirements_override.nix create mode 100644 shell.nix diff --git a/requirements.nix b/requirements.nix new file mode 100644 index 00000000000..9211a07c7c7 --- /dev/null +++ b/requirements.nix @@ -0,0 +1,670 @@ +# generated using pypi2nix tool (version: 1.8.1) +# See more at: https://github.com/garbas/pypi2nix +# +# COMMAND: +# pypi2nix -V 2.7 -r requirements_server.txt -E libffi -E openssl +# + +{ pkgs ? import {} +}: + +let + + inherit (pkgs) makeWrapper; + inherit (pkgs.stdenv.lib) fix' extends inNixShell; + + pythonPackages = + import "${toString pkgs.path}/pkgs/top-level/python-packages.nix" { + inherit pkgs; + inherit (pkgs) stdenv; + python = pkgs.python27Full; + # patching pip so it does not try to remove files when running nix-shell + overrides = + self: super: { + bootstrapped-pip = super.bootstrapped-pip.overrideDerivation (old: { + patchPhase = old.patchPhase + '' + sed -i -e "s|paths_to_remove.remove(auto_confirm)|#paths_to_remove.remove(auto_confirm)|" -e "s|self.uninstalled = paths_to_remove|#self.uninstalled = paths_to_remove|" $out/${pkgs.python35.sitePackages}/pip/req/req_install.py + ''; + }); + }; + }; + + commonBuildInputs = with pkgs; [ libffi openssl ]; + commonDoCheck = false; + + withPackages = pkgs': + let + pkgs = builtins.removeAttrs pkgs' ["__unfix__"]; + interpreter = pythonPackages.buildPythonPackage { + name = "python27Full-interpreter"; + buildInputs = [ makeWrapper ] ++ (builtins.attrValues pkgs); + buildCommand = '' + mkdir -p $out/bin + ln -s ${pythonPackages.python.interpreter} $out/bin/${pythonPackages.python.executable} + for dep in ${builtins.concatStringsSep " " (builtins.attrValues pkgs)}; do + if [ -d "$dep/bin" ]; then + for prog in "$dep/bin/"*; do + if [ -f $prog ]; then + ln -s $prog $out/bin/`basename $prog` + fi + done + fi + done + for prog in "$out/bin/"*; do + wrapProgram "$prog" --prefix PYTHONPATH : "$PYTHONPATH" + done + pushd $out/bin + ln -s ${pythonPackages.python.executable} python + ln -s ${pythonPackages.python.executable} python2 + popd + ''; + passthru.interpreter = pythonPackages.python; + }; + in { + __old = pythonPackages; + inherit interpreter; + mkDerivation = pythonPackages.buildPythonPackage; + packages = pkgs; + overrideDerivation = drv: f: + pythonPackages.buildPythonPackage (drv.drvAttrs // f drv.drvAttrs // { meta = drv.meta; }); + withPackages = pkgs'': + withPackages (pkgs // pkgs''); + }; + + python = withPackages {}; + + generated = self: { + + "Automat" = python.mkDerivation { + name = "Automat-0.6.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/de/05/b8e453085cf8a7f27bb1226596f4ccf5cc9e758377d60284f990bbdc592c/Automat-0.6.0.tar.gz"; sha256 = "3c1fd04ecf08ac87b4dd3feae409542e9bf7827257097b2b6ed5692f69d6f6a8"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Twisted" + self."attrs" + self."six" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/glyph/Automat"; + license = licenses.mit; + description = "Self-service finite-state machines for the programmer on the go."; + }; + }; + + + + "Flask" = python.mkDerivation { + name = "Flask-0.12.2"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/eb/12/1c7bd06fcbd08ba544f25bf2c6612e305a70ea51ca0eda8007344ec3f123/Flask-0.12.2.tar.gz"; sha256 = "49f44461237b69ecd901cc7ce66feea0319b9158743dd27a2899962ab214dac1"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Jinja2" + self."Werkzeug" + self."click" + self."itsdangerous" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://github.com/pallets/flask/"; + license = licenses.bsdOriginal; + description = "A microframework based on Werkzeug, Jinja2 and good intentions"; + }; + }; + + + + "Jinja2" = python.mkDerivation { + name = "Jinja2-2.10"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/56/e6/332789f295cf22308386cf5bbd1f4e00ed11484299c5d7383378cf48ba47/Jinja2-2.10.tar.gz"; sha256 = "f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."MarkupSafe" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://jinja.pocoo.org/"; + license = licenses.bsdOriginal; + description = "A small but fast and easy to use stand-alone template engine written in pure python."; + }; + }; + + + + "MarkupSafe" = python.mkDerivation { + name = "MarkupSafe-1.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/4d/de/32d741db316d8fdb7680822dd37001ef7a448255de9699ab4bfcbdf4172b/MarkupSafe-1.0.tar.gz"; sha256 = "a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://github.com/pallets/markupsafe"; + license = licenses.bsdOriginal; + description = "Implements a XML/HTML/XHTML Markup safe string for Python"; + }; + }; + + + + "SQLAlchemy" = python.mkDerivation { + name = "SQLAlchemy-1.2.5"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/7e/57/435347429f4ff5016b72c9b179c70a6cc67e5f9b4d3f3e9b51362c40b356/SQLAlchemy-1.2.5.tar.gz"; sha256 = "249000654107a420a40200f1e0b555a79dfd4eff235b2ff60bc77714bd045f2d"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://www.sqlalchemy.org"; + license = licenses.mit; + description = "Database Abstraction Library"; + }; + }; + + + + "Twisted" = python.mkDerivation { + name = "Twisted-17.9.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/a2/37/298f9547606c45d75aa9792369302cc63aa4bbcf7b5f607560180dd099d2/Twisted-17.9.0.tar.bz2"; sha256 = "0da1a7e35d5fcae37bc9c7978970b5feb3bc82822155b8654ec63925c05af75c"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Automat" + self."constantly" + self."cryptography" + self."hyperlink" + self."idna" + self."incremental" + self."pyOpenSSL" + self."pyasn1" + self."service-identity" + self."zope.interface" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://twistedmatrix.com/"; + license = licenses.mit; + description = "An asynchronous networking framework written in Python"; + }; + }; + + + + "Werkzeug" = python.mkDerivation { + name = "Werkzeug-0.14.1"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/9f/08/a3bb1c045ec602dc680906fc0261c267bed6b3bb4609430aff92c3888ec8/Werkzeug-0.14.1.tar.gz"; sha256 = "c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://www.palletsprojects.org/p/werkzeug/"; + license = licenses.bsdOriginal; + description = "The comprehensive WSGI web application library."; + }; + }; + + + + "asn1crypto" = python.mkDerivation { + name = "asn1crypto-0.24.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/fc/f1/8db7daa71f414ddabfa056c4ef792e1461ff655c2ae2928a2b675bfed6b4/asn1crypto-0.24.0.tar.gz"; sha256 = "9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/wbond/asn1crypto"; + license = licenses.mit; + description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP"; + }; + }; + + + + "attrs" = python.mkDerivation { + name = "attrs-17.4.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/8b/0b/a06cfcb69d0cb004fde8bc6f0fd192d96d565d1b8aa2829f0f20adb796e5/attrs-17.4.0.tar.gz"; sha256 = "1c7960ccfd6a005cd9f7ba884e6316b5e430a3f1a6c37c5f87d8b43f83b54ec9"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."six" + self."zope.interface" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://www.attrs.org/"; + license = licenses.mit; + description = "Classes Without Boilerplate"; + }; + }; + + + + "certifi" = python.mkDerivation { + name = "certifi-2018.1.18"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/15/d4/2f888fc463d516ff7bf2379a4e9a552fef7f22a94147655d9b1097108248/certifi-2018.1.18.tar.gz"; sha256 = "edbc3f203427eef571f79a7692bb160a2b0f7ccaa31953e99bd17e307cf63f7d"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://certifi.io/"; + license = licenses.mpl20; + description = "Python package for providing Mozilla's CA Bundle."; + }; + }; + + + + "cffi" = python.mkDerivation { + name = "cffi-1.11.5"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/e7/a7/4cd50e57cc6f436f1cc3a7e8fa700ff9b8b4d471620629074913e3735fb2/cffi-1.11.5.tar.gz"; sha256 = "e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."pycparser" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://cffi.readthedocs.org"; + license = licenses.mit; + description = "Foreign Function Interface for Python calling C code."; + }; + }; + + + + "chardet" = python.mkDerivation { + name = "chardet-3.0.4"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d/chardet-3.0.4.tar.gz"; sha256 = "84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/chardet/chardet"; + license = licenses.lgpl2; + description = "Universal encoding detector for Python 2 and 3"; + }; + }; + + + + "click" = python.mkDerivation { + name = "click-6.7"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/95/d9/c3336b6b5711c3ab9d1d3a80f1a3e2afeb9d8c02a7166462f6cc96570897/click-6.7.tar.gz"; sha256 = "f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://github.com/mitsuhiko/click"; + license = licenses.bsdOriginal; + description = "A simple wrapper around optparse for powerful command line utilities."; + }; + }; + + + + "constantly" = python.mkDerivation { + name = "constantly-15.1.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/95/f1/207a0a478c4bb34b1b49d5915e2db574cadc415c9ac3a7ef17e29b2e8951/constantly-15.1.0.tar.gz"; sha256 = "586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/twisted/constantly"; + license = licenses.mit; + description = "Symbolic constants in Python"; + }; + }; + + + + "couchdbkit" = python.mkDerivation { + name = "couchdbkit-0.6.5"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/a1/13/9e9ff695a385c44f62b4766341b97f2bd8b596962df2a0beabf358468b70/couchdbkit-0.6.5.tar.gz"; sha256 = "9b607f509727e6ada2dbd576a4120c214b1c54f3bb8bf6e2e0eb2cfbb11a0e00"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."restkit" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://couchdbkit.org"; + license = "License :: OSI Approved :: Apache Software License"; + description = "Python couchdb kit"; + }; + }; + + + + "cryptography" = python.mkDerivation { + name = "cryptography-2.2.2"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/ec/b2/faa78c1ab928d2b2c634c8b41ff1181f0abdd9adf9193211bd606ffa57e2/cryptography-2.2.2.tar.gz"; sha256 = "9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."asn1crypto" + self."cffi" + self."enum34" + self."idna" + self."ipaddress" + self."six" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/pyca/cryptography"; + license = licenses.bsdOriginal; + description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers."; + }; + }; + + + + "enum34" = python.mkDerivation { + name = "enum34-1.1.6"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz"; sha256 = "8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://bitbucket.org/stoneleaf/enum34"; + license = licenses.bsdOriginal; + description = "Python 3.4 Enum backported to 3.3, 3.2, 3.1, 2.7, 2.6, 2.5, and 2.4"; + }; + }; + + + + "http-parser" = python.mkDerivation { + name = "http-parser-0.8.3"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/07/c4/22e3c76c2313c26dd5f84f1205b916ff38ea951aab0c4544b6e2f5920d64/http-parser-0.8.3.tar.gz"; sha256 = "e2aff90a60def3e476bd71694d8757c0f95ebf2fedf0a8ae34ee306e0b20db83"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://github.com/benoitc/http-parser"; + license = licenses.mit; + description = "http request/response parser"; + }; + }; + + + + "hyperlink" = python.mkDerivation { + name = "hyperlink-18.0.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/41/e1/0abd4b480ec04892b1db714560f8c855d43df81895c98506442babf3652f/hyperlink-18.0.0.tar.gz"; sha256 = "f01b4ff744f14bc5d0a22a6b9f1525ab7d6312cb0ff967f59414bbac52f0a306"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."idna" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/python-hyper/hyperlink"; + license = licenses.mit; + description = "A featureful, immutable, and correct URL for Python."; + }; + }; + + + + "idna" = python.mkDerivation { + name = "idna-2.6"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/f4/bd/0467d62790828c23c47fc1dfa1b1f052b24efdf5290f071c7a91d0d82fd3/idna-2.6.tar.gz"; sha256 = "2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/kjd/idna"; + license = licenses.bsdOriginal; + description = "Internationalized Domain Names in Applications (IDNA)"; + }; + }; + + + + "incremental" = python.mkDerivation { + name = "incremental-17.5.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/8f/26/02c4016aa95f45479eea37c90c34f8fab6775732ae62587a874b619ca097/incremental-17.5.0.tar.gz"; sha256 = "7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Twisted" + self."click" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/twisted/incremental"; + license = licenses.mit; + description = "UNKNOWN"; + }; + }; + + + + "ipaddress" = python.mkDerivation { + name = "ipaddress-1.0.19"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/f0/ba/860a4a3e283456d6b7e2ab39ce5cf11a3490ee1a363652ac50abf9f0f5df/ipaddress-1.0.19.tar.gz"; sha256 = "200d8686011d470b5e4de207d803445deee427455cd0cb7c982b68cf82524f81"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/phihag/ipaddress"; + license = licenses.psfl; + description = "IPv4/IPv6 manipulation library"; + }; + }; + + + + "itsdangerous" = python.mkDerivation { + name = "itsdangerous-0.24"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/dc/b4/a60bcdba945c00f6d608d8975131ab3f25b22f2bcfe1dab221165194b2d4/itsdangerous-0.24.tar.gz"; sha256 = "cbb3fcf8d3e33df861709ecaf89d9e6629cff0a217bc2848f1b41cd30d360519"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://github.com/mitsuhiko/itsdangerous"; + license = licenses.bsdOriginal; + description = "Various helpers to pass trusted data to untrusted environments and back."; + }; + }; + + + + "pyOpenSSL" = python.mkDerivation { + name = "pyOpenSSL-17.5.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/3b/15/a5d90ab1a41075e8f0fae334f13452549528f82142b3b9d0c9d86ab7178c/pyOpenSSL-17.5.0.tar.gz"; sha256 = "2c10cfba46a52c0b0950118981d61e72c1e5b1aac451ca1bc77de1a679456773"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."cryptography" + self."six" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://pyopenssl.org/"; + license = licenses.asl20; + description = "Python wrapper module around the OpenSSL library"; + }; + }; + + + + "pyasn1" = python.mkDerivation { + name = "pyasn1-0.4.2"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/eb/3d/b7d0fdf4a882e26674c68c20f40682491377c4db1439870f5b6f862f76ed/pyasn1-0.4.2.tar.gz"; sha256 = "d258b0a71994f7770599835249cece1caef3c70def868c4915e6e5ca49b67d15"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/etingof/pyasn1"; + license = licenses.bsdOriginal; + description = "ASN.1 types and codecs"; + }; + }; + + + + "pyasn1-modules" = python.mkDerivation { + name = "pyasn1-modules-0.2.1"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/ab/76/36ab0e099e6bd27ed95b70c2c86c326d3affa59b9b535c63a2f892ac9f45/pyasn1-modules-0.2.1.tar.gz"; sha256 = "af00ea8f2022b6287dc375b2c70f31ab5af83989fc6fe9eacd4976ce26cd7ccc"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."pyasn1" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/etingof/pyasn1-modules"; + license = licenses.bsdOriginal; + description = "A collection of ASN.1-based protocols modules."; + }; + }; + + + + "pycparser" = python.mkDerivation { + name = "pycparser-2.18"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/8c/2d/aad7f16146f4197a11f8e91fb81df177adcc2073d36a17b1491fd09df6ed/pycparser-2.18.tar.gz"; sha256 = "99a8ca03e29851d96616ad0404b4aad7d9ee16f25c9f9708a11faf2810f7b226"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/eliben/pycparser"; + license = licenses.bsdOriginal; + description = "C parser in Python"; + }; + }; + + + + "requests" = python.mkDerivation { + name = "requests-2.18.4"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/b0/e1/eab4fc3752e3d240468a8c0b284607899d2fbfb236a56b7377a329aa8d09/requests-2.18.4.tar.gz"; sha256 = "9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."certifi" + self."chardet" + self."cryptography" + self."idna" + self."pyOpenSSL" + self."urllib3" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://python-requests.org"; + license = licenses.asl20; + description = "Python HTTP for Humans."; + }; + }; + + + + "restkit" = python.mkDerivation { + name = "restkit-4.2.2"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/76/b9/d90120add1be718f853c53008cf5b62d74abad1d32bd1e7097dd913ae053/restkit-4.2.2.tar.gz"; sha256 = "c0bda8eb7c643b5e818b612dab49121393abc8589c6cbe9b84085079d598599d"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."http-parser" + self."socketpool" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://benoitc.github.com/restkit"; + license = licenses.mit; + description = "Python REST kit"; + }; + }; + + + + "service-identity" = python.mkDerivation { + name = "service-identity-17.0.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/de/2a/cab6e30be82c8fcd2339ef618036720eda954cf05daef514e386661c9221/service_identity-17.0.0.tar.gz"; sha256 = "4001fbb3da19e0df22c47a06d29681a398473af4aa9d745eca525b3b2c2302ab"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."attrs" + self."idna" + self."pyOpenSSL" + self."pyasn1" + self."pyasn1-modules" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://service-identity.readthedocs.io/"; + license = licenses.mit; + description = "Service identity verification for pyOpenSSL."; + }; + }; + + + + "six" = python.mkDerivation { + name = "six-1.11.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/16/d8/bc6316cf98419719bd59c91742194c111b6f2e85abac88e496adefaf7afe/six-1.11.0.tar.gz"; sha256 = "70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://pypi.python.org/pypi/six/"; + license = licenses.mit; + description = "Python 2 and 3 compatibility utilities"; + }; + }; + + + + "socketpool" = python.mkDerivation { + name = "socketpool-0.5.3"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/d1/39/fae99a735227234ffec389b252c6de2bc7816bf627f56b4c558dc46c85aa/socketpool-0.5.3.tar.gz"; sha256 = "a06733434a56c4b60b8fcaa168102d2386253d36425804d55532a6bbbda6e2ec"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://github.com/benoitc/socketpool"; + license = licenses.mit; + description = "Python socket pool"; + }; + }; + + + + "urllib3" = python.mkDerivation { + name = "urllib3-1.22"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/ee/11/7c59620aceedcc1ef65e156cc5ce5a24ef87be4107c2b74458464e437a5d/urllib3-1.22.tar.gz"; sha256 = "cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."certifi" + self."cryptography" + self."idna" + self."ipaddress" + self."pyOpenSSL" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://urllib3.readthedocs.io/"; + license = licenses.mit; + description = "HTTP library with thread-safe connection pooling, file post, and more."; + }; + }; + + + + "zope.interface" = python.mkDerivation { + name = "zope.interface-4.4.3"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/bd/d2/25349ed41f9dcff7b3baf87bd88a4c82396cf6e02f1f42bb68657a3132af/zope.interface-4.4.3.tar.gz"; sha256 = "d6d26d5dfbfd60c65152938fcb82f949e8dada37c041f72916fef6621ba5c5ce"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/zopefoundation/zope.interface"; + license = licenses.zpl21; + description = "Interfaces for Python"; + }; + }; + + }; + localOverridesFile = ./requirements_override.nix; + overrides = import localOverridesFile { inherit pkgs python; }; + commonOverrides = [ + + ]; + allOverrides = + (if (builtins.pathExists localOverridesFile) + then [overrides] else [] ) ++ commonOverrides; + +in python.withPackages + (fix' (pkgs.lib.fold + extends + generated + allOverrides + ) + ) \ No newline at end of file diff --git a/requirements_override.nix b/requirements_override.nix new file mode 100644 index 00000000000..57addd0a206 --- /dev/null +++ b/requirements_override.nix @@ -0,0 +1,19 @@ +{ pkgs, python }: + +self: super: { + # Twisted = python.overrideDerivation super.Twisted (old: { + # propagatedBuildInputs = (builtins.filter + # (drv: drv.name != self.Automat.name && drv.name != self.incremental.name) + # old.propagatedBuildInputs); + # }); + Automat = python.overrideDerivation super.Automat (old: { + propagatedBuildInputs = (builtins.filter + (drv: drv.name != self.Twisted.name) + old.propagatedBuildInputs); + }); + incremental = python.overrideDerivation super.incremental (old: { + propagatedBuildInputs = (builtins.filter + (drv: drv.name != self.Twisted.name) + old.propagatedBuildInputs); + }); +} diff --git a/shell.nix b/shell.nix new file mode 100644 index 00000000000..b915358d3f1 --- /dev/null +++ b/shell.nix @@ -0,0 +1,24 @@ +with (import {}); +let + faraday_server = stdenv.mkDerivation { + name = "faraday_server"; + src = builtins.filterSource cleanPycs ./.; + buildInputs = [coreutils makeWrapper python_env.interpreter]; + builder = "${bash}/bin/bash"; + args = ["-c" '' + + source $stdenv/setup + mkdir -p $out/{bin,src} + cp -rv ${./.}/* $out/src/ + + makeWrapper \ + ${python_env.interpreter}/bin/python \ + $out/bin/faraday-server \ + --prefix PATH : $out/src \ + --add-flags ${./.}/faraday-server.py + '']; + }; + python_env = (import ./requirements.nix {}); + cleanPycs = path: type: type == "directory" || builtins.match ".*\.pyc" path != null; +in + mkShell { buildInputs = [faraday_server]; } From 9bd8c2e6a39f10f6425cfc64381b8cb8e845890e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 27 Mar 2018 18:57:46 -0300 Subject: [PATCH 1042/1506] Move pid file to ~/.faraday To keep clean the source code directory --- .gitignore | 5 +---- server/config.py | 3 ++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index dfc6eca8658..33df2185266 100644 --- a/.gitignore +++ b/.gitignore @@ -42,9 +42,6 @@ node_modules # Couch .couchadmin -# Ignore faraday server pid -server/.faraday-server.pid - #Ignore images executive report reports/executive/images/ease-of-resolution-piechart.png reports/executive/images/impact-piechart.png @@ -59,4 +56,4 @@ jsconfig.json .idea # vFeed DB -data/vfeed.db \ No newline at end of file +data/vfeed.db diff --git a/server/config.py b/server/config.py index c6b3f016ca0..c7b7dd14dbd 100644 --- a/server/config.py +++ b/server/config.py @@ -17,7 +17,8 @@ FARADAY_BASE = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) FARADAY_SERVER_DBS_DIR = os.path.join(FARADAY_BASE, 'server/workspaces') -FARADAY_SERVER_PID_FILE = os.path.join(FARADAY_BASE, 'server/.faraday-server.pid') +FARADAY_SERVER_PID_FILE = os.path.join( + CONSTANTS.CONST_FARADAY_HOME_PATH, 'faraday-server.pid') REQUIREMENTS_FILE = os.path.join(FARADAY_BASE, 'requirements_server.txt') DEFAULT_CONFIG_FILE = os.path.join(FARADAY_BASE, 'server/default.ini') VERSION_FILE = os.path.join(FARADAY_BASE, CONSTANTS.CONST_VERSION_FILE) From 01cf31d147412972dc3743eea7897379b40ef2ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 27 Mar 2018 19:25:42 -0300 Subject: [PATCH 1043/1506] Expand ~ in constant Otherwise the server broke in nix --- config/globals.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/globals.py b/config/globals.py index fae56203aed..a064f56e355 100644 --- a/config/globals.py +++ b/config/globals.py @@ -5,9 +5,11 @@ ''' +import os + CONST_VERSION_FILE = 'VERSION' CONST_REQUIREMENTS_FILE = 'requirements.txt' -CONST_FARADAY_HOME_PATH = '~/.faraday' +CONST_FARADAY_HOME_PATH = os.path.expanduser('~/.faraday') CONST_FARADAY_PLUGINS_PATH = 'plugins' CONST_FARADAY_PLUGINS_REPO_PATH = 'plugins/repo' CONST_FARADAY_IMAGES = 'images/' From ef9a9cf034c41c64d7813b1656a7a914a38e6ca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 27 Mar 2018 19:35:25 -0300 Subject: [PATCH 1044/1506] Move config.json static file to API endpoint Generating it on every server start was extremely ugly. Use an API endpoint for this --- server/config.py | 6 +----- server/modules/info.py | 6 ++++++ server/www/config/.gitignore | 4 ---- server/www/scripts/config/services/config.js | 2 +- server/www/scripts/index/providers/index.js | 6 +++--- 5 files changed, 11 insertions(+), 13 deletions(-) delete mode 100644 server/www/config/.gitignore diff --git a/server/config.py b/server/config.py index c7b7dd14dbd..3de3dabee0a 100644 --- a/server/config.py +++ b/server/config.py @@ -22,7 +22,6 @@ REQUIREMENTS_FILE = os.path.join(FARADAY_BASE, 'requirements_server.txt') DEFAULT_CONFIG_FILE = os.path.join(FARADAY_BASE, 'server/default.ini') VERSION_FILE = os.path.join(FARADAY_BASE, CONSTANTS.CONST_VERSION_FILE) -WEB_CONFIG_FILE = os.path.join(FARADAY_BASE, 'server/www/config/config.json') REPORTS_VIEWS_DIR = os.path.join(FARADAY_BASE, 'views/reports') LOCAL_CONFIG_FILE = os.path.expanduser( os.path.join(CONSTANTS.CONST_FARADAY_HOME_PATH, 'config/server.ini')) @@ -90,10 +89,7 @@ def gen_web_config(): "osint": __get_osint(), 'vuln_model_db': CONSTANTS.CONST_VULN_MODEL_DB } - if os.path.isfile(WEB_CONFIG_FILE): - os.remove(WEB_CONFIG_FILE) - with open(WEB_CONFIG_FILE, "w") as doc_file: - json.dump(doc, doc_file) + return doc def is_debug_mode(): diff --git a/server/modules/info.py b/server/modules/info.py index d2fb0243071..de2ed87bcb9 100644 --- a/server/modules/info.py +++ b/server/modules/info.py @@ -5,6 +5,7 @@ import flask import os from server.app import app +from server.config import gen_web_config @app.route('/info', methods=['GET']) @@ -20,3 +21,8 @@ def show_info(): response.status_code = 200 return response + + +@app.route('/config') +def get_config(): + return flask.jsonify(gen_web_config()) diff --git a/server/www/config/.gitignore b/server/www/config/.gitignore deleted file mode 100644 index 5e7d2734cfc..00000000000 --- a/server/www/config/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/server/www/scripts/config/services/config.js b/server/www/scripts/config/services/config.js index 741e7c265f2..414d8e6cf89 100644 --- a/server/www/scripts/config/services/config.js +++ b/server/www/scripts/config/services/config.js @@ -5,7 +5,7 @@ angular.module('faradayApp') .factory('configSrv', ['BASEURL', '$http', function(BASEURL, $http) { - var p = $http.get('config/config.json') + var p = $http.get(BASEURL + '_api/config') .then(function(conf) { configSrv.faraday_version = conf.data.ver; configSrv.license_db = conf.data.lic_db; diff --git a/server/www/scripts/index/providers/index.js b/server/www/scripts/index/providers/index.js index 70d53010058..a1c9286fc0a 100644 --- a/server/www/scripts/index/providers/index.js +++ b/server/www/scripts/index/providers/index.js @@ -3,12 +3,12 @@ // See the file 'doc/LICENSE' for the license information angular.module('faradayApp') - .factory('indexFact', ['$http', function($http) { + .factory('indexFact', ['$http', 'BASEURL', function($http, BASEURL) { var indexFact = {}; indexFact.getConf = function() { - return $http.get('config/config.json'); + return $http.get(BASEURL + '_api/config'); }; return indexFact; - }]); \ No newline at end of file + }]); From 918df7c4a52862f3b99c09f78e301e12b81b117a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 6 Apr 2018 15:34:19 -0300 Subject: [PATCH 1045/1506] Move workspaces directory out of source code directory Put in ~/.faraday/workspaces/ instead --- server/config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/config.py b/server/config.py index 3de3dabee0a..d4282cd61af 100644 --- a/server/config.py +++ b/server/config.py @@ -16,7 +16,10 @@ LOGGING_LEVEL = INFO FARADAY_BASE = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -FARADAY_SERVER_DBS_DIR = os.path.join(FARADAY_BASE, 'server/workspaces') +FARADAY_SERVER_DBS_DIR = os.path.join(CONSTANTS.CONST_FARADAY_HOME_PATH, 'workspaces') +if not os.path.exists(FARADAY_SERVER_DBS_DIR): + # Temporary hack, remove me + os.mkdir(FARADAY_SERVER_DBS_DIR) FARADAY_SERVER_PID_FILE = os.path.join( CONSTANTS.CONST_FARADAY_HOME_PATH, 'faraday-server.pid') REQUIREMENTS_FILE = os.path.join(FARADAY_BASE, 'requirements_server.txt') From 6c3ac0e6e319d944c7da72dc884cf8eee9e78d65 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 9 Apr 2018 16:28:20 -0300 Subject: [PATCH 1046/1506] Update model for tasks --- server/models.py | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/server/models.py b/server/models.py index 91f596f5b0c..e2e7a8eab2b 100644 --- a/server/models.py +++ b/server/models.py @@ -63,7 +63,8 @@ 'source_code', 'comment', 'executive_report', - 'workspace' + 'workspace', + 'task' ] @@ -710,6 +711,7 @@ class VulnerabilityGeneric(VulnerabilityABC): confirmed = Column(Boolean, nullable=False, default=False) status = Column(Enum(*STATUSES, name='vulnerability_statuses'), nullable=False, default="open") type = Column(Enum(*VULN_TYPES, name='vulnerability_types'), nullable=False) + issuetracker = Column(Text) workspace_id = Column( Integer, @@ -802,12 +804,15 @@ class VulnerabilityGeneric(VulnerabilityABC): __mapper_args__ = { 'polymorphic_on': type } + @property def attachments(self): return db.session.query(File).filter_by( object_id=self.id, object_type='vulnerability' ) + + class Vulnerability(VulnerabilityGeneric): __tablename__ = None host_id = Column(Integer, ForeignKey(Host.id), index=True) @@ -1108,7 +1113,7 @@ def parent(self): def _make_vuln_count_property(type_=None, only_confirmed=False, - use_column_property=True): + use_column_property=True, extra_query=None): query = (select([func.count(text('vulnerability.id'))]). select_from(table('vulnerability')). where(text('vulnerability.workspace_id = workspace.id')) @@ -1119,14 +1124,17 @@ def _make_vuln_count_property(type_=None, only_confirmed=False, # In this case type_ is supplied from a whitelist so this is safe query = query.where(text("vulnerability.type = '%s'" % type_)) if only_confirmed: - if str(db.engine.url).startswith('sqlite://'): + if db.session.bind.dialect.name == 'sqlite': # SQLite has no "true" expression, we have to use the integer 1 # instead query = query.where(text("vulnerability.confirmed = 1")) - else: + elif db.session.bind.dialect.name == 'postgresql': # I suppose that we're using PostgreSQL, that can't compare # booleans with integers query = query.where(text("vulnerability.confirmed = true")) + + if extra_query: + query = query.where(text(extra_query)) if use_column_property: return column_property(query, deferred=True) else: @@ -1300,6 +1308,8 @@ class File(Metadata): nullable=False) # plain attached file object_id = Column(Integer, nullable=False) object_type = Column(Enum(*OBJECT_TYPES, name='object_types'), nullable=False) + + class UserAvatar(Metadata): __tablename_ = 'user_avatar' @@ -1376,6 +1386,18 @@ class TaskTemplate(TaskABC): # ) +class TaskAssignedTo(db.Model): + __tablename__ = "task_assigned_to_association" + id = Column(Integer, primary_key=True) + task_id = Column( + Integer, ForeignKey('task.id'), nullable=False) + task = relationship('Task') + + user_id = Column(Integer, ForeignKey('faraday_user.id'), nullable=False) + user = relationship('User', + foreign_keys=[user_id]) + + class Task(TaskABC): STATUSES = [ 'new', @@ -1394,8 +1416,9 @@ class Task(TaskABC): 'concrete': True } - assigned_to_id = Column(Integer, ForeignKey('faraday_user.id'), nullable=True) - assigned_to = relationship('User', backref='assigned_tasks', foreign_keys=[assigned_to_id]) + assigned_to = relationship( + "User", + secondary="task_assigned_to_association") methodology_id = Column( Integer, @@ -1486,6 +1509,7 @@ class Comment(Metadata): object_id = Column(Integer, nullable=False) object_type = Column(Enum(*OBJECT_TYPES, name='object_types'), nullable=False) + @property def parent(self): return @@ -1513,6 +1537,8 @@ class ExecutiveReport(Metadata): summary = BlankColumn(Text) title = BlankColumn(Text) confirmed = Column(Boolean, nullable=False, default=False) + vuln_count = Column(Integer, default=0) # saves the amount of vulns when the report was generated. + workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship( 'Workspace', @@ -1529,12 +1555,15 @@ class ExecutiveReport(Metadata): @property def parent(self): return + @property def attachments(self): return db.session.query(File).filter_by( object_id=self.id, object_type='executive_report' ) + + # This constraint uses Columns from different classes # Since it applies to the table vulnerability it should be adVulnerability.ded to the Vulnerability class # However, since it contains columns from children classes, this cannot be done From 1389043232001d837323ce1b4cd1e929944d587a Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 10 Apr 2018 14:25:34 -0300 Subject: [PATCH 1047/1506] Bug fix on vulnerability creator --- server/importer.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index 27cf155c43d..d970cf217ea 100644 --- a/server/importer.py +++ b/server/importer.py @@ -1,6 +1,8 @@ # Faraday Penetration Test IDE # Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information +import string +from random import SystemRandom import os import re @@ -579,6 +581,18 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation parent = session.query(Host).filter_by(id=parent_id).first() if level == 4: parent = session.query(Service).filter_by(id=parent_id).first() + owner_name = document.get('owner', None) + creator = None + if owner_name: + creator = session.query(User).filter_by(username=owner_name).first() + + if not creator: + rng = SystemRandom() + password = "".join( + [rng.choice(string.ascii_letters + string.digits) for _ in + xrange(12)]) + creator, _ = get_or_create(session, User, username=owner_name) + creator.password = password if document['type'] == 'VulnerabilityWeb': method = document.get('method') path = document.get('path') @@ -595,13 +609,15 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation path=path, website=website, workspace=workspace, + creator=creator, ) if document['type'] == 'Vulnerability': vuln_params = { 'name': document.get('name'), 'workspace': workspace, - 'description': document.get('desc') + 'description': document.get('desc'), + 'creator': creator, } if type(parent) == Host: vuln_params.update({'host_id': parent.id}) @@ -613,6 +629,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation **vuln_params ) vulnerability.severity = severity + vulnerability.creator = creator vulnerability.confirmed = document.get('confirmed', False) or False vulnerability.data = document.get('data') vulnerability.ease_of_resolution = document.get('easeofresolution') if document.get('easeofresolution') else None From 2900acc5f09cc3fe21e460379f3edb9436e91136 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 11 Apr 2018 16:05:35 -0300 Subject: [PATCH 1048/1506] Copy default configuration. Fix a bug when the .faraday directory is empty --- model/workspace.py | 4 +++- server/app.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/model/workspace.py b/model/workspace.py index f207644785f..f8a5b875413 100644 --- a/model/workspace.py +++ b/model/workspace.py @@ -22,7 +22,9 @@ class Workspace(object): """ class_signature = "Workspace" - def __init__(self, name, desc=None, manager=None, shared=CONF.getAutoShareWorkspace()): + def __init__(self, name, desc=None, manager=None, shared=None): + if not shared: + shared = CONF.getAutoShareWorkspace() self.name = name self.description = desc self.customer = "" diff --git a/server/app.py b/server/app.py index e3e27414ce4..4f0dc3f3436 100644 --- a/server/app.py +++ b/server/app.py @@ -8,7 +8,7 @@ from os.path import join, expanduser from random import SystemRandom -from server.config import LOCAL_CONFIG_FILE +from server.config import LOCAL_CONFIG_FILE, copy_default_config_to_local from server.models import User try: @@ -122,6 +122,8 @@ def log_queries_count(response): def save_new_secret_key(app): + if not os.path.exists(LOCAL_CONFIG_FILE): + copy_default_config_to_local() config = ConfigParser() config.read(LOCAL_CONFIG_FILE) rng = SystemRandom() From c61e91c3e9f718cbf90fc06b5aaa0c3467d9ec07 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 11 Apr 2018 16:17:36 -0300 Subject: [PATCH 1049/1506] Fix bug with importer task "asiggned to" --- server/importer.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index d970cf217ea..9d8a121e86c 100644 --- a/server/importer.py +++ b/server/importer.py @@ -892,7 +892,14 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation task.methodology = methodology task.description = document.get('description') - task.assigned_to = session.query(User).filter_by(username=document.get('username')).first() + + assigned_users = [] + + for username in document.get('assigned_to', []): + if username: + assigned_users.append(session.query(User).filter_by(username=username).first()) + + task.assigned_to = assigned_users mapped_status = { 'New': 'new', From fa68ecf45b15d1a132ce51de1352eedc718b292b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 11 Apr 2018 16:44:52 -0300 Subject: [PATCH 1050/1506] Fix admin user creation. Copy default config files on new instances --- config/configuration.py | 1 + server/commands/initdb.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/config/configuration.py b/config/configuration.py index 48987324aff..3e686fb391d 100644 --- a/config/configuration.py +++ b/config/configuration.py @@ -66,6 +66,7 @@ def __init__(self, xml_file=DEFAULT_XML): """ Initializer that handles a configuration automagically. """ self.filepath = xml_file + self._api_con_info = '' if self._isConfig(): self._getConfig() diff --git a/server/commands/initdb.py b/server/commands/initdb.py index e718260a91e..f6011468306 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -1,4 +1,5 @@ import getpass +import shutil import string import os @@ -12,6 +13,7 @@ from sqlalchemy import create_engine from config.configuration import getInstanceConfiguration +from faraday import FARADAY_USER_CONFIG_XML, FARADAY_BASE_CONFIG_XML try: # py2.7 @@ -122,15 +124,18 @@ def _create_admin_user(self, conn_string): already_created = False try: engine.execute("INSERT INTO \"faraday_user\" (username, name, password, " - "is_ldap, active) VALUES ('faraday', 'Administrator', " - "'{0}', false, true);".format(random_password)) - except sqlalchemy.exc.IntegrityError: + "is_ldap, active, last_login_ip, current_login_ip, role) VALUES ('faraday', 'Administrator', " + "'{0}', false, true, '127.0.0.1', '127.0.0.1', 'admin');".format(random_password)) + except sqlalchemy.exc.IntegrityError as ex: + print(ex) # when re using database user could be created previusly already_created = True print( "{yellow}WARNING{white}: Faraday administrator user already exists.".format( yellow=Fore.YELLOW, white=Fore.WHITE)) if not already_created: + if not os.path.isfile(FARADAY_USER_CONFIG_XML): + shutil.copy(FARADAY_BASE_CONFIG_XML, FARADAY_USER_CONFIG_XML) CONF = getInstanceConfiguration() CONF.setAPIUrl('http://localhost:5985') CONF.setAPIUsername('faraday') From 40b9b2ea022c33180f4a276037bd8e6e7f0dc965 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 11 Apr 2018 17:10:36 -0300 Subject: [PATCH 1051/1506] Split nix files into default.nix and shell.nix --- default.nix | 26 ++++++++++++++++++++++++++ shell.nix | 25 ++++--------------------- 2 files changed, 30 insertions(+), 21 deletions(-) create mode 100644 default.nix diff --git a/default.nix b/default.nix new file mode 100644 index 00000000000..2d89705cc42 --- /dev/null +++ b/default.nix @@ -0,0 +1,26 @@ +with (import {}); +let + python_env = (import ./requirements.nix {}); + cleanPycs = path: type: type == "directory" || builtins.match ".*\.pyc" path != null; +in + { + server = stdenv.mkDerivation { + name = "faraday_server"; + src = builtins.filterSource cleanPycs ./.; + buildInputs = [coreutils makeWrapper python_env.interpreter]; + builder = "${bash}/bin/bash"; + args = ["-c" '' + + source $stdenv/setup + mkdir -p $out/{bin,src} + cp -rv ${./.}/* $out/src/ + + makeWrapper \ + ${python_env.interpreter}/bin/python \ + $out/bin/faraday-server \ + --prefix PATH : ${./.} \ + --add-flags ${./.}/faraday-server.py + '']; + }; + python = python_env; + } diff --git a/shell.nix b/shell.nix index b915358d3f1..b5c11e83217 100644 --- a/shell.nix +++ b/shell.nix @@ -1,24 +1,7 @@ with (import {}); let - faraday_server = stdenv.mkDerivation { - name = "faraday_server"; - src = builtins.filterSource cleanPycs ./.; - buildInputs = [coreutils makeWrapper python_env.interpreter]; - builder = "${bash}/bin/bash"; - args = ["-c" '' - - source $stdenv/setup - mkdir -p $out/{bin,src} - cp -rv ${./.}/* $out/src/ - - makeWrapper \ - ${python_env.interpreter}/bin/python \ - $out/bin/faraday-server \ - --prefix PATH : $out/src \ - --add-flags ${./.}/faraday-server.py - '']; - }; - python_env = (import ./requirements.nix {}); - cleanPycs = path: type: type == "directory" || builtins.match ".*\.pyc" path != null; in - mkShell { buildInputs = [faraday_server]; } + mkShell { + buildInputs = with (import ./default.nix); + [server python.interpreter tmux]; + } From b0eaca4849f1c583d3e84f115acd1859a5fd78ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 11 Apr 2018 17:32:06 -0300 Subject: [PATCH 1052/1506] Refactor nix files --- default.nix | 24 +++---------------- .../server/requirements.nix | 0 .../server/requirements_override.nix | 0 shell.nix | 2 +- 4 files changed, 4 insertions(+), 22 deletions(-) rename requirements.nix => nix/server/requirements.nix (100%) rename requirements_override.nix => nix/server/requirements_override.nix (100%) diff --git a/default.nix b/default.nix index 2d89705cc42..b6f6df09eeb 100644 --- a/default.nix +++ b/default.nix @@ -1,26 +1,8 @@ with (import {}); let - python_env = (import ./requirements.nix {}); - cleanPycs = path: type: type == "directory" || builtins.match ".*\.pyc" path != null; + server_pkgs = callPackage ./nix/server {}; in { - server = stdenv.mkDerivation { - name = "faraday_server"; - src = builtins.filterSource cleanPycs ./.; - buildInputs = [coreutils makeWrapper python_env.interpreter]; - builder = "${bash}/bin/bash"; - args = ["-c" '' - - source $stdenv/setup - mkdir -p $out/{bin,src} - cp -rv ${./.}/* $out/src/ - - makeWrapper \ - ${python_env.interpreter}/bin/python \ - $out/bin/faraday-server \ - --prefix PATH : ${./.} \ - --add-flags ${./.}/faraday-server.py - '']; - }; - python = python_env; + server = server_pkgs.server; + python = server_pkgs.python; } diff --git a/requirements.nix b/nix/server/requirements.nix similarity index 100% rename from requirements.nix rename to nix/server/requirements.nix diff --git a/requirements_override.nix b/nix/server/requirements_override.nix similarity index 100% rename from requirements_override.nix rename to nix/server/requirements_override.nix diff --git a/shell.nix b/shell.nix index b5c11e83217..3d3d1b4efde 100644 --- a/shell.nix +++ b/shell.nix @@ -3,5 +3,5 @@ let in mkShell { buildInputs = with (import ./default.nix); - [server python.interpreter tmux]; + [server python.interpreter]; } From 76a32857db343567d5cec81be0bfb910b63d7675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 11 Apr 2018 17:36:46 -0300 Subject: [PATCH 1053/1506] Add faraday pid to gitignore It won't be used from anymore but some user could have this --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 33df2185266..dfc6eca8658 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,9 @@ node_modules # Couch .couchadmin +# Ignore faraday server pid +server/.faraday-server.pid + #Ignore images executive report reports/executive/images/ease-of-resolution-piechart.png reports/executive/images/impact-piechart.png @@ -56,4 +59,4 @@ jsconfig.json .idea # vFeed DB -data/vfeed.db +data/vfeed.db \ No newline at end of file From c09792adf7837e0041388265eec464eba13229f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 11 Apr 2018 17:39:54 -0300 Subject: [PATCH 1054/1506] Add missing nixfile --- nix/server/default.nix | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 nix/server/default.nix diff --git a/nix/server/default.nix b/nix/server/default.nix new file mode 100644 index 00000000000..5d88c8e643e --- /dev/null +++ b/nix/server/default.nix @@ -0,0 +1,28 @@ +{ pkgs ? import {} +}: +let + python_env = (pkgs.callPackage ./requirements.nix {inherit pkgs;}); + cleanPycs = path: type: type == "directory" || builtins.match ".*\.pyc" path != null; + src = ./../..; +in + { + server = pkgs.stdenv.mkDerivation { + name = "faraday_server"; + src = builtins.filterSource cleanPycs src; + buildInputs = with pkgs; [coreutils makeWrapper python_env.interpreter]; + builder = "${pkgs.bash}/bin/bash"; + args = ["-c" '' + + source $stdenv/setup + mkdir -p $out/{bin,src} + cp -rv ${src}/* $out/src/ + + makeWrapper \ + ${python_env.interpreter}/bin/python \ + $out/bin/faraday-server \ + --prefix PATH : $out \ + --add-flags ${src}/faraday-server.py + '']; + }; + python = python_env; + } From 353ae724a1845df8ae273290f6274144acf28240 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 11 Apr 2018 17:49:33 -0300 Subject: [PATCH 1055/1506] Remove role check on GTK client. remove save password on gtk client --- faraday.py | 2 +- server/commands/initdb.py | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/faraday.py b/faraday.py index d507a931758..2d6e3ee5125 100755 --- a/faraday.py +++ b/faraday.py @@ -565,7 +565,7 @@ def doLoginLoop(): user_info = get_user_info() - if user_info is None or 'username' not in user_info or 'roles' not in user_info or 'client' in user_info['roles']: + if user_info is None or 'username' not in user_info: print("You can't login as a client. You have %s attempt(s) left." % (3 - attempt)) continue diff --git a/server/commands/initdb.py b/server/commands/initdb.py index f6011468306..c996a76a6fa 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -136,11 +136,7 @@ def _create_admin_user(self, conn_string): if not already_created: if not os.path.isfile(FARADAY_USER_CONFIG_XML): shutil.copy(FARADAY_BASE_CONFIG_XML, FARADAY_USER_CONFIG_XML) - CONF = getInstanceConfiguration() - CONF.setAPIUrl('http://localhost:5985') - CONF.setAPIUsername('faraday') - CONF.setAPIPassword(random_password) - CONF.saveConfig() + print("Admin user created with {red}username: {white}faraday and " " {red}password{white}: {" "random_password}".format(random_password=random_password, From 6029909ff7f2368df5874c72633f9fc7b26eaa9c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 11 Apr 2018 19:01:27 -0300 Subject: [PATCH 1056/1506] Fix importer bug when the host does not have description --- server/importer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/importer.py b/server/importer.py index 9d8a121e86c..af6d93b64d7 100644 --- a/server/importer.py +++ b/server/importer.py @@ -478,6 +478,8 @@ def merge_with_host(self, host, interface, workspace): if interface['network_segment']: host.net_segment = interface['network_segment'] if interface['description']: + if not host.description: + host.description = '' host.description += '\n Interface data: {0}'.format(interface['description']) if type(interface['hostnames']) in (str, unicode): interface['hostnames'] = [interface['hostnames']] From e1c7363ec745ff623d3f168ce9a7cdb8353db551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 11 Apr 2018 19:02:52 -0300 Subject: [PATCH 1057/1506] Changes to make faraday 3 work on nix/nixos --- nix/server/requirements.nix | 900 ++++++++++++++++++- server/api/modules/info.py | 3 +- server/www/scripts/config/services/config.js | 2 +- 3 files changed, 892 insertions(+), 13 deletions(-) diff --git a/nix/server/requirements.nix b/nix/server/requirements.nix index 9211a07c7c7..fd23b12b408 100644 --- a/nix/server/requirements.nix +++ b/nix/server/requirements.nix @@ -2,7 +2,7 @@ # See more at: https://github.com/garbas/pypi2nix # # COMMAND: -# pypi2nix -V 2.7 -r requirements_server.txt -E libffi -E openssl +# pypi2nix -V 2.7 -r ../../requirements_server.txt -E libffi -E openssl -E postgresql -E pkgconfig zlib libjpeg openjpeg libtiff freetype lcms2 libwebp tcl # { pkgs ? import {} @@ -29,7 +29,7 @@ let }; }; - commonBuildInputs = with pkgs; [ libffi openssl ]; + commonBuildInputs = with pkgs; [ libffi openssl postgresql pkgconfig zlib libjpeg openjpeg libtiff freetype lcms2 libwebp tcl ]; commonDoCheck = false; withPackages = pkgs': @@ -94,6 +94,23 @@ let + "Babel" = python.mkDerivation { + name = "Babel-2.5.3"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/0e/d5/9b1d6a79c975d0e9a32bd337a1465518c2519b14b214682ca9892752417e/Babel-2.5.3.tar.gz"; sha256 = "8ce4cb6fdd4393edd323227cba3a077bceb2a6ce5201c902c65e730046f41f14"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."pytz" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://babel.pocoo.org/"; + license = licenses.bsdOriginal; + description = "Internationalization utilities"; + }; + }; + + + "Flask" = python.mkDerivation { name = "Flask-0.12.2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/eb/12/1c7bd06fcbd08ba544f25bf2c6612e305a70ea51ca0eda8007344ec3f123/Flask-0.12.2.tar.gz"; sha256 = "49f44461237b69ecd901cc7ce66feea0319b9158743dd27a2899962ab214dac1"; }; @@ -114,12 +131,181 @@ let + "Flask-BabelEx" = python.mkDerivation { + name = "Flask-BabelEx-0.9.3"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/80/ad/cc2b0becd98050eed775ca85d6e5fa784547acff69f968183098df8a52b3/Flask-BabelEx-0.9.3.tar.gz"; sha256 = "cf79cdedb5ce860166120136b0e059e9d97b8df07a3bc2411f6243de04b754b4"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Babel" + self."Flask" + self."Jinja2" + self."speaklater" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://github.com/mrjoes/flask-babelex"; + license = licenses.bsdOriginal; + description = "Adds i18n/l10n support to Flask applications"; + }; + }; + + + + "Flask-Classful" = python.mkDerivation { + name = "Flask-Classful-0.14.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/b7/2a/26d548a19ce7411f49ddb0abd3c3319f4ab47354cc4354225e5a7a91a6bf/Flask-Classful-0.14.0.tar.gz"; sha256 = "036589065f0f6e35e37c1146616cb1b82b2bf1111f68e02a8939478b32bf524d"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Flask" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/teracyhq/flask-classful"; + license = licenses.bsdOriginal; + description = "Class based views for Flask"; + }; + }; + + + + "Flask-Login" = python.mkDerivation { + name = "Flask-Login-0.4.1"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/c1/ff/bd9a4d2d81bf0c07d9e53e8cd3d675c56553719bbefd372df69bf1b3c1e4/Flask-Login-0.4.1.tar.gz"; sha256 = "c815c1ac7b3e35e2081685e389a665f2c74d7e077cb93cecabaea352da4752ec"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Flask" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/maxcountryman/flask-login"; + license = licenses.mit; + description = "User session management for Flask"; + }; + }; + + + + "Flask-Mail" = python.mkDerivation { + name = "Flask-Mail-0.9.1"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/05/2f/6a545452040c2556559779db87148d2a85e78a26f90326647b51dc5e81e9/Flask-Mail-0.9.1.tar.gz"; sha256 = "22e5eb9a940bf407bcf30410ecc3708f3c56cc44b29c34e1726fe85006935f41"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Flask" + self."blinker" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/rduplain/flask-mail"; + license = licenses.bsdOriginal; + description = "Flask extension for sending email"; + }; + }; + + + + "Flask-Principal" = python.mkDerivation { + name = "Flask-Principal-0.4.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/14/c7/2531aca6ab7baa3774fde2dfc9c9dd6d5a42576a1013a93701bfdc402fdd/Flask-Principal-0.4.0.tar.gz"; sha256 = "f5d6134b5caebfdbb86f32d56d18ee44b080876a27269560a96ea35f75c99453"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Flask" + self."blinker" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://packages.python.org/Flask-Principal/"; + license = licenses.mit; + description = "Identity management for flask"; + }; + }; + + + + "Flask-SQLAlchemy" = python.mkDerivation { + name = "Flask-SQLAlchemy-2.3.1"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/d0/24/fb7c9d97f0f2ac2f484b867796f6fde1b5064ec753eaa68ce49ac8584b5e/Flask-SQLAlchemy-2.3.1.tar.gz"; sha256 = "ab879cf88d30f2961dd9b4d709dcd31a25e0306855324c7d9a74fca6ad6ef8c3"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Flask" + self."SQLAlchemy" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://github.com/mitsuhiko/flask-sqlalchemy"; + license = licenses.bsdOriginal; + description = "Adds SQLAlchemy support to your Flask application"; + }; + }; + + + + "Flask-Security" = python.mkDerivation { + name = "Flask-Security-3.0.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/ba/c1/16e460fec7961509b10aaf8cc986fa7a1df5dced2844f42cd46732621211/Flask-Security-3.0.0.tar.gz"; sha256 = "d61daa5f5a48f89f30f50555872bdf581b2c65804668b0313345cd7beff26432"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Flask" + self."Flask-BabelEx" + self."Flask-Login" + self."Flask-Mail" + self."Flask-Principal" + self."Flask-SQLAlchemy" + self."Flask-WTF" + self."SQLAlchemy" + self."bcrypt" + self."itsdangerous" + self."passlib" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/mattupstate/flask-security"; + license = licenses.mit; + description = "Simple security for Flask apps."; + }; + }; + + + + "Flask-WTF" = python.mkDerivation { + name = "Flask-WTF-0.14.2"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/ba/15/00a9693180f214225a2c0b1bb9077f3b0b21f2e86522cbba22e8ad6e570c/Flask-WTF-0.14.2.tar.gz"; sha256 = "5d14d55cfd35f613d99ee7cba0fc3fbbe63ba02f544d349158c14ca15561cc36"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Flask" + self."WTForms" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/lepture/flask-wtf"; + license = licenses.bsdOriginal; + description = "Simple integration of Flask and WTForms."; + }; + }; + + + + "IPy" = python.mkDerivation { + name = "IPy-0.83"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/88/28/79162bfc351a3f1ab44d663ab3f03fb495806fdb592170990a1568ffbf63/IPy-0.83.tar.gz"; sha256 = "61da5a532b159b387176f6eabf11946e7458b6df8fb8b91ff1d345ca7a6edab8"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/autocracy/python-ipy"; + license = licenses.bsdOriginal; + description = "Class and tools for handling of IPv4 and IPv6 addresses and networks"; + }; + }; + + + "Jinja2" = python.mkDerivation { name = "Jinja2-2.10"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/56/e6/332789f295cf22308386cf5bbd1f4e00ed11484299c5d7383378cf48ba47/Jinja2-2.10.tar.gz"; sha256 = "f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4"; }; doCheck = commonDoCheck; buildInputs = commonBuildInputs; propagatedBuildInputs = [ + self."Babel" self."MarkupSafe" ]; meta = with pkgs.stdenv.lib; { @@ -146,12 +332,46 @@ let - "SQLAlchemy" = python.mkDerivation { - name = "SQLAlchemy-1.2.5"; - src = pkgs.fetchurl { url = "https://pypi.python.org/packages/7e/57/435347429f4ff5016b72c9b179c70a6cc67e5f9b4d3f3e9b51362c40b356/SQLAlchemy-1.2.5.tar.gz"; sha256 = "249000654107a420a40200f1e0b555a79dfd4eff235b2ff60bc77714bd045f2d"; }; + "Pillow" = python.mkDerivation { + name = "Pillow-4.2.1"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/55/aa/f7f983fb72710a9daa4b3374b7c160091d3f94f5c09221f9336ade9027f3/Pillow-4.2.1.tar.gz"; sha256 = "c724f65870e545316f9e82e4c6d608ab5aa9dd82d5185e5b2e72119378740073"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."olefile" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://python-pillow.org"; + license = "Standard PIL License"; + description = "Python Imaging Library (Fork)"; + }; + }; + + + + "Pygments" = python.mkDerivation { + name = "Pygments-2.2.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/71/2a/2e4e77803a8bd6408a2903340ac498cb0a2181811af7c9ec92cb70b0308a/Pygments-2.2.0.tar.gz"; sha256 = "dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc"; }; doCheck = commonDoCheck; buildInputs = commonBuildInputs; propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://pygments.org/"; + license = licenses.bsdOriginal; + description = "Pygments is a syntax highlighting package written in Python."; + }; + }; + + + + "SQLAlchemy" = python.mkDerivation { + name = "SQLAlchemy-1.2.0b2"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/59/11/f7877b47db0df1576568846f06dca8fc403bb404118c641418c3183a0eeb/SQLAlchemy-1.2.0b2.tar.gz"; sha256 = "2a01c36941b87cdc5ab32aec5b0f315f1a3dfb8d8b8efc97be0419cfd78fc590"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."psycopg2" + ]; meta = with pkgs.stdenv.lib; { homepage = "http://www.sqlalchemy.org"; license = licenses.mit; @@ -162,8 +382,8 @@ let "Twisted" = python.mkDerivation { - name = "Twisted-17.9.0"; - src = pkgs.fetchurl { url = "https://pypi.python.org/packages/a2/37/298f9547606c45d75aa9792369302cc63aa4bbcf7b5f607560180dd099d2/Twisted-17.9.0.tar.bz2"; sha256 = "0da1a7e35d5fcae37bc9c7978970b5feb3bc82822155b8654ec63925c05af75c"; }; + name = "Twisted-17.5.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/31/bf/7f86a8f8b9778e90d8b2921e9f442a8c8aa33fd2489fc10f236bc8af1749/Twisted-17.5.0.tar.bz2"; sha256 = "f198a494f0df2482f7c5f99d7f3eef33d22763ffc76641b36fec476b878002ea"; }; doCheck = commonDoCheck; buildInputs = commonBuildInputs; propagatedBuildInputs = [ @@ -187,6 +407,38 @@ let + "Unidecode" = python.mkDerivation { + name = "Unidecode-1.0.22"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/9d/36/49d0ee152b6a1631f03a541532c6201942430060aa97fe011cf01a2cce64/Unidecode-1.0.22.tar.gz"; sha256 = "8c33dd588e0c9bc22a76eaa0c715a5434851f726131bd44a6c26471746efabf5"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = ""; + license = licenses.gpl2Plus; + description = "ASCII transliterations of Unicode text"; + }; + }; + + + + "WTForms" = python.mkDerivation { + name = "WTForms-2.1"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/bf/91/2e553b86c55e9cf2f33265de50e052441fb753af46f5f20477fe9c61280e/WTForms-2.1.zip"; sha256 = "ffdf10bd1fa565b8233380cb77a304cd36fd55c73023e91d4b803c96bc11d46f"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Babel" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://wtforms.simplecodes.com/"; + license = licenses.bsdOriginal; + description = "A flexible forms validation and rendering library for python web development."; + }; + }; + + + "Werkzeug" = python.mkDerivation { name = "Werkzeug-0.14.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/9f/08/a3bb1c045ec602dc680906fc0261c267bed6b3bb4609430aff92c3888ec8/Werkzeug-0.14.1.tar.gz"; sha256 = "c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c"; }; @@ -235,6 +487,91 @@ let + "autobahn" = python.mkDerivation { + name = "autobahn-17.10.1"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/e4/2e/01a64212b1eb580d601fa20f146c962235e3493795f46e3b254597ec635d/autobahn-17.10.1.tar.gz"; sha256 = "8cf74132a18da149c5ea3dcbb5e055f6f4fe5a0238b33258d29e89bd276a8078"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Twisted" + self."pyOpenSSL" + self."service-identity" + self."six" + self."txaio" + self."zope.interface" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://crossbar.io/autobahn"; + license = licenses.mit; + description = "WebSocket client & server library, WAMP real-time framework"; + }; + }; + + + + "backports.csv" = python.mkDerivation { + name = "backports.csv-1.0.5"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/6a/0b/2071ad285e87dd26f5c02147ba13abf7ec777ff20416a60eb15ea204ca76/backports.csv-1.0.5.tar.gz"; sha256 = "8c421385cbc6042ba90c68c871c5afc13672acaf91e1508546d6cda6725ebfc6"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/ryanhiebert/backports.csv"; + license = ""; + description = "Backport of Python 3 csv module"; + }; + }; + + + + "backports.ssl-match-hostname" = python.mkDerivation { + name = "backports.ssl-match-hostname-3.5.0.1"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/76/21/2dc61178a2038a5cb35d14b61467c6ac632791ed05131dda72c20e7b9e23/backports.ssl_match_hostname-3.5.0.1.tar.gz"; sha256 = "502ad98707319f4a51fa2ca1c677bd659008d27ded9f6380c79e8932e38dcdf2"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://bitbucket.org/brandon/backports.ssl_match_hostname"; + license = licenses.psfl; + description = "The ssl.match_hostname() function from Python 3.5"; + }; + }; + + + + "bcrypt" = python.mkDerivation { + name = "bcrypt-3.1.4"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/f3/ec/bb6b384b5134fd881b91b6aa3a88ccddaad0103857760711a5ab8c799358/bcrypt-3.1.4.tar.gz"; sha256 = "67ed1a374c9155ec0840214ce804616de49c3df9c5bc66740687c1c9b1cd9e8d"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."cffi" + self."six" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/pyca/bcrypt/"; + license = licenses.asl20; + description = "Modern password hashing for your software and your servers"; + }; + }; + + + + "blinker" = python.mkDerivation { + name = "blinker-1.4"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/1b/51/e2a9f3b757eb802f61dc1f2b09c8c99f6eb01cf06416c0671253536517b6/blinker-1.4.tar.gz"; sha256 = "471aee25f3992bd325afa3772f1063dbdbbca947a041b8b89466dc00d606f8b6"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://pythonhosted.org/blinker/"; + license = licenses.mit; + description = "Fast, simple object-to-object and broadcast signaling"; + }; + }; + + + "certifi" = python.mkDerivation { name = "certifi-2018.1.18"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/15/d4/2f888fc463d516ff7bf2379a4e9a552fef7f22a94147655d9b1097108248/certifi-2018.1.18.tar.gz"; sha256 = "edbc3f203427eef571f79a7692bb160a2b0f7ccaa31953e99bd17e307cf63f7d"; }; @@ -282,6 +619,26 @@ let + "cli-helpers" = python.mkDerivation { + name = "cli-helpers-1.0.2"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/d5/13/3783ef3fa23ab76b56d4b8f96ee90808d2c167bafc5eaa4ad3c78b75abe6/cli_helpers-1.0.2.tar.gz"; sha256 = "f77837c5fbcbea39e0cb782506515459a0da75465489bae35e46da7f51c5b9fc"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Pygments" + self."backports.csv" + self."tabulate" + self."terminaltables" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/dbcli/cli_helpers"; + license = licenses.bsdOriginal; + description = "Helpers for building command-line apps"; + }; + }; + + + "click" = python.mkDerivation { name = "click-6.7"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/95/d9/c3336b6b5711c3ab9d1d3a80f1a3e2afeb9d8c02a7166462f6cc96570897/click-6.7.tar.gz"; sha256 = "f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b"; }; @@ -297,6 +654,38 @@ let + "colorama" = python.mkDerivation { + name = "colorama-0.3.9"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/e6/76/257b53926889e2835355d74fec73d82662100135293e17d382e2b74d1669/colorama-0.3.9.tar.gz"; sha256 = "48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/tartley/colorama"; + license = licenses.bsdOriginal; + description = "Cross-platform colored terminal text."; + }; + }; + + + + "configobj" = python.mkDerivation { + name = "configobj-5.0.6"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/64/61/079eb60459c44929e684fa7d9e2fdca403f67d64dd9dbac27296be2e0fab/configobj-5.0.6.tar.gz"; sha256 = "a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."six" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/DiffSK/configobj"; + license = licenses.bsdOriginal; + description = "Config file reading, writing and validation."; + }; + }; + + + "constantly" = python.mkDerivation { name = "constantly-15.1.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/95/f1/207a0a478c4bb34b1b49d5915e2db574cadc415c9ac3a7ef17e29b2e8951/constantly-15.1.0.tar.gz"; sha256 = "586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35"; }; @@ -340,6 +729,7 @@ let self."enum34" self."idna" self."ipaddress" + self."pytz" self."six" ]; meta = with pkgs.stdenv.lib; { @@ -351,6 +741,21 @@ let + "deprecation" = python.mkDerivation { + name = "deprecation-1.0.1"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/8c/e3/e5c66eba8fa2fd567065fa70ada98b990f449f74fb812b408fa7aafe82c9/deprecation-1.0.1.tar.gz"; sha256 = "b9bff5cc91f601ef2a8a0200bc6cde3f18a48c2ed3d1ecbfc16076b14b3ad935"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://deprecation.readthedocs.io/"; + license = licenses.asl20; + description = "A library to handle automated deprecations"; + }; + }; + + + "enum34" = python.mkDerivation { name = "enum34-1.1.6"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz"; sha256 = "8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1"; }; @@ -366,7 +771,46 @@ let - "http-parser" = python.mkDerivation { + "filedepot" = python.mkDerivation { + name = "filedepot-0.5.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/75/89/c744b5326d956dd1e8b2e591cb50759965d16ccfd49e7d3124a85a821069/filedepot-0.5.0.tar.gz"; sha256 = "e319a2b163c37fa1f3da21ad7c81a49a12de66da87d1af428fb143f092b96e5d"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Pillow" + self."SQLAlchemy" + self."Unidecode" + self."requests" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/amol-/depot"; + license = licenses.mit; + description = "Toolkit for storing files and attachments in web applications"; + }; + }; + + + + "filteralchemy" = python.mkDerivation { + name = "filteralchemy-0.1.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/ad/3c/20280257ee8411e905fdd09e4bf7f2bbc37d7e47986413c57b30acb473a0/filteralchemy-0.1.0.tar.gz"; sha256 = "38b9784aa85d31a393282eb2b313e5955b5c7632d9bbfc5c653c040e937a7d96"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."marshmallow-sqlalchemy" + self."six" + self."webargs" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/jmcarp/filteralchemy"; + license = "Copyright 2015 Joshua Carp"; + description = "Declarative query builder for SQLAlchemy"; + }; + }; + + + + "http-parser" = python.mkDerivation { name = "http-parser-0.8.3"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/07/c4/22e3c76c2313c26dd5f84f1205b916ff38ea951aab0c4544b6e2f5920d64/http-parser-0.8.3.tar.gz"; sha256 = "e2aff90a60def3e476bd71694d8757c0f95ebf2fedf0a8ae34ee306e0b20db83"; }; doCheck = commonDoCheck; @@ -381,6 +825,21 @@ let + "humanize" = python.mkDerivation { + name = "humanize-0.5.1"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/8c/e0/e512e4ac6d091fc990bbe13f9e0378f34cf6eecd1c6c268c9e598dcf5bb9/humanize-0.5.1.tar.gz"; sha256 = "a43f57115831ac7c70de098e6ac46ac13be00d69abbf60bdcac251344785bb19"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://github.com/jmoiron/humanize"; + license = licenses.mit; + description = "python humanize utilities"; + }; + }; + + + "hyperlink" = python.mkDerivation { name = "hyperlink-18.0.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/41/e1/0abd4b480ec04892b1db714560f8c855d43df81895c98506442babf3652f/hyperlink-18.0.0.tar.gz"; sha256 = "f01b4ff744f14bc5d0a22a6b9f1525ab7d6312cb0ff967f59414bbac52f0a306"; }; @@ -461,9 +920,172 @@ let + "marshmallow" = python.mkDerivation { + name = "marshmallow-2.15.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/dc/34/b99d68c78378783c96254cebdf82b6ffa887a1f0e955ec5362eff2f1b2c2/marshmallow-2.15.0.tar.gz"; sha256 = "d3f31fe7be2106b1d783cbd0765ef4e1c6615505514695f33082805f929dd584"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."python-dateutil" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/marshmallow-code/marshmallow"; + license = licenses.mit; + description = "A lightweight library for converting complex datatypes to and from native Python datatypes."; + }; + }; + + + + "marshmallow-sqlalchemy" = python.mkDerivation { + name = "marshmallow-sqlalchemy-0.13.2"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/22/e4/47fbf458292cab8a17641b69cec3d77b127bd2400139164838a0b67133e7/marshmallow-sqlalchemy-0.13.2.tar.gz"; sha256 = "9804ef2829f781f469a06528d107c2a763f109c687266ab8b1f000f9684184ae"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."SQLAlchemy" + self."marshmallow" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/marshmallow-code/marshmallow-sqlalchemy"; + license = licenses.mit; + description = "SQLAlchemy integration with the marshmallow (de)serialization library"; + }; + }; + + + + "nplusone" = python.mkDerivation { + name = "nplusone-0.8.1"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/b0/9a/8f8bf4dfed57ee26d16c4ddcf38ff4ffee38048e15f4e1a6686e1fbeeadb/nplusone-0.8.1.tar.gz"; sha256 = "6edfa3bea1a99bb22e3b218bcafdefeddb55ebc3b362dc8921844b3b5000e29d"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."blinker" + self."six" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/jmcarp/nplusone"; + license = "Copyright 2016 Joshua Carp"; + description = "Detecting the n+1 queries problem in Python"; + }; + }; + + + + "olefile" = python.mkDerivation { + name = "olefile-0.45.1"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/d3/8a/e0f0e56d6a542dd987f9290ef7b5164636ee597ce8c2932c19c78292d5ec/olefile-0.45.1.zip"; sha256 = "2b6575f5290de8ab1086f8c5490591f7e0885af682c7c1793bdaf6e64078d385"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://www.decalage.info/python/olefileio"; + license = licenses.bsdOriginal; + description = "Python package to parse, read and write Microsoft OLE2 files (Structured Storage or Compound Document, Microsoft Office) - Improved version of the OleFileIO module from PIL, the Python Image Library."; + }; + }; + + + + "passlib" = python.mkDerivation { + name = "passlib-1.7.1"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/25/4b/6fbfc66aabb3017cd8c3bd97b37f769d7503ead2899bf76e570eb91270de/passlib-1.7.1.tar.gz"; sha256 = "3d948f64138c25633613f303bcc471126eae67c04d5e3f6b7b8ce6242f8653e0"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."bcrypt" + self."cryptography" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://bitbucket.org/ecollins/passlib"; + license = licenses.bsdOriginal; + description = "comprehensive password hashing framework supporting over 30 schemes"; + }; + }; + + + + "pgcli" = python.mkDerivation { + name = "pgcli-1.8.2"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/91/84/e3be2851c87a9550cc747ba85ade36d1b51bb9a747ac93025a01a25baa41/pgcli-1.8.2.tar.gz"; sha256 = "79b32cb0a44e03fc6c26d43080459d48d39d6d81391eb09062a82c940b61441c"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Pygments" + self."cli-helpers" + self."click" + self."configobj" + self."humanize" + self."pgspecial" + self."prompt-toolkit" + self."psycopg2" + self."setproctitle" + self."sqlparse" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://pgcli.com"; + license = licenses.bsdOriginal; + description = "CLI for Postgres Database. With auto-completion and syntax highlighting."; + }; + }; + + + + "pgspecial" = python.mkDerivation { + name = "pgspecial-1.10.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/d5/d8/cd12f64c42b9878e3e677ba2c43f7abdc87eadba5f18b640f8efda555b55/pgspecial-1.10.0.tar.gz"; sha256 = "eadb0108cdbcf8b38a69bcc9e403b352dbd6d30622e417f48e659180150ee1b6"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."click" + self."sqlparse" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://www.dbcli.com"; + license = licenses.bsdOriginal; + description = "Meta-commands handler for Postgres Database."; + }; + }; + + + + "prompt-toolkit" = python.mkDerivation { + name = "prompt-toolkit-1.0.15"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/8a/ad/cf6b128866e78ad6d7f1dc5b7f99885fb813393d9860778b2984582e81b5/prompt_toolkit-1.0.15.tar.gz"; sha256 = "858588f1983ca497f1cf4ffde01d978a3ea02b01c8a26a8bbc5cd2e66d816917"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."six" + self."wcwidth" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/jonathanslenders/python-prompt-toolkit"; + license = licenses.bsdOriginal; + description = "Library for building powerful interactive command lines in Python"; + }; + }; + + + + "psycopg2" = python.mkDerivation { + name = "psycopg2-2.7.3.2"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/dd/47/000b405d73ca22980684fd7bd3318690cc03cfa3b2ae1c5b7fff8050b28a/psycopg2-2.7.3.2.tar.gz"; sha256 = "5c3213be557d0468f9df8fe2487eaf2990d9799202c5ff5cb8d394d09fad9b2a"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://initd.org/psycopg/"; + license = licenses.lgpl2; + description = "psycopg2 - Python-PostgreSQL Database Adapter"; + }; + }; + + + "pyOpenSSL" = python.mkDerivation { - name = "pyOpenSSL-17.5.0"; - src = pkgs.fetchurl { url = "https://pypi.python.org/packages/3b/15/a5d90ab1a41075e8f0fae334f13452549528f82142b3b9d0c9d86ab7178c/pyOpenSSL-17.5.0.tar.gz"; sha256 = "2c10cfba46a52c0b0950118981d61e72c1e5b1aac451ca1bc77de1a679456773"; }; + name = "pyOpenSSL-17.2.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/b0/9e/7088f6165c40c46416aff434eb806c1d64ad6ec6dbc201f5ad4d0484704e/pyOpenSSL-17.2.0.tar.gz"; sha256 = "5d617ce36b07c51f330aa63b83bf7f25c40a0e95958876d54d1982f8c91b4834"; }; doCheck = commonDoCheck; buildInputs = commonBuildInputs; propagatedBuildInputs = [ @@ -526,6 +1148,87 @@ let + "pydot" = python.mkDerivation { + name = "pydot-1.2.4"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/c3/f1/e61d6dfe6c1768ed2529761a68f70939e2569da043e9f15a8d84bf56cadf/pydot-1.2.4.tar.gz"; sha256 = "92d2e2d15531d00710f2d6fb5540d2acabc5399d464f2f20d5d21073af241eb6"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."pyparsing" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/erocarrera/pydot"; + license = licenses.mit; + description = "Python interface to Graphviz's Dot"; + }; + }; + + + + "pyparsing" = python.mkDerivation { + name = "pyparsing-2.2.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/3c/ec/a94f8cf7274ea60b5413df054f82a8980523efd712ec55a59e7c3357cf7c/pyparsing-2.2.0.tar.gz"; sha256 = "0832bcf47acd283788593e7a0f542407bd9550a55a8a8435214a1960e04bcb04"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://pyparsing.wikispaces.com/"; + license = licenses.mit; + description = "Python parsing module"; + }; + }; + + + + "python-dateutil" = python.mkDerivation { + name = "python-dateutil-2.6.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/51/fc/39a3fbde6864942e8bb24c93663734b74e281b984d1b8c4f95d64b0c21f6/python-dateutil-2.6.0.tar.gz"; sha256 = "62a2f8df3d66f878373fd0072eacf4ee52194ba302e00082828e0d263b0418d2"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."six" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://dateutil.readthedocs.io"; + license = licenses.bsdOriginal; + description = "Extensions to the standard Python datetime module"; + }; + }; + + + + "python-slugify" = python.mkDerivation { + name = "python-slugify-1.2.4"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/9f/b0/2723356c20fb01b0e09f6ee03c0c629f4e30811e7d92ebd15453d648e5f0/python-slugify-1.2.4.tar.gz"; sha256 = "57a385df7a1c6dbd15f7666eaff0ff29d3f60363b228b1197c5308ed3ba5f824"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Unidecode" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/un33k/python-slugify"; + license = licenses.mit; + description = "A Python Slugify application that handles Unicode"; + }; + }; + + + + "pytz" = python.mkDerivation { + name = "pytz-2018.4"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/10/76/52efda4ef98e7544321fd8d5d512e11739c1df18b0649551aeccfb1c8376/pytz-2018.4.tar.gz"; sha256 = "c06425302f2cf668f1bba7a0a03f3c1d34d4ebeef2c72003da308b3947c7f749"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://pythonhosted.org/pytz"; + license = licenses.mit; + description = "World timezone definitions, modern and historical"; + }; + }; + + + "requests" = python.mkDerivation { name = "requests-2.18.4"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/b0/e1/eab4fc3752e3d240468a8c0b284607899d2fbfb236a56b7377a329aa8d09/requests-2.18.4.tar.gz"; sha256 = "9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e"; }; @@ -587,6 +1290,21 @@ let + "setproctitle" = python.mkDerivation { + name = "setproctitle-1.1.10"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/5a/0d/dc0d2234aacba6cf1a729964383e3452c52096dc695581248b548786f2b3/setproctitle-1.1.10.tar.gz"; sha256 = "6283b7a58477dd8478fbb9e76defb37968ee4ba47b05ec1c053cb39638bd7398"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/dvarrazzo/py-setproctitle"; + license = licenses.bsdOriginal; + description = "A Python module to customize the process title"; + }; + }; + + + "six" = python.mkDerivation { name = "six-1.11.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/16/d8/bc6316cf98419719bd59c91742194c111b6f2e85abac88e496adefaf7afe/six-1.11.0.tar.gz"; sha256 = "70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9"; }; @@ -617,6 +1335,119 @@ let + "speaklater" = python.mkDerivation { + name = "speaklater-1.3"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/11/92/5ae1effe0ccb8561c034a0111d53c8788660ddb7ed4992f0da1bb5c525e5/speaklater-1.3.tar.gz"; sha256 = "59fea336d0eed38c1f0bf3181ee1222d0ef45f3a9dd34ebe65e6bfffdd6a65a9"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://github.com/mitsuhiko/speaklater"; + license = licenses.bsdOriginal; + description = "implements a lazy string for python useful for use with gettext"; + }; + }; + + + + "sqlalchemy-schemadisplay" = python.mkDerivation { + name = "sqlalchemy-schemadisplay-1.3"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/ac/6a/de5911b2837278f3cf89b99b0fd94461f789b8f083537ff14ff9aa6d3397/sqlalchemy_schemadisplay-1.3.tar.gz"; sha256 = "0a9f26d77be9d92c9564d87cc17668fe141a816036c5f5d7c8cb053b253957e0"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."pydot" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/fschulze/sqlalchemy_schemadisplay"; + license = licenses.mit; + description = "Turn SQLAlchemy DB Model into a graph"; + }; + }; + + + + "sqlparse" = python.mkDerivation { + name = "sqlparse-0.2.4"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/79/3c/2ad76ba49f9e3d88d2b58e135b7821d93741856d1fe49970171f73529303/sqlparse-0.2.4.tar.gz"; sha256 = "ce028444cfab83be538752a2ffdb56bc417b7784ff35bb9a3062413717807dec"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/andialbrecht/sqlparse"; + license = licenses.bsdOriginal; + description = "Non-validating SQL parser"; + }; + }; + + + + "tabulate" = python.mkDerivation { + name = "tabulate-0.8.2"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/12/c2/11d6845db5edf1295bc08b2f488cf5937806586afe42936c3f34c097ebdc/tabulate-0.8.2.tar.gz"; sha256 = "e4ca13f26d0a6be2a2915428dc21e732f1e44dad7f76d7030b2ef1ec251cf7f2"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."wcwidth" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://bitbucket.org/astanin/python-tabulate"; + license = licenses.mit; + description = "Pretty-print tabular data"; + }; + }; + + + + "terminaltables" = python.mkDerivation { + name = "terminaltables-3.1.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/9b/c4/4a21174f32f8a7e1104798c445dacdc1d4df86f2f26722767034e4de4bff/terminaltables-3.1.0.tar.gz"; sha256 = "f3eb0eb92e3833972ac36796293ca0906e998dc3be91fbe1f8615b331b853b81"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/Robpol86/terminaltables"; + license = licenses.mit; + description = "Generate simple tables in terminals from a nested list of strings."; + }; + }; + + + + "tqdm" = python.mkDerivation { + name = "tqdm-4.15.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/01/f7/2058bd94a903f445e8ff19c0af64b9456187acab41090ff2da21c7c7e193/tqdm-4.15.0.tar.gz"; sha256 = "6ec1dc74efacf2cda936b4a6cf4082ce224c76763bdec9f17e437c8cfcaa9953"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/tqdm/tqdm"; + license = licenses.mpl20; + description = "Fast, Extensible Progress Meter"; + }; + }; + + + + "txaio" = python.mkDerivation { + name = "txaio-2.10.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/b8/87/efcae4040c2a0af9c871116a6dbf02ee582b396e6de3797fb30cdcc4a7e4/txaio-2.10.0.tar.gz"; sha256 = "4797f9f6a9866fe887c96abc0110a226dd5744c894dc3630870542597ad30853"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Twisted" + self."six" + self."zope.interface" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/crossbario/txaio"; + license = licenses.mit; + description = "Compatibility API between asyncio/Twisted/Trollius"; + }; + }; + + + "urllib3" = python.mkDerivation { name = "urllib3-1.22"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/ee/11/7c59620aceedcc1ef65e156cc5ce5a24ef87be4107c2b74458464e437a5d/urllib3-1.22.tar.gz"; sha256 = "cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f"; }; @@ -638,6 +1469,55 @@ let + "wcwidth" = python.mkDerivation { + name = "wcwidth-0.1.7"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/55/11/e4a2bb08bb450fdbd42cc709dd40de4ed2c472cf0ccb9e64af22279c5495/wcwidth-0.1.7.tar.gz"; sha256 = "3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/jquast/wcwidth"; + license = licenses.mit; + description = "Measures number of Terminal column cells of wide-character codes"; + }; + }; + + + + "webargs" = python.mkDerivation { + name = "webargs-2.1.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/9c/9b/a6dce2167bf86d65f1055d9f4f62b523c5891a7beeb15c8fe63f8948a37f/webargs-2.1.0.tar.gz"; sha256 = "d15d81531b7c0f73dec140bf0cd45c15f061f88eb08fcc29854d94682fd3911c"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."marshmallow" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/sloria/webargs"; + license = licenses.mit; + description = "A friendly library for parsing and validating HTTP request arguments, with built-in support for popular web frameworks, including Flask, Django, Bottle, Tornado, Pyramid, webapp2, Falcon, and aiohttp."; + }; + }; + + + + "websocket-client" = python.mkDerivation { + name = "websocket-client-0.46.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/83/91/162f2c76729633d1dc36b09746895c7766bc183bba94cb4d2ec398676060/websocket_client-0.46.0.tar.gz"; sha256 = "933f6bbf08b381f2adbca9e93d7e7958ba212b42c73acb310b18f0fbe74f3738"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."six" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/websocket-client/websocket-client.git"; + license = licenses.lgpl2; + description = "WebSocket client for python. hybi13 is supported."; + }; + }; + + + "zope.interface" = python.mkDerivation { name = "zope.interface-4.4.3"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/bd/d2/25349ed41f9dcff7b3baf87bd88a4c82396cf6e02f1f42bb68657a3132af/zope.interface-4.4.3.tar.gz"; sha256 = "d6d26d5dfbfd60c65152938fcb82f949e8dada37c041f72916fef6621ba5c5ce"; }; diff --git a/server/api/modules/info.py b/server/api/modules/info.py index 3a0f6908e19..ff51f911820 100644 --- a/server/api/modules/info.py +++ b/server/api/modules/info.py @@ -5,7 +5,6 @@ import flask from flask import Blueprint -from server.app import app from server.config import gen_web_config @@ -26,6 +25,6 @@ def show_info(): return response -@app.route('/config') +@info_api.route('/config') def get_config(): return flask.jsonify(gen_web_config()) diff --git a/server/www/scripts/config/services/config.js b/server/www/scripts/config/services/config.js index 6bafe7395f9..65c3973a4d7 100644 --- a/server/www/scripts/config/services/config.js +++ b/server/www/scripts/config/services/config.js @@ -3,7 +3,7 @@ // See the file 'doc/LICENSE' for the license information angular.module('faradayApp') - .factory('configSrv', ['$http', function($http) { + .factory('configSrv', ['$http', 'BASEURL', function($http, BASEURL) { var p = $http.get(BASEURL + '_api/config') .then(function(conf) { From 6ae3427b1abda15fa9d75dbd7602ba0b5fe2ff5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 12 Apr 2018 14:01:20 -0300 Subject: [PATCH 1058/1506] Add dev requirements to nix python environment --- nix/server/requirements.nix | 323 ++++++++++++++++++++++++++- nix/server/requirements_override.nix | 10 + 2 files changed, 330 insertions(+), 3 deletions(-) diff --git a/nix/server/requirements.nix b/nix/server/requirements.nix index fd23b12b408..7802a288eb2 100644 --- a/nix/server/requirements.nix +++ b/nix/server/requirements.nix @@ -2,7 +2,7 @@ # See more at: https://github.com/garbas/pypi2nix # # COMMAND: -# pypi2nix -V 2.7 -r ../../requirements_server.txt -E libffi -E openssl -E postgresql -E pkgconfig zlib libjpeg openjpeg libtiff freetype lcms2 libwebp tcl +# pypi2nix -r ../../requirements_server.txt -r ../../requirements_dev.txt -E libffi -E openssl -E postgresql -E pkgconfig -E zlib -E libjpeg -E openjpeg -E libtiff -E freetype -E lcms2 -E libwebp -E tcl -V 2.7 # { pkgs ? import {} @@ -111,6 +111,26 @@ let + "Faker" = python.mkDerivation { + name = "Faker-0.8.13"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/fc/4a/28395bebd1d42834e4ff3d08253a081e36f6765c0ff4c87fa8b503373e31/Faker-0.8.13.tar.gz"; sha256 = "48fed4b4a191e2b42ad20c14115f1c6d36d338b80192075d7573f0f42d7fb321"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."ipaddress" + self."python-dateutil" + self."six" + self."text-unidecode" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/joke2k/faker"; + license = licenses.mit; + description = "Faker is a Python package that generates fake data for you."; + }; + }; + + + "Flask" = python.mkDerivation { name = "Flask-0.12.2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/eb/12/1c7bd06fcbd08ba544f25bf2c6612e305a70ea51ca0eda8007344ec3f123/Flask-0.12.2.tar.gz"; sha256 = "49f44461237b69ecd901cc7ce66feea0319b9158743dd27a2899962ab214dac1"; }; @@ -254,8 +274,11 @@ let self."Flask-WTF" self."SQLAlchemy" self."bcrypt" + self."coverage" self."itsdangerous" + self."mock" self."passlib" + self."pytest" ]; meta = with pkgs.stdenv.lib; { homepage = "https://github.com/mattupstate/flask-security"; @@ -444,7 +467,10 @@ let src = pkgs.fetchurl { url = "https://pypi.python.org/packages/9f/08/a3bb1c045ec602dc680906fc0261c267bed6b3bb4609430aff92c3888ec8/Werkzeug-0.14.1.tar.gz"; sha256 = "c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c"; }; doCheck = commonDoCheck; buildInputs = commonBuildInputs; - propagatedBuildInputs = [ ]; + propagatedBuildInputs = [ + self."coverage" + self."pytest" + ]; meta = with pkgs.stdenv.lib; { homepage = "https://www.palletsprojects.org/p/werkzeug/"; license = licenses.bsdOriginal; @@ -475,6 +501,9 @@ let doCheck = commonDoCheck; buildInputs = commonBuildInputs; propagatedBuildInputs = [ + self."coverage" + self."hypothesis" + self."pytest" self."six" self."zope.interface" ]; @@ -494,7 +523,9 @@ let buildInputs = commonBuildInputs; propagatedBuildInputs = [ self."Twisted" + self."mock" self."pyOpenSSL" + self."pytest" self."service-identity" self."six" self."txaio" @@ -546,6 +577,7 @@ let buildInputs = commonBuildInputs; propagatedBuildInputs = [ self."cffi" + self."pytest" self."six" ]; meta = with pkgs.stdenv.lib; { @@ -557,6 +589,21 @@ let + "beautifulsoup4" = python.mkDerivation { + name = "beautifulsoup4-4.6.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/fa/8d/1d14391fdaed5abada4e0f63543fef49b8331a34ca60c88bd521bcf7f782/beautifulsoup4-4.6.0.tar.gz"; sha256 = "808b6ac932dccb0a4126558f7dfdcf41710dd44a4ef497a0bb59a77f9f078e89"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://www.crummy.com/software/BeautifulSoup/bs4/"; + license = licenses.mit; + description = "Screen-scraping library"; + }; + }; + + + "blinker" = python.mkDerivation { name = "blinker-1.4"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/1b/51/e2a9f3b757eb802f61dc1f2b09c8c99f6eb01cf06416c0671253536517b6/blinker-1.4.tar.gz"; sha256 = "471aee25f3992bd325afa3772f1063dbdbbca947a041b8b89466dc00d606f8b6"; }; @@ -701,6 +748,21 @@ let + "cookies" = python.mkDerivation { + name = "cookies-2.2.1"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/f3/95/b66a0ca09c5ec9509d8729e0510e4b078d2451c5e33f47bd6fc33c01517c/cookies-2.2.1.tar.gz"; sha256 = "d6b698788cae4cfa4e62ef8643a9ca332b79bd96cb314294b864ae8d7eb3ee8e"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/sashahart/cookies"; + license = licenses.mit; + description = "Friendlier RFC 6265-compliant cookie parser/renderer"; + }; + }; + + + "couchdbkit" = python.mkDerivation { name = "couchdbkit-0.6.5"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/a1/13/9e9ff695a385c44f62b4766341b97f2bd8b596962df2a0beabf358468b70/couchdbkit-0.6.5.tar.gz"; sha256 = "9b607f509727e6ada2dbd576a4120c214b1c54f3bb8bf6e2e0eb2cfbb11a0e00"; }; @@ -718,6 +780,21 @@ let + "coverage" = python.mkDerivation { + name = "coverage-4.5.1"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/35/fe/e7df7289d717426093c68d156e0fd9117c8f4872b6588e8a8928a0f68424/coverage-4.5.1.tar.gz"; sha256 = "56e448f051a201c5ebbaa86a5efd0ca90d327204d8b059ab25ad0f35fbfd79f1"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://bitbucket.org/ned/coveragepy"; + license = licenses.asl20; + description = "Code coverage measurement for Python"; + }; + }; + + + "cryptography" = python.mkDerivation { name = "cryptography-2.2.2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/ec/b2/faa78c1ab928d2b2c634c8b41ff1181f0abdd9adf9193211bd606ffa57e2/cryptography-2.2.2.tar.gz"; sha256 = "9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63"; }; @@ -727,8 +804,10 @@ let self."asn1crypto" self."cffi" self."enum34" + self."hypothesis" self."idna" self."ipaddress" + self."pytest" self."pytz" self."six" ]; @@ -771,6 +850,23 @@ let + "factory-boy" = python.mkDerivation { + name = "factory-boy-2.10.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/ea/b6/d94a2df8981af1698f7bc14820033cf288263212fc97980dea1a1a5a7eed/factory_boy-2.10.0.tar.gz"; sha256 = "bd5a096d0f102d79b6c78cef1c8c0b650f2e1a3ecba351c735c6d2df8dabd29c"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Faker" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/FactoryBoy/factory_boy"; + license = licenses.mit; + description = "A versatile test fixtures replacement based on thoughtbot's factory_bot for Ruby."; + }; + }; + + + "filedepot" = python.mkDerivation { name = "filedepot-0.5.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/75/89/c744b5326d956dd1e8b2e591cb50759965d16ccfd49e7d3124a85a821069/filedepot-0.5.0.tar.gz"; sha256 = "e319a2b163c37fa1f3da21ad7c81a49a12de66da87d1af428fb143f092b96e5d"; }; @@ -780,6 +876,8 @@ let self."Pillow" self."SQLAlchemy" self."Unidecode" + self."coverage" + self."mock" self."requests" ]; meta = with pkgs.stdenv.lib; { @@ -810,6 +908,21 @@ let + "funcsigs" = python.mkDerivation { + name = "funcsigs-1.0.2"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/94/4a/db842e7a0545de1cdb0439bb80e6e42dfe82aaeaadd4072f2263a4fbed23/funcsigs-1.0.2.tar.gz"; sha256 = "a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://funcsigs.readthedocs.org"; + license = "License :: OSI Approved :: Apache Software License"; + description = "Python function signatures from PEP362 for Python 2.6, 2.7 and 3.2+"; + }; + }; + + + "http-parser" = python.mkDerivation { name = "http-parser-0.8.3"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/07/c4/22e3c76c2313c26dd5f84f1205b916ff38ea951aab0c4544b6e2f5920d64/http-parser-0.8.3.tar.gz"; sha256 = "e2aff90a60def3e476bd71694d8757c0f95ebf2fedf0a8ae34ee306e0b20db83"; }; @@ -857,6 +970,28 @@ let + "hypothesis" = python.mkDerivation { + name = "hypothesis-3.48.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/c7/0b/64a7db8a09397ab3be1c2bf77ca74b6f494758010049003c63573a0e6ca0/hypothesis-3.48.0.tar.gz"; sha256 = "b3c1e6fb2f2e88b3957687e5e05c2385c4c509173539d72cfb4c808a5dff3dcd"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Faker" + self."attrs" + self."coverage" + self."enum34" + self."pytest" + self."pytz" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/HypothesisWorks/hypothesis-python"; + license = licenses.mpl20; + description = "A library for property based testing"; + }; + }; + + + "idna" = python.mkDerivation { name = "idna-2.6"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/f4/bd/0467d62790828c23c47fc1dfa1b1f052b24efdf5290f071c7a91d0d82fd3/idna-2.6.tar.gz"; sha256 = "2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f"; }; @@ -890,6 +1025,21 @@ let + "inflection" = python.mkDerivation { + name = "inflection-0.3.1"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/d5/35/a6eb45b4e2356fe688b21570864d4aa0d0a880ce387defe9c589112077f8/inflection-0.3.1.tar.gz"; sha256 = "18ea7fb7a7d152853386523def08736aa8c32636b047ade55f7578c4edeb16ca"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://github.com/jpvanhal/inflection"; + license = licenses.mit; + description = "A port of Ruby on Rails inflector to Python"; + }; + }; + + + "ipaddress" = python.mkDerivation { name = "ipaddress-1.0.19"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/f0/ba/860a4a3e283456d6b7e2ab39ce5cf11a3490ee1a363652ac50abf9f0f5df/ipaddress-1.0.19.tar.gz"; sha256 = "200d8686011d470b5e4de207d803445deee427455cd0cb7c982b68cf82524f81"; }; @@ -955,6 +1105,44 @@ let + "mock" = python.mkDerivation { + name = "mock-2.0.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/0c/53/014354fc93c591ccc4abff12c473ad565a2eb24dcd82490fae33dbf2539f/mock-2.0.0.tar.gz"; sha256 = "b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."Jinja2" + self."Pygments" + self."funcsigs" + self."pbr" + self."six" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/testing-cabal/mock"; + license = licenses.bsdOriginal; + description = "Rolling backport of unittest.mock for all Pythons"; + }; + }; + + + + "more-itertools" = python.mkDerivation { + name = "more-itertools-4.1.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/db/0b/f5660bf6299ec5b9f17bd36096fa8148a1c843fa77ddfddf9bebac9301f7/more-itertools-4.1.0.tar.gz"; sha256 = "c9ce7eccdcb901a2c75d326ea134e0886abfbea5f93e91cc95de9507c0816c44"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."six" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/erikrose/more-itertools"; + license = licenses.mit; + description = "More routines for operating on iterables, beyond itertools"; + }; + }; + + + "nplusone" = python.mkDerivation { name = "nplusone-0.8.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/b0/9a/8f8bf4dfed57ee26d16c4ddcf38ff4ffee38048e15f4e1a6686e1fbeeadb/nplusone-0.8.1.tar.gz"; sha256 = "6edfa3bea1a99bb22e3b218bcafdefeddb55ebc3b362dc8921844b3b5000e29d"; }; @@ -1006,6 +1194,21 @@ let + "pbr" = python.mkDerivation { + name = "pbr-4.0.2"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/c6/46/f414e7d9ba9621c8acd3e7a82e08c47e0de34ad3e213c16e458b6c04d432/pbr-4.0.2.tar.gz"; sha256 = "dae4aaa78eafcad10ce2581fc34d694faa616727837fd8e55c1a00951ad6744f"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://docs.openstack.org/pbr/latest/"; + license = "License :: OSI Approved :: Apache Software License"; + description = "Python Build Reasonableness"; + }; + }; + + + "pgcli" = python.mkDerivation { name = "pgcli-1.8.2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/91/84/e3be2851c87a9550cc747ba85ade36d1b51bb9a747ac93025a01a25baa41/pgcli-1.8.2.tar.gz"; sha256 = "79b32cb0a44e03fc6c26d43080459d48d39d6d81391eb09062a82c940b61441c"; }; @@ -1050,6 +1253,21 @@ let + "pluggy" = python.mkDerivation { + name = "pluggy-0.6.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/11/bf/cbeb8cdfaffa9f2ea154a30ae31a9d04a1209312e2919138b4171a1f8199/pluggy-0.6.0.tar.gz"; sha256 = "7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/pytest-dev/pluggy"; + license = licenses.mit; + description = "plugin and hook calling mechanisms for python"; + }; + }; + + + "prompt-toolkit" = python.mkDerivation { name = "prompt-toolkit-1.0.15"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/8a/ad/cf6b128866e78ad6d7f1dc5b7f99885fb813393d9860778b2984582e81b5/prompt_toolkit-1.0.15.tar.gz"; sha256 = "858588f1983ca497f1cf4ffde01d978a3ea02b01c8a26a8bbc5cd2e66d816917"; }; @@ -1083,6 +1301,21 @@ let + "py" = python.mkDerivation { + name = "py-1.5.3"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/f7/84/b4c6e84672c4ceb94f727f3da8344037b62cee960d80e999b1cd9b832d83/py-1.5.3.tar.gz"; sha256 = "29c9fab495d7528e80ba1e343b958684f4ace687327e6f789a94bf3d1915f881"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://py.readthedocs.io/"; + license = licenses.mit; + description = "library with cross-python path, ini-parsing, io, code, log facilities"; + }; + }; + + + "pyOpenSSL" = python.mkDerivation { name = "pyOpenSSL-17.2.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/b0/9e/7088f6165c40c46416aff434eb806c1d64ad6ec6dbc201f5ad4d0484704e/pyOpenSSL-17.2.0.tar.gz"; sha256 = "5d617ce36b07c51f330aa63b83bf7f25c40a0e95958876d54d1982f8c91b4834"; }; @@ -1090,6 +1323,7 @@ let buildInputs = commonBuildInputs; propagatedBuildInputs = [ self."cryptography" + self."pytest" self."six" ]; meta = with pkgs.stdenv.lib; { @@ -1180,6 +1414,48 @@ let + "pytest" = python.mkDerivation { + name = "pytest-3.5.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/2d/56/6019153cdd743300c5688ab3b07702355283e53c83fbf922242c053ffb7b/pytest-3.5.0.tar.gz"; sha256 = "fae491d1874f199537fd5872b5e1f0e74a009b979df9d53d1553fd03da1703e1"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."attrs" + self."colorama" + self."funcsigs" + self."more-itertools" + self."pluggy" + self."py" + self."six" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "http://pytest.org"; + license = licenses.mit; + description = "pytest: simple powerful testing with Python"; + }; + }; + + + + "pytest-factoryboy" = python.mkDerivation { + name = "pytest-factoryboy-2.0.1"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/80/15/294f84a4d3159f6be5524b8f02b70aae82c86e2341c27ac00fdd26b0d44f/pytest-factoryboy-2.0.1.tar.gz"; sha256 = "ad438d191d2b2a0f26956d437c1963875db573147a84ffd85d7bbeaefae22458"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."factory-boy" + self."inflection" + self."pytest" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/pytest-dev/pytest-factoryboy"; + license = licenses.mit; + description = "Factory Boy support for pytest."; + }; + }; + + + "python-dateutil" = python.mkDerivation { name = "python-dateutil-2.6.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/51/fc/39a3fbde6864942e8bb24c93663734b74e281b984d1b8c4f95d64b0c21f6/python-dateutil-2.6.0.tar.gz"; sha256 = "62a2f8df3d66f878373fd0072eacf4ee52194ba302e00082828e0d263b0418d2"; }; @@ -1251,6 +1527,28 @@ let + "responses" = python.mkDerivation { + name = "responses-0.9.0"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/67/cb/0a5390f7b8944cfa7e4079a839adba964d858d27a60af7b2683248148339/responses-0.9.0.tar.gz"; sha256 = "c6082710f4abfb60793899ca5f21e7ceb25aabf321560cc0726f8b59006811c9"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ + self."cookies" + self."coverage" + self."mock" + self."pytest" + self."requests" + self."six" + ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/getsentry/responses"; + license = licenses.asl20; + description = "A utility library for mocking out the `requests` Python library."; + }; + }; + + + "restkit" = python.mkDerivation { name = "restkit-4.2.2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/76/b9/d90120add1be718f853c53008cf5b62d74abad1d32bd1e7097dd913ae053/restkit-4.2.2.tar.gz"; sha256 = "c0bda8eb7c643b5e818b612dab49121393abc8589c6cbe9b84085079d598599d"; }; @@ -1414,6 +1712,21 @@ let + "text-unidecode" = python.mkDerivation { + name = "text-unidecode-1.2"; + src = pkgs.fetchurl { url = "https://pypi.python.org/packages/f0/a2/40adaae7cbdd007fb12777e550b5ce344b56189921b9f70f37084c021ca4/text-unidecode-1.2.tar.gz"; sha256 = "5a1375bb2ba7968740508ae38d92e1f889a0832913cb1c447d5e2046061a396d"; }; + doCheck = commonDoCheck; + buildInputs = commonBuildInputs; + propagatedBuildInputs = [ ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/kmike/text-unidecode/"; + license = licenses.artistic2; + description = "The most basic Text::Unidecode port"; + }; + }; + + + "tqdm" = python.mkDerivation { name = "tqdm-4.15.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/01/f7/2058bd94a903f445e8ff19c0af64b9456187acab41090ff2da21c7c7e193/tqdm-4.15.0.tar.gz"; sha256 = "6ec1dc74efacf2cda936b4a6cf4082ce224c76763bdec9f17e437c8cfcaa9953"; }; @@ -1436,6 +1749,8 @@ let buildInputs = commonBuildInputs; propagatedBuildInputs = [ self."Twisted" + self."mock" + self."pytest" self."six" self."zope.interface" ]; @@ -1523,7 +1838,9 @@ let src = pkgs.fetchurl { url = "https://pypi.python.org/packages/bd/d2/25349ed41f9dcff7b3baf87bd88a4c82396cf6e02f1f42bb68657a3132af/zope.interface-4.4.3.tar.gz"; sha256 = "d6d26d5dfbfd60c65152938fcb82f949e8dada37c041f72916fef6621ba5c5ce"; }; doCheck = commonDoCheck; buildInputs = commonBuildInputs; - propagatedBuildInputs = [ ]; + propagatedBuildInputs = [ + self."coverage" + ]; meta = with pkgs.stdenv.lib; { homepage = "https://github.com/zopefoundation/zope.interface"; license = licenses.zpl21; diff --git a/nix/server/requirements_override.nix b/nix/server/requirements_override.nix index 57addd0a206..957fb4e5ef9 100644 --- a/nix/server/requirements_override.nix +++ b/nix/server/requirements_override.nix @@ -16,4 +16,14 @@ self: super: { (drv: drv.name != self.Twisted.name) old.propagatedBuildInputs); }); + hypothesis = python.overrideDerivation super.hypothesis (old: { + propagatedBuildInputs = (builtins.filter + (drv: drv.name != self.attrs.name) + old.propagatedBuildInputs); + }); + attrs = python.overrideDerivation super.attrs (old: { + propagatedBuildInputs = (builtins.filter + (drv: drv.name != self.hypothesis.name && drv.name != self.pytest.name) + old.propagatedBuildInputs); + }); } From cc9753526443fc5842eea796a3c3656eb19dece8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 12 Apr 2018 14:30:40 -0300 Subject: [PATCH 1059/1506] Use correct version of filteralchemy and remove nonsense test --- nix/server/requirements.nix | 241 ++----------------------------- test_cases/test_server_config.py | 23 --- 2 files changed, 16 insertions(+), 248 deletions(-) diff --git a/nix/server/requirements.nix b/nix/server/requirements.nix index 7802a288eb2..cb16aca7728 100644 --- a/nix/server/requirements.nix +++ b/nix/server/requirements.nix @@ -23,7 +23,10 @@ let self: super: { bootstrapped-pip = super.bootstrapped-pip.overrideDerivation (old: { patchPhase = old.patchPhase + '' - sed -i -e "s|paths_to_remove.remove(auto_confirm)|#paths_to_remove.remove(auto_confirm)|" -e "s|self.uninstalled = paths_to_remove|#self.uninstalled = paths_to_remove|" $out/${pkgs.python35.sitePackages}/pip/req/req_install.py + sed -i \ + -e "s|paths_to_remove.remove(auto_confirm)|#paths_to_remove.remove(auto_confirm)|" \ + -e "s|self.uninstalled = paths_to_remove|#self.uninstalled = paths_to_remove|" \ + $out/${pkgs.python35.sitePackages}/pip/req/req_install.py ''; }); }; @@ -40,11 +43,13 @@ let buildInputs = [ makeWrapper ] ++ (builtins.attrValues pkgs); buildCommand = '' mkdir -p $out/bin - ln -s ${pythonPackages.python.interpreter} $out/bin/${pythonPackages.python.executable} - for dep in ${builtins.concatStringsSep " " (builtins.attrValues pkgs)}; do + ln -s ${pythonPackages.python.interpreter} \ + $out/bin/${pythonPackages.python.executable} + for dep in ${builtins.concatStringsSep " " + (builtins.attrValues pkgs)}; do if [ -d "$dep/bin" ]; then for prog in "$dep/bin/"*; do - if [ -f $prog ]; then + if [ -x "$prog" ] && [ -f "$prog" ]; then ln -s $prog $out/bin/`basename $prog` fi done @@ -55,7 +60,8 @@ let done pushd $out/bin ln -s ${pythonPackages.python.executable} python - ln -s ${pythonPackages.python.executable} python2 + ln -s ${pythonPackages.python.executable} \ + python2 popd ''; passthru.interpreter = pythonPackages.python; @@ -66,7 +72,9 @@ let mkDerivation = pythonPackages.buildPythonPackage; packages = pkgs; overrideDerivation = drv: f: - pythonPackages.buildPythonPackage (drv.drvAttrs // f drv.drvAttrs // { meta = drv.meta; }); + pythonPackages.buildPythonPackage ( + drv.drvAttrs // f drv.drvAttrs // { meta = drv.meta; } + ); withPackages = pkgs'': withPackages (pkgs // pkgs''); }; @@ -74,7 +82,6 @@ let python = withPackages {}; generated = self: { - "Automat" = python.mkDerivation { name = "Automat-0.6.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/de/05/b8e453085cf8a7f27bb1226596f4ccf5cc9e758377d60284f990bbdc592c/Automat-0.6.0.tar.gz"; sha256 = "3c1fd04ecf08ac87b4dd3feae409542e9bf7827257097b2b6ed5692f69d6f6a8"; }; @@ -92,8 +99,6 @@ let }; }; - - "Babel" = python.mkDerivation { name = "Babel-2.5.3"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/0e/d5/9b1d6a79c975d0e9a32bd337a1465518c2519b14b214682ca9892752417e/Babel-2.5.3.tar.gz"; sha256 = "8ce4cb6fdd4393edd323227cba3a077bceb2a6ce5201c902c65e730046f41f14"; }; @@ -109,8 +114,6 @@ let }; }; - - "Faker" = python.mkDerivation { name = "Faker-0.8.13"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/fc/4a/28395bebd1d42834e4ff3d08253a081e36f6765c0ff4c87fa8b503373e31/Faker-0.8.13.tar.gz"; sha256 = "48fed4b4a191e2b42ad20c14115f1c6d36d338b80192075d7573f0f42d7fb321"; }; @@ -129,8 +132,6 @@ let }; }; - - "Flask" = python.mkDerivation { name = "Flask-0.12.2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/eb/12/1c7bd06fcbd08ba544f25bf2c6612e305a70ea51ca0eda8007344ec3f123/Flask-0.12.2.tar.gz"; sha256 = "49f44461237b69ecd901cc7ce66feea0319b9158743dd27a2899962ab214dac1"; }; @@ -149,8 +150,6 @@ let }; }; - - "Flask-BabelEx" = python.mkDerivation { name = "Flask-BabelEx-0.9.3"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/80/ad/cc2b0becd98050eed775ca85d6e5fa784547acff69f968183098df8a52b3/Flask-BabelEx-0.9.3.tar.gz"; sha256 = "cf79cdedb5ce860166120136b0e059e9d97b8df07a3bc2411f6243de04b754b4"; }; @@ -169,8 +168,6 @@ let }; }; - - "Flask-Classful" = python.mkDerivation { name = "Flask-Classful-0.14.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/b7/2a/26d548a19ce7411f49ddb0abd3c3319f4ab47354cc4354225e5a7a91a6bf/Flask-Classful-0.14.0.tar.gz"; sha256 = "036589065f0f6e35e37c1146616cb1b82b2bf1111f68e02a8939478b32bf524d"; }; @@ -186,8 +183,6 @@ let }; }; - - "Flask-Login" = python.mkDerivation { name = "Flask-Login-0.4.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/c1/ff/bd9a4d2d81bf0c07d9e53e8cd3d675c56553719bbefd372df69bf1b3c1e4/Flask-Login-0.4.1.tar.gz"; sha256 = "c815c1ac7b3e35e2081685e389a665f2c74d7e077cb93cecabaea352da4752ec"; }; @@ -203,8 +198,6 @@ let }; }; - - "Flask-Mail" = python.mkDerivation { name = "Flask-Mail-0.9.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/05/2f/6a545452040c2556559779db87148d2a85e78a26f90326647b51dc5e81e9/Flask-Mail-0.9.1.tar.gz"; sha256 = "22e5eb9a940bf407bcf30410ecc3708f3c56cc44b29c34e1726fe85006935f41"; }; @@ -221,8 +214,6 @@ let }; }; - - "Flask-Principal" = python.mkDerivation { name = "Flask-Principal-0.4.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/14/c7/2531aca6ab7baa3774fde2dfc9c9dd6d5a42576a1013a93701bfdc402fdd/Flask-Principal-0.4.0.tar.gz"; sha256 = "f5d6134b5caebfdbb86f32d56d18ee44b080876a27269560a96ea35f75c99453"; }; @@ -239,8 +230,6 @@ let }; }; - - "Flask-SQLAlchemy" = python.mkDerivation { name = "Flask-SQLAlchemy-2.3.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/d0/24/fb7c9d97f0f2ac2f484b867796f6fde1b5064ec753eaa68ce49ac8584b5e/Flask-SQLAlchemy-2.3.1.tar.gz"; sha256 = "ab879cf88d30f2961dd9b4d709dcd31a25e0306855324c7d9a74fca6ad6ef8c3"; }; @@ -257,8 +246,6 @@ let }; }; - - "Flask-Security" = python.mkDerivation { name = "Flask-Security-3.0.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/ba/c1/16e460fec7961509b10aaf8cc986fa7a1df5dced2844f42cd46732621211/Flask-Security-3.0.0.tar.gz"; sha256 = "d61daa5f5a48f89f30f50555872bdf581b2c65804668b0313345cd7beff26432"; }; @@ -287,8 +274,6 @@ let }; }; - - "Flask-WTF" = python.mkDerivation { name = "Flask-WTF-0.14.2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/ba/15/00a9693180f214225a2c0b1bb9077f3b0b21f2e86522cbba22e8ad6e570c/Flask-WTF-0.14.2.tar.gz"; sha256 = "5d14d55cfd35f613d99ee7cba0fc3fbbe63ba02f544d349158c14ca15561cc36"; }; @@ -305,8 +290,6 @@ let }; }; - - "IPy" = python.mkDerivation { name = "IPy-0.83"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/88/28/79162bfc351a3f1ab44d663ab3f03fb495806fdb592170990a1568ffbf63/IPy-0.83.tar.gz"; sha256 = "61da5a532b159b387176f6eabf11946e7458b6df8fb8b91ff1d345ca7a6edab8"; }; @@ -320,8 +303,6 @@ let }; }; - - "Jinja2" = python.mkDerivation { name = "Jinja2-2.10"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/56/e6/332789f295cf22308386cf5bbd1f4e00ed11484299c5d7383378cf48ba47/Jinja2-2.10.tar.gz"; sha256 = "f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4"; }; @@ -338,8 +319,6 @@ let }; }; - - "MarkupSafe" = python.mkDerivation { name = "MarkupSafe-1.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/4d/de/32d741db316d8fdb7680822dd37001ef7a448255de9699ab4bfcbdf4172b/MarkupSafe-1.0.tar.gz"; sha256 = "a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665"; }; @@ -353,8 +332,6 @@ let }; }; - - "Pillow" = python.mkDerivation { name = "Pillow-4.2.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/55/aa/f7f983fb72710a9daa4b3374b7c160091d3f94f5c09221f9336ade9027f3/Pillow-4.2.1.tar.gz"; sha256 = "c724f65870e545316f9e82e4c6d608ab5aa9dd82d5185e5b2e72119378740073"; }; @@ -370,8 +347,6 @@ let }; }; - - "Pygments" = python.mkDerivation { name = "Pygments-2.2.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/71/2a/2e4e77803a8bd6408a2903340ac498cb0a2181811af7c9ec92cb70b0308a/Pygments-2.2.0.tar.gz"; sha256 = "dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc"; }; @@ -385,8 +360,6 @@ let }; }; - - "SQLAlchemy" = python.mkDerivation { name = "SQLAlchemy-1.2.0b2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/59/11/f7877b47db0df1576568846f06dca8fc403bb404118c641418c3183a0eeb/SQLAlchemy-1.2.0b2.tar.gz"; sha256 = "2a01c36941b87cdc5ab32aec5b0f315f1a3dfb8d8b8efc97be0419cfd78fc590"; }; @@ -402,8 +375,6 @@ let }; }; - - "Twisted" = python.mkDerivation { name = "Twisted-17.5.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/31/bf/7f86a8f8b9778e90d8b2921e9f442a8c8aa33fd2489fc10f236bc8af1749/Twisted-17.5.0.tar.bz2"; sha256 = "f198a494f0df2482f7c5f99d7f3eef33d22763ffc76641b36fec476b878002ea"; }; @@ -428,8 +399,6 @@ let }; }; - - "Unidecode" = python.mkDerivation { name = "Unidecode-1.0.22"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/9d/36/49d0ee152b6a1631f03a541532c6201942430060aa97fe011cf01a2cce64/Unidecode-1.0.22.tar.gz"; sha256 = "8c33dd588e0c9bc22a76eaa0c715a5434851f726131bd44a6c26471746efabf5"; }; @@ -443,8 +412,6 @@ let }; }; - - "WTForms" = python.mkDerivation { name = "WTForms-2.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/bf/91/2e553b86c55e9cf2f33265de50e052441fb753af46f5f20477fe9c61280e/WTForms-2.1.zip"; sha256 = "ffdf10bd1fa565b8233380cb77a304cd36fd55c73023e91d4b803c96bc11d46f"; }; @@ -460,8 +427,6 @@ let }; }; - - "Werkzeug" = python.mkDerivation { name = "Werkzeug-0.14.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/9f/08/a3bb1c045ec602dc680906fc0261c267bed6b3bb4609430aff92c3888ec8/Werkzeug-0.14.1.tar.gz"; sha256 = "c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c"; }; @@ -478,8 +443,6 @@ let }; }; - - "asn1crypto" = python.mkDerivation { name = "asn1crypto-0.24.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/fc/f1/8db7daa71f414ddabfa056c4ef792e1461ff655c2ae2928a2b675bfed6b4/asn1crypto-0.24.0.tar.gz"; sha256 = "9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49"; }; @@ -493,8 +456,6 @@ let }; }; - - "attrs" = python.mkDerivation { name = "attrs-17.4.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/8b/0b/a06cfcb69d0cb004fde8bc6f0fd192d96d565d1b8aa2829f0f20adb796e5/attrs-17.4.0.tar.gz"; sha256 = "1c7960ccfd6a005cd9f7ba884e6316b5e430a3f1a6c37c5f87d8b43f83b54ec9"; }; @@ -514,8 +475,6 @@ let }; }; - - "autobahn" = python.mkDerivation { name = "autobahn-17.10.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/e4/2e/01a64212b1eb580d601fa20f146c962235e3493795f46e3b254597ec635d/autobahn-17.10.1.tar.gz"; sha256 = "8cf74132a18da149c5ea3dcbb5e055f6f4fe5a0238b33258d29e89bd276a8078"; }; @@ -538,8 +497,6 @@ let }; }; - - "backports.csv" = python.mkDerivation { name = "backports.csv-1.0.5"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/6a/0b/2071ad285e87dd26f5c02147ba13abf7ec777ff20416a60eb15ea204ca76/backports.csv-1.0.5.tar.gz"; sha256 = "8c421385cbc6042ba90c68c871c5afc13672acaf91e1508546d6cda6725ebfc6"; }; @@ -553,23 +510,6 @@ let }; }; - - - "backports.ssl-match-hostname" = python.mkDerivation { - name = "backports.ssl-match-hostname-3.5.0.1"; - src = pkgs.fetchurl { url = "https://pypi.python.org/packages/76/21/2dc61178a2038a5cb35d14b61467c6ac632791ed05131dda72c20e7b9e23/backports.ssl_match_hostname-3.5.0.1.tar.gz"; sha256 = "502ad98707319f4a51fa2ca1c677bd659008d27ded9f6380c79e8932e38dcdf2"; }; - doCheck = commonDoCheck; - buildInputs = commonBuildInputs; - propagatedBuildInputs = [ ]; - meta = with pkgs.stdenv.lib; { - homepage = "http://bitbucket.org/brandon/backports.ssl_match_hostname"; - license = licenses.psfl; - description = "The ssl.match_hostname() function from Python 3.5"; - }; - }; - - - "bcrypt" = python.mkDerivation { name = "bcrypt-3.1.4"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/f3/ec/bb6b384b5134fd881b91b6aa3a88ccddaad0103857760711a5ab8c799358/bcrypt-3.1.4.tar.gz"; sha256 = "67ed1a374c9155ec0840214ce804616de49c3df9c5bc66740687c1c9b1cd9e8d"; }; @@ -587,8 +527,6 @@ let }; }; - - "beautifulsoup4" = python.mkDerivation { name = "beautifulsoup4-4.6.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/fa/8d/1d14391fdaed5abada4e0f63543fef49b8331a34ca60c88bd521bcf7f782/beautifulsoup4-4.6.0.tar.gz"; sha256 = "808b6ac932dccb0a4126558f7dfdcf41710dd44a4ef497a0bb59a77f9f078e89"; }; @@ -602,8 +540,6 @@ let }; }; - - "blinker" = python.mkDerivation { name = "blinker-1.4"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/1b/51/e2a9f3b757eb802f61dc1f2b09c8c99f6eb01cf06416c0671253536517b6/blinker-1.4.tar.gz"; sha256 = "471aee25f3992bd325afa3772f1063dbdbbca947a041b8b89466dc00d606f8b6"; }; @@ -617,8 +553,6 @@ let }; }; - - "certifi" = python.mkDerivation { name = "certifi-2018.1.18"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/15/d4/2f888fc463d516ff7bf2379a4e9a552fef7f22a94147655d9b1097108248/certifi-2018.1.18.tar.gz"; sha256 = "edbc3f203427eef571f79a7692bb160a2b0f7ccaa31953e99bd17e307cf63f7d"; }; @@ -632,8 +566,6 @@ let }; }; - - "cffi" = python.mkDerivation { name = "cffi-1.11.5"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/e7/a7/4cd50e57cc6f436f1cc3a7e8fa700ff9b8b4d471620629074913e3735fb2/cffi-1.11.5.tar.gz"; sha256 = "e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4"; }; @@ -649,8 +581,6 @@ let }; }; - - "chardet" = python.mkDerivation { name = "chardet-3.0.4"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d/chardet-3.0.4.tar.gz"; sha256 = "84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"; }; @@ -664,8 +594,6 @@ let }; }; - - "cli-helpers" = python.mkDerivation { name = "cli-helpers-1.0.2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/d5/13/3783ef3fa23ab76b56d4b8f96ee90808d2c167bafc5eaa4ad3c78b75abe6/cli_helpers-1.0.2.tar.gz"; sha256 = "f77837c5fbcbea39e0cb782506515459a0da75465489bae35e46da7f51c5b9fc"; }; @@ -684,8 +612,6 @@ let }; }; - - "click" = python.mkDerivation { name = "click-6.7"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/95/d9/c3336b6b5711c3ab9d1d3a80f1a3e2afeb9d8c02a7166462f6cc96570897/click-6.7.tar.gz"; sha256 = "f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b"; }; @@ -699,8 +625,6 @@ let }; }; - - "colorama" = python.mkDerivation { name = "colorama-0.3.9"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/e6/76/257b53926889e2835355d74fec73d82662100135293e17d382e2b74d1669/colorama-0.3.9.tar.gz"; sha256 = "48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1"; }; @@ -714,8 +638,6 @@ let }; }; - - "configobj" = python.mkDerivation { name = "configobj-5.0.6"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/64/61/079eb60459c44929e684fa7d9e2fdca403f67d64dd9dbac27296be2e0fab/configobj-5.0.6.tar.gz"; sha256 = "a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902"; }; @@ -731,8 +653,6 @@ let }; }; - - "constantly" = python.mkDerivation { name = "constantly-15.1.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/95/f1/207a0a478c4bb34b1b49d5915e2db574cadc415c9ac3a7ef17e29b2e8951/constantly-15.1.0.tar.gz"; sha256 = "586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35"; }; @@ -746,8 +666,6 @@ let }; }; - - "cookies" = python.mkDerivation { name = "cookies-2.2.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/f3/95/b66a0ca09c5ec9509d8729e0510e4b078d2451c5e33f47bd6fc33c01517c/cookies-2.2.1.tar.gz"; sha256 = "d6b698788cae4cfa4e62ef8643a9ca332b79bd96cb314294b864ae8d7eb3ee8e"; }; @@ -761,8 +679,6 @@ let }; }; - - "couchdbkit" = python.mkDerivation { name = "couchdbkit-0.6.5"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/a1/13/9e9ff695a385c44f62b4766341b97f2bd8b596962df2a0beabf358468b70/couchdbkit-0.6.5.tar.gz"; sha256 = "9b607f509727e6ada2dbd576a4120c214b1c54f3bb8bf6e2e0eb2cfbb11a0e00"; }; @@ -778,8 +694,6 @@ let }; }; - - "coverage" = python.mkDerivation { name = "coverage-4.5.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/35/fe/e7df7289d717426093c68d156e0fd9117c8f4872b6588e8a8928a0f68424/coverage-4.5.1.tar.gz"; sha256 = "56e448f051a201c5ebbaa86a5efd0ca90d327204d8b059ab25ad0f35fbfd79f1"; }; @@ -793,8 +707,6 @@ let }; }; - - "cryptography" = python.mkDerivation { name = "cryptography-2.2.2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/ec/b2/faa78c1ab928d2b2c634c8b41ff1181f0abdd9adf9193211bd606ffa57e2/cryptography-2.2.2.tar.gz"; sha256 = "9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63"; }; @@ -818,8 +730,6 @@ let }; }; - - "deprecation" = python.mkDerivation { name = "deprecation-1.0.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/8c/e3/e5c66eba8fa2fd567065fa70ada98b990f449f74fb812b408fa7aafe82c9/deprecation-1.0.1.tar.gz"; sha256 = "b9bff5cc91f601ef2a8a0200bc6cde3f18a48c2ed3d1ecbfc16076b14b3ad935"; }; @@ -833,8 +743,6 @@ let }; }; - - "enum34" = python.mkDerivation { name = "enum34-1.1.6"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz"; sha256 = "8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1"; }; @@ -848,8 +756,6 @@ let }; }; - - "factory-boy" = python.mkDerivation { name = "factory-boy-2.10.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/ea/b6/d94a2df8981af1698f7bc14820033cf288263212fc97980dea1a1a5a7eed/factory_boy-2.10.0.tar.gz"; sha256 = "bd5a096d0f102d79b6c78cef1c8c0b650f2e1a3ecba351c735c6d2df8dabd29c"; }; @@ -865,8 +771,6 @@ let }; }; - - "filedepot" = python.mkDerivation { name = "filedepot-0.5.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/75/89/c744b5326d956dd1e8b2e591cb50759965d16ccfd49e7d3124a85a821069/filedepot-0.5.0.tar.gz"; sha256 = "e319a2b163c37fa1f3da21ad7c81a49a12de66da87d1af428fb143f092b96e5d"; }; @@ -887,11 +791,9 @@ let }; }; - - "filteralchemy" = python.mkDerivation { name = "filteralchemy-0.1.0"; - src = pkgs.fetchurl { url = "https://pypi.python.org/packages/ad/3c/20280257ee8411e905fdd09e4bf7f2bbc37d7e47986413c57b30acb473a0/filteralchemy-0.1.0.tar.gz"; sha256 = "38b9784aa85d31a393282eb2b313e5955b5c7632d9bbfc5c653c040e937a7d96"; }; + src = pkgs.fetchgit { url = "https://github.com/sh4r3m4n/filteralchemy.git"; sha256 = "0zq1hnnsbvm004r4gq44c2ld2axzh0k4692vzxf74ip5g9fq4kka"; rev = "9ecc951c74786a8174a1d1c278a2c932945e1906"; }; doCheck = commonDoCheck; buildInputs = commonBuildInputs; propagatedBuildInputs = [ @@ -906,8 +808,6 @@ let }; }; - - "funcsigs" = python.mkDerivation { name = "funcsigs-1.0.2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/94/4a/db842e7a0545de1cdb0439bb80e6e42dfe82aaeaadd4072f2263a4fbed23/funcsigs-1.0.2.tar.gz"; sha256 = "a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50"; }; @@ -921,8 +821,6 @@ let }; }; - - "http-parser" = python.mkDerivation { name = "http-parser-0.8.3"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/07/c4/22e3c76c2313c26dd5f84f1205b916ff38ea951aab0c4544b6e2f5920d64/http-parser-0.8.3.tar.gz"; sha256 = "e2aff90a60def3e476bd71694d8757c0f95ebf2fedf0a8ae34ee306e0b20db83"; }; @@ -936,8 +834,6 @@ let }; }; - - "humanize" = python.mkDerivation { name = "humanize-0.5.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/8c/e0/e512e4ac6d091fc990bbe13f9e0378f34cf6eecd1c6c268c9e598dcf5bb9/humanize-0.5.1.tar.gz"; sha256 = "a43f57115831ac7c70de098e6ac46ac13be00d69abbf60bdcac251344785bb19"; }; @@ -951,8 +847,6 @@ let }; }; - - "hyperlink" = python.mkDerivation { name = "hyperlink-18.0.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/41/e1/0abd4b480ec04892b1db714560f8c855d43df81895c98506442babf3652f/hyperlink-18.0.0.tar.gz"; sha256 = "f01b4ff744f14bc5d0a22a6b9f1525ab7d6312cb0ff967f59414bbac52f0a306"; }; @@ -968,8 +862,6 @@ let }; }; - - "hypothesis" = python.mkDerivation { name = "hypothesis-3.48.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/c7/0b/64a7db8a09397ab3be1c2bf77ca74b6f494758010049003c63573a0e6ca0/hypothesis-3.48.0.tar.gz"; sha256 = "b3c1e6fb2f2e88b3957687e5e05c2385c4c509173539d72cfb4c808a5dff3dcd"; }; @@ -990,8 +882,6 @@ let }; }; - - "idna" = python.mkDerivation { name = "idna-2.6"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/f4/bd/0467d62790828c23c47fc1dfa1b1f052b24efdf5290f071c7a91d0d82fd3/idna-2.6.tar.gz"; sha256 = "2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f"; }; @@ -1005,8 +895,6 @@ let }; }; - - "incremental" = python.mkDerivation { name = "incremental-17.5.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/8f/26/02c4016aa95f45479eea37c90c34f8fab6775732ae62587a874b619ca097/incremental-17.5.0.tar.gz"; sha256 = "7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3"; }; @@ -1023,8 +911,6 @@ let }; }; - - "inflection" = python.mkDerivation { name = "inflection-0.3.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/d5/35/a6eb45b4e2356fe688b21570864d4aa0d0a880ce387defe9c589112077f8/inflection-0.3.1.tar.gz"; sha256 = "18ea7fb7a7d152853386523def08736aa8c32636b047ade55f7578c4edeb16ca"; }; @@ -1038,8 +924,6 @@ let }; }; - - "ipaddress" = python.mkDerivation { name = "ipaddress-1.0.19"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/f0/ba/860a4a3e283456d6b7e2ab39ce5cf11a3490ee1a363652ac50abf9f0f5df/ipaddress-1.0.19.tar.gz"; sha256 = "200d8686011d470b5e4de207d803445deee427455cd0cb7c982b68cf82524f81"; }; @@ -1053,8 +937,6 @@ let }; }; - - "itsdangerous" = python.mkDerivation { name = "itsdangerous-0.24"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/dc/b4/a60bcdba945c00f6d608d8975131ab3f25b22f2bcfe1dab221165194b2d4/itsdangerous-0.24.tar.gz"; sha256 = "cbb3fcf8d3e33df861709ecaf89d9e6629cff0a217bc2848f1b41cd30d360519"; }; @@ -1068,8 +950,6 @@ let }; }; - - "marshmallow" = python.mkDerivation { name = "marshmallow-2.15.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/dc/34/b99d68c78378783c96254cebdf82b6ffa887a1f0e955ec5362eff2f1b2c2/marshmallow-2.15.0.tar.gz"; sha256 = "d3f31fe7be2106b1d783cbd0765ef4e1c6615505514695f33082805f929dd584"; }; @@ -1085,8 +965,6 @@ let }; }; - - "marshmallow-sqlalchemy" = python.mkDerivation { name = "marshmallow-sqlalchemy-0.13.2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/22/e4/47fbf458292cab8a17641b69cec3d77b127bd2400139164838a0b67133e7/marshmallow-sqlalchemy-0.13.2.tar.gz"; sha256 = "9804ef2829f781f469a06528d107c2a763f109c687266ab8b1f000f9684184ae"; }; @@ -1103,8 +981,6 @@ let }; }; - - "mock" = python.mkDerivation { name = "mock-2.0.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/0c/53/014354fc93c591ccc4abff12c473ad565a2eb24dcd82490fae33dbf2539f/mock-2.0.0.tar.gz"; sha256 = "b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba"; }; @@ -1124,8 +1000,6 @@ let }; }; - - "more-itertools" = python.mkDerivation { name = "more-itertools-4.1.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/db/0b/f5660bf6299ec5b9f17bd36096fa8148a1c843fa77ddfddf9bebac9301f7/more-itertools-4.1.0.tar.gz"; sha256 = "c9ce7eccdcb901a2c75d326ea134e0886abfbea5f93e91cc95de9507c0816c44"; }; @@ -1141,8 +1015,6 @@ let }; }; - - "nplusone" = python.mkDerivation { name = "nplusone-0.8.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/b0/9a/8f8bf4dfed57ee26d16c4ddcf38ff4ffee38048e15f4e1a6686e1fbeeadb/nplusone-0.8.1.tar.gz"; sha256 = "6edfa3bea1a99bb22e3b218bcafdefeddb55ebc3b362dc8921844b3b5000e29d"; }; @@ -1159,8 +1031,6 @@ let }; }; - - "olefile" = python.mkDerivation { name = "olefile-0.45.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/d3/8a/e0f0e56d6a542dd987f9290ef7b5164636ee597ce8c2932c19c78292d5ec/olefile-0.45.1.zip"; sha256 = "2b6575f5290de8ab1086f8c5490591f7e0885af682c7c1793bdaf6e64078d385"; }; @@ -1174,8 +1044,6 @@ let }; }; - - "passlib" = python.mkDerivation { name = "passlib-1.7.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/25/4b/6fbfc66aabb3017cd8c3bd97b37f769d7503ead2899bf76e570eb91270de/passlib-1.7.1.tar.gz"; sha256 = "3d948f64138c25633613f303bcc471126eae67c04d5e3f6b7b8ce6242f8653e0"; }; @@ -1192,8 +1060,6 @@ let }; }; - - "pbr" = python.mkDerivation { name = "pbr-4.0.2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/c6/46/f414e7d9ba9621c8acd3e7a82e08c47e0de34ad3e213c16e458b6c04d432/pbr-4.0.2.tar.gz"; sha256 = "dae4aaa78eafcad10ce2581fc34d694faa616727837fd8e55c1a00951ad6744f"; }; @@ -1207,8 +1073,6 @@ let }; }; - - "pgcli" = python.mkDerivation { name = "pgcli-1.8.2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/91/84/e3be2851c87a9550cc747ba85ade36d1b51bb9a747ac93025a01a25baa41/pgcli-1.8.2.tar.gz"; sha256 = "79b32cb0a44e03fc6c26d43080459d48d39d6d81391eb09062a82c940b61441c"; }; @@ -1233,8 +1097,6 @@ let }; }; - - "pgspecial" = python.mkDerivation { name = "pgspecial-1.10.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/d5/d8/cd12f64c42b9878e3e677ba2c43f7abdc87eadba5f18b640f8efda555b55/pgspecial-1.10.0.tar.gz"; sha256 = "eadb0108cdbcf8b38a69bcc9e403b352dbd6d30622e417f48e659180150ee1b6"; }; @@ -1251,8 +1113,6 @@ let }; }; - - "pluggy" = python.mkDerivation { name = "pluggy-0.6.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/11/bf/cbeb8cdfaffa9f2ea154a30ae31a9d04a1209312e2919138b4171a1f8199/pluggy-0.6.0.tar.gz"; sha256 = "7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff"; }; @@ -1266,8 +1126,6 @@ let }; }; - - "prompt-toolkit" = python.mkDerivation { name = "prompt-toolkit-1.0.15"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/8a/ad/cf6b128866e78ad6d7f1dc5b7f99885fb813393d9860778b2984582e81b5/prompt_toolkit-1.0.15.tar.gz"; sha256 = "858588f1983ca497f1cf4ffde01d978a3ea02b01c8a26a8bbc5cd2e66d816917"; }; @@ -1284,8 +1142,6 @@ let }; }; - - "psycopg2" = python.mkDerivation { name = "psycopg2-2.7.3.2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/dd/47/000b405d73ca22980684fd7bd3318690cc03cfa3b2ae1c5b7fff8050b28a/psycopg2-2.7.3.2.tar.gz"; sha256 = "5c3213be557d0468f9df8fe2487eaf2990d9799202c5ff5cb8d394d09fad9b2a"; }; @@ -1299,8 +1155,6 @@ let }; }; - - "py" = python.mkDerivation { name = "py-1.5.3"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/f7/84/b4c6e84672c4ceb94f727f3da8344037b62cee960d80e999b1cd9b832d83/py-1.5.3.tar.gz"; sha256 = "29c9fab495d7528e80ba1e343b958684f4ace687327e6f789a94bf3d1915f881"; }; @@ -1314,8 +1168,6 @@ let }; }; - - "pyOpenSSL" = python.mkDerivation { name = "pyOpenSSL-17.2.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/b0/9e/7088f6165c40c46416aff434eb806c1d64ad6ec6dbc201f5ad4d0484704e/pyOpenSSL-17.2.0.tar.gz"; sha256 = "5d617ce36b07c51f330aa63b83bf7f25c40a0e95958876d54d1982f8c91b4834"; }; @@ -1333,8 +1185,6 @@ let }; }; - - "pyasn1" = python.mkDerivation { name = "pyasn1-0.4.2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/eb/3d/b7d0fdf4a882e26674c68c20f40682491377c4db1439870f5b6f862f76ed/pyasn1-0.4.2.tar.gz"; sha256 = "d258b0a71994f7770599835249cece1caef3c70def868c4915e6e5ca49b67d15"; }; @@ -1348,8 +1198,6 @@ let }; }; - - "pyasn1-modules" = python.mkDerivation { name = "pyasn1-modules-0.2.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/ab/76/36ab0e099e6bd27ed95b70c2c86c326d3affa59b9b535c63a2f892ac9f45/pyasn1-modules-0.2.1.tar.gz"; sha256 = "af00ea8f2022b6287dc375b2c70f31ab5af83989fc6fe9eacd4976ce26cd7ccc"; }; @@ -1365,8 +1213,6 @@ let }; }; - - "pycparser" = python.mkDerivation { name = "pycparser-2.18"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/8c/2d/aad7f16146f4197a11f8e91fb81df177adcc2073d36a17b1491fd09df6ed/pycparser-2.18.tar.gz"; sha256 = "99a8ca03e29851d96616ad0404b4aad7d9ee16f25c9f9708a11faf2810f7b226"; }; @@ -1380,8 +1226,6 @@ let }; }; - - "pydot" = python.mkDerivation { name = "pydot-1.2.4"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/c3/f1/e61d6dfe6c1768ed2529761a68f70939e2569da043e9f15a8d84bf56cadf/pydot-1.2.4.tar.gz"; sha256 = "92d2e2d15531d00710f2d6fb5540d2acabc5399d464f2f20d5d21073af241eb6"; }; @@ -1397,8 +1241,6 @@ let }; }; - - "pyparsing" = python.mkDerivation { name = "pyparsing-2.2.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/3c/ec/a94f8cf7274ea60b5413df054f82a8980523efd712ec55a59e7c3357cf7c/pyparsing-2.2.0.tar.gz"; sha256 = "0832bcf47acd283788593e7a0f542407bd9550a55a8a8435214a1960e04bcb04"; }; @@ -1412,8 +1254,6 @@ let }; }; - - "pytest" = python.mkDerivation { name = "pytest-3.5.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/2d/56/6019153cdd743300c5688ab3b07702355283e53c83fbf922242c053ffb7b/pytest-3.5.0.tar.gz"; sha256 = "fae491d1874f199537fd5872b5e1f0e74a009b979df9d53d1553fd03da1703e1"; }; @@ -1435,8 +1275,6 @@ let }; }; - - "pytest-factoryboy" = python.mkDerivation { name = "pytest-factoryboy-2.0.1"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/80/15/294f84a4d3159f6be5524b8f02b70aae82c86e2341c27ac00fdd26b0d44f/pytest-factoryboy-2.0.1.tar.gz"; sha256 = "ad438d191d2b2a0f26956d437c1963875db573147a84ffd85d7bbeaefae22458"; }; @@ -1454,8 +1292,6 @@ let }; }; - - "python-dateutil" = python.mkDerivation { name = "python-dateutil-2.6.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/51/fc/39a3fbde6864942e8bb24c93663734b74e281b984d1b8c4f95d64b0c21f6/python-dateutil-2.6.0.tar.gz"; sha256 = "62a2f8df3d66f878373fd0072eacf4ee52194ba302e00082828e0d263b0418d2"; }; @@ -1471,8 +1307,6 @@ let }; }; - - "python-slugify" = python.mkDerivation { name = "python-slugify-1.2.4"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/9f/b0/2723356c20fb01b0e09f6ee03c0c629f4e30811e7d92ebd15453d648e5f0/python-slugify-1.2.4.tar.gz"; sha256 = "57a385df7a1c6dbd15f7666eaff0ff29d3f60363b228b1197c5308ed3ba5f824"; }; @@ -1488,8 +1322,6 @@ let }; }; - - "pytz" = python.mkDerivation { name = "pytz-2018.4"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/10/76/52efda4ef98e7544321fd8d5d512e11739c1df18b0649551aeccfb1c8376/pytz-2018.4.tar.gz"; sha256 = "c06425302f2cf668f1bba7a0a03f3c1d34d4ebeef2c72003da308b3947c7f749"; }; @@ -1503,8 +1335,6 @@ let }; }; - - "requests" = python.mkDerivation { name = "requests-2.18.4"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/b0/e1/eab4fc3752e3d240468a8c0b284607899d2fbfb236a56b7377a329aa8d09/requests-2.18.4.tar.gz"; sha256 = "9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e"; }; @@ -1525,8 +1355,6 @@ let }; }; - - "responses" = python.mkDerivation { name = "responses-0.9.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/67/cb/0a5390f7b8944cfa7e4079a839adba964d858d27a60af7b2683248148339/responses-0.9.0.tar.gz"; sha256 = "c6082710f4abfb60793899ca5f21e7ceb25aabf321560cc0726f8b59006811c9"; }; @@ -1547,8 +1375,6 @@ let }; }; - - "restkit" = python.mkDerivation { name = "restkit-4.2.2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/76/b9/d90120add1be718f853c53008cf5b62d74abad1d32bd1e7097dd913ae053/restkit-4.2.2.tar.gz"; sha256 = "c0bda8eb7c643b5e818b612dab49121393abc8589c6cbe9b84085079d598599d"; }; @@ -1565,8 +1391,6 @@ let }; }; - - "service-identity" = python.mkDerivation { name = "service-identity-17.0.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/de/2a/cab6e30be82c8fcd2339ef618036720eda954cf05daef514e386661c9221/service_identity-17.0.0.tar.gz"; sha256 = "4001fbb3da19e0df22c47a06d29681a398473af4aa9d745eca525b3b2c2302ab"; }; @@ -1586,8 +1410,6 @@ let }; }; - - "setproctitle" = python.mkDerivation { name = "setproctitle-1.1.10"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/5a/0d/dc0d2234aacba6cf1a729964383e3452c52096dc695581248b548786f2b3/setproctitle-1.1.10.tar.gz"; sha256 = "6283b7a58477dd8478fbb9e76defb37968ee4ba47b05ec1c053cb39638bd7398"; }; @@ -1601,8 +1423,6 @@ let }; }; - - "six" = python.mkDerivation { name = "six-1.11.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/16/d8/bc6316cf98419719bd59c91742194c111b6f2e85abac88e496adefaf7afe/six-1.11.0.tar.gz"; sha256 = "70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9"; }; @@ -1616,8 +1436,6 @@ let }; }; - - "socketpool" = python.mkDerivation { name = "socketpool-0.5.3"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/d1/39/fae99a735227234ffec389b252c6de2bc7816bf627f56b4c558dc46c85aa/socketpool-0.5.3.tar.gz"; sha256 = "a06733434a56c4b60b8fcaa168102d2386253d36425804d55532a6bbbda6e2ec"; }; @@ -1631,8 +1449,6 @@ let }; }; - - "speaklater" = python.mkDerivation { name = "speaklater-1.3"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/11/92/5ae1effe0ccb8561c034a0111d53c8788660ddb7ed4992f0da1bb5c525e5/speaklater-1.3.tar.gz"; sha256 = "59fea336d0eed38c1f0bf3181ee1222d0ef45f3a9dd34ebe65e6bfffdd6a65a9"; }; @@ -1646,8 +1462,6 @@ let }; }; - - "sqlalchemy-schemadisplay" = python.mkDerivation { name = "sqlalchemy-schemadisplay-1.3"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/ac/6a/de5911b2837278f3cf89b99b0fd94461f789b8f083537ff14ff9aa6d3397/sqlalchemy_schemadisplay-1.3.tar.gz"; sha256 = "0a9f26d77be9d92c9564d87cc17668fe141a816036c5f5d7c8cb053b253957e0"; }; @@ -1663,8 +1477,6 @@ let }; }; - - "sqlparse" = python.mkDerivation { name = "sqlparse-0.2.4"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/79/3c/2ad76ba49f9e3d88d2b58e135b7821d93741856d1fe49970171f73529303/sqlparse-0.2.4.tar.gz"; sha256 = "ce028444cfab83be538752a2ffdb56bc417b7784ff35bb9a3062413717807dec"; }; @@ -1678,8 +1490,6 @@ let }; }; - - "tabulate" = python.mkDerivation { name = "tabulate-0.8.2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/12/c2/11d6845db5edf1295bc08b2f488cf5937806586afe42936c3f34c097ebdc/tabulate-0.8.2.tar.gz"; sha256 = "e4ca13f26d0a6be2a2915428dc21e732f1e44dad7f76d7030b2ef1ec251cf7f2"; }; @@ -1695,8 +1505,6 @@ let }; }; - - "terminaltables" = python.mkDerivation { name = "terminaltables-3.1.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/9b/c4/4a21174f32f8a7e1104798c445dacdc1d4df86f2f26722767034e4de4bff/terminaltables-3.1.0.tar.gz"; sha256 = "f3eb0eb92e3833972ac36796293ca0906e998dc3be91fbe1f8615b331b853b81"; }; @@ -1710,8 +1518,6 @@ let }; }; - - "text-unidecode" = python.mkDerivation { name = "text-unidecode-1.2"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/f0/a2/40adaae7cbdd007fb12777e550b5ce344b56189921b9f70f37084c021ca4/text-unidecode-1.2.tar.gz"; sha256 = "5a1375bb2ba7968740508ae38d92e1f889a0832913cb1c447d5e2046061a396d"; }; @@ -1725,8 +1531,6 @@ let }; }; - - "tqdm" = python.mkDerivation { name = "tqdm-4.15.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/01/f7/2058bd94a903f445e8ff19c0af64b9456187acab41090ff2da21c7c7e193/tqdm-4.15.0.tar.gz"; sha256 = "6ec1dc74efacf2cda936b4a6cf4082ce224c76763bdec9f17e437c8cfcaa9953"; }; @@ -1740,8 +1544,6 @@ let }; }; - - "txaio" = python.mkDerivation { name = "txaio-2.10.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/b8/87/efcae4040c2a0af9c871116a6dbf02ee582b396e6de3797fb30cdcc4a7e4/txaio-2.10.0.tar.gz"; sha256 = "4797f9f6a9866fe887c96abc0110a226dd5744c894dc3630870542597ad30853"; }; @@ -1761,8 +1563,6 @@ let }; }; - - "urllib3" = python.mkDerivation { name = "urllib3-1.22"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/ee/11/7c59620aceedcc1ef65e156cc5ce5a24ef87be4107c2b74458464e437a5d/urllib3-1.22.tar.gz"; sha256 = "cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f"; }; @@ -1782,8 +1582,6 @@ let }; }; - - "wcwidth" = python.mkDerivation { name = "wcwidth-0.1.7"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/55/11/e4a2bb08bb450fdbd42cc709dd40de4ed2c472cf0ccb9e64af22279c5495/wcwidth-0.1.7.tar.gz"; sha256 = "3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e"; }; @@ -1797,8 +1595,6 @@ let }; }; - - "webargs" = python.mkDerivation { name = "webargs-2.1.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/9c/9b/a6dce2167bf86d65f1055d9f4f62b523c5891a7beeb15c8fe63f8948a37f/webargs-2.1.0.tar.gz"; sha256 = "d15d81531b7c0f73dec140bf0cd45c15f061f88eb08fcc29854d94682fd3911c"; }; @@ -1814,8 +1610,6 @@ let }; }; - - "websocket-client" = python.mkDerivation { name = "websocket-client-0.46.0"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/83/91/162f2c76729633d1dc36b09746895c7766bc183bba94cb4d2ec398676060/websocket_client-0.46.0.tar.gz"; sha256 = "933f6bbf08b381f2adbca9e93d7e7958ba212b42c73acb310b18f0fbe74f3738"; }; @@ -1831,8 +1625,6 @@ let }; }; - - "zope.interface" = python.mkDerivation { name = "zope.interface-4.4.3"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/bd/d2/25349ed41f9dcff7b3baf87bd88a4c82396cf6e02f1f42bb68657a3132af/zope.interface-4.4.3.tar.gz"; sha256 = "d6d26d5dfbfd60c65152938fcb82f949e8dada37c041f72916fef6621ba5c5ce"; }; @@ -1847,12 +1639,11 @@ let description = "Interfaces for Python"; }; }; - }; localOverridesFile = ./requirements_override.nix; overrides = import localOverridesFile { inherit pkgs python; }; commonOverrides = [ - + ]; allOverrides = (if (builtins.pathExists localOverridesFile) diff --git a/test_cases/test_server_config.py b/test_cases/test_server_config.py index f58e7a13ac8..c688bbf03a1 100644 --- a/test_cases/test_server_config.py +++ b/test_cases/test_server_config.py @@ -26,26 +26,3 @@ def test_copy_default_config_to_local_does_not_exist(copyfile, makedirs): assert copy_default_config_to_local() is None assert not makedirs.called assert not copyfile.called - - -@mock.patch('os.path.isfile') -@mock.patch('os.remove') -@mock.patch('json.dump') -def test_gen_web_config(dump, remove, isfile): - gen_web_config() - assert isfile.called - assert remove.called - assert dump.called - assert len(dump.call_args_list) == 1 - for call in dump.call_args_list: - call[0][0] == { - 'lic_db': 'faraday_licenses', - 'osint': {u'host': u'shodan.io', - u'icon': u'shodan', - u'label': u'Shodan', - u'prefix': u'/search?query=', - u'suffix': u'', - u'use_external_icon': False}, - 'ver': '2.7.1', - 'vuln_model_db': 'cwe'} - From f22486257abb277ec6e5f089724f5e0d3fae1c79 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 12 Apr 2018 14:42:16 -0300 Subject: [PATCH 1060/1506] remove creator to avoid unique constrait error --- server/importer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/importer.py b/server/importer.py index af6d93b64d7..a2536589fa4 100644 --- a/server/importer.py +++ b/server/importer.py @@ -611,7 +611,6 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation path=path, website=website, workspace=workspace, - creator=creator, ) if document['type'] == 'Vulnerability': @@ -619,7 +618,6 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation 'name': document.get('name'), 'workspace': workspace, 'description': document.get('desc'), - 'creator': creator, } if type(parent) == Host: vuln_params.update({'host_id': parent.id}) From 7acf1455b311a6df9be6dd0b7288647d6f07adbc Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 12 Apr 2018 15:14:33 -0300 Subject: [PATCH 1061/1506] Code simplification for initdb. avoid asking user input. The script is only for localhost configuration --- server/commands/initdb.py | 95 +++++++++++++-------------------------- 1 file changed, 32 insertions(+), 63 deletions(-) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index c996a76a6fa..e83731c1c30 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -71,44 +71,21 @@ def run(self): configure_existing_user = None current_psql_output = TemporaryFile() with open(psql_log_filename, 'a+') as psql_log_file: - while configure_existing_user is None: - configure_existing_user = raw_input('Do you {blue} already have {white} a postgresql username and password? (yes/no): '.format(blue=Fore.BLUE, white=Fore.WHITE)) - if configure_existing_user.lower() == 'yes': - configure_existing_user = True - elif configure_existing_user.lower() == 'no': - configure_existing_user = False - else: - print('Invalid option. Please type "yes" or "no" (ctrl-c to cancel): ') - configure_existing_user = None - hostname = raw_input( - 'Please enter the postgresql hostname or ip address (enter for "localhost"): ') or 'localhost' - - if not configure_existing_user: - if hostname not in ['localhost', '127.0.0.1']: - print('ERROR: can only create postgresql user on localhost') - sys.exit(1) - - username, password, process_status = self._configure_new_postgres_user(current_psql_output) + hostname = 'localhost' + username, password, process_status = self._configure_new_postgres_user(current_psql_output) + current_psql_output.seek(0) + psql_output = current_psql_output.read() + # persist log in the faraday log psql_log.log + psql_log_file.write(psql_output) + self._check_psql_output(current_psql_output, process_status) + + if hostname.lower() in ['localhost', '127.0.0.1']: + database_name = 'faraday' + current_psql_output = TemporaryFile() + database_name, process_status = self._create_database(database_name, username, current_psql_output) current_psql_output.seek(0) - psql_output = current_psql_output.read() - # persist log in the faraday log psql_log.log - psql_log_file.write(psql_output) self._check_psql_output(current_psql_output, process_status) - if hostname.lower() in ['localhost', '127.0.0.1']: - database_name = raw_input( - 'Please enter the {blue} database name {white} (press enter to use "faraday"): '.format( - blue=Fore.BLUE, white=Fore.WHITE)) or 'faraday' - current_psql_output = TemporaryFile() - database_name, process_status = self._create_database(database_name, username, current_psql_output) - current_psql_output.seek(0) - self._check_psql_output(current_psql_output, process_status) - else: - username, password = self._configure_existing_postgres_user() - database_name = raw_input( - 'Please enter the {blue} database name {white} (press enter to use "faraday"): '.format( - blue=Fore.BLUE, white=Fore.WHITE)) or 'faraday' - current_psql_output.close() conn_string = self._save_config(config, username, password, database_name, hostname) self._create_tables(conn_string) @@ -127,7 +104,6 @@ def _create_admin_user(self, conn_string): "is_ldap, active, last_login_ip, current_login_ip, role) VALUES ('faraday', 'Administrator', " "'{0}', false, true, '127.0.0.1', '127.0.0.1', 'admin');".format(random_password)) except sqlalchemy.exc.IntegrityError as ex: - print(ex) # when re using database user could be created previusly already_created = True print( @@ -174,7 +150,7 @@ def _configure_new_postgres_user(self, psql_log_file): """ username_default = 'faraday_db_admin' print('This script will {blue} create a new postgres user {white} and {blue} save faraday-server settings {white}(server.ini). '.format(blue=Fore.BLUE, white=Fore.WHITE)) - username = raw_input('Please enter the new {blue} database username {white} (press enter to use "{0}"): '.format(username_default, blue=Fore.BLUE, white=Fore.WHITE)) or username_default + username = 'faraday' postgres_command = ['sudo', '-u', 'postgres'] password = self.generate_random_pw(25) command = postgres_command + ['psql', '-c', 'CREATE ROLE {0} WITH LOGIN PASSWORD \'{1}\';'.format(username, password)] @@ -187,33 +163,26 @@ def _configure_new_postgres_user(self, psql_log_file): if already_exists_error in output: print("{yellow}WARNING{white}: Role {username} already exists, skipping creation ".format(yellow=Fore.YELLOW, white=Fore.WHITE, username=username)) - invalid_pwd = True - while invalid_pwd: - password = getpass.getpass("Database password (ctrl-c to " - "cancel): ") - # check credentials - # this case only applies to instances without 'trust' config - # todo: check postgres config - try: - connection = psycopg2.connect(dbname='postgres', - user=username, - password=password) - cur = connection.cursor() - cur.execute('SELECT * FROM pg_catalog.pg_tables;') - cur.fetchall() - connection.commit() - connection.close() - invalid_pwd = False - except psycopg2.Error as e: - if 'authentication failed' in e.message: - print('{red}ERROR{white}: User {username} already ' - 'exists and provided password ' - 'is incorrect'.format(white=Fore.WHITE, - red=Fore.RED, - username=username)) - else: - raise + try: + connection = psycopg2.connect(dbname='postgres', + user=username, + password=password) + cur = connection.cursor() + cur.execute('SELECT * FROM pg_catalog.pg_tables;') + cur.fetchall() + connection.commit() + connection.close() + invalid_pwd = False + except psycopg2.Error as e: + if 'authentication failed' in e.message: + print('{red}ERROR{white}: User {username} already ' + 'exists and provided password ' + 'is incorrect'.format(white=Fore.WHITE, + red=Fore.RED, + username=username)) + else: + raise return_code = 0 return username, password, return_code From df828a9995d3447fe186239deabcbbcbf7a06b33 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 12 Apr 2018 16:58:49 -0300 Subject: [PATCH 1062/1506] Update exist message when db user already exists --- server/commands/initdb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index e83731c1c30..f16ac86fbca 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -177,10 +177,10 @@ def _configure_new_postgres_user(self, psql_log_file): except psycopg2.Error as e: if 'authentication failed' in e.message: print('{red}ERROR{white}: User {username} already ' - 'exists and provided password ' - 'is incorrect'.format(white=Fore.WHITE, - red=Fore.RED, - username=username)) + 'exists'.format(white=Fore.WHITE, + red=Fore.RED, + username=username)) + sys.exit(1) else: raise return_code = 0 From cf69a302964e533909bac10dadbbb37e225a975b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 12 Apr 2018 17:04:17 -0300 Subject: [PATCH 1063/1506] New users should be not activated --- server/importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index a2536589fa4..110d27aa007 100644 --- a/server/importer.py +++ b/server/importer.py @@ -593,7 +593,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation password = "".join( [rng.choice(string.ascii_letters + string.digits) for _ in xrange(12)]) - creator, _ = get_or_create(session, User, username=owner_name) + creator, _ = get_or_create(session, User, username=owner_name, active=False) creator.password = password if document['type'] == 'VulnerabilityWeb': method = document.get('method') From 47bebcf74d5607e96922964a6b5d0235a736c7a7 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 12 Apr 2018 17:06:00 -0300 Subject: [PATCH 1064/1506] Avoid more users creation --- manage.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/manage.py b/manage.py index 7f3cbedf402..c072204df06 100755 --- a/manage.py +++ b/manage.py @@ -4,6 +4,7 @@ import click import requests +import sys from requests import ConnectionError from sqlalchemy.exc import OperationalError from pgcli.main import PGCli @@ -108,6 +109,9 @@ def validate_email(ctx, param, value): confirmation_prompt=True) def create_user(username, email, password): with app.app_context(): + if db.session.query(User).count() > 0: + print("Can't create more users. Please contact support") + sys.exit(1) app.user_datastore.create_user(username=username, email=email, password=password, From 1df04c55e72e90a455414d2ef121537e12fe0592 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 12 Apr 2018 17:10:41 -0300 Subject: [PATCH 1065/1506] Chekc user count when starting faraday-server --- faraday-server.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/faraday-server.py b/faraday-server.py index ea7051d23f4..224e0569e88 100755 --- a/faraday-server.py +++ b/faraday-server.py @@ -12,7 +12,7 @@ import server.config import server.couchdb import server.utils.logger -from server.models import db, Workspace +from server.models import db, Workspace, User from server.utils import daemonize from server.web import app from utils import dependencies @@ -79,6 +79,10 @@ def run_server(args): web_server = server.web.WebServer(enable_ssl=args.ssl) daemonize.create_pid_file() + with app.app_context(): + if db.session.query(User).count() > 1: + print("Can't start faraday-server. User invariant failed. Please contact support") + sys.exit(1) web_server.run() logger.info('Faraday Server is ready') From 35c9e0e3d8ace1a027a37de3b7f44ba0ead37ec2 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 13 Apr 2018 11:43:20 -0300 Subject: [PATCH 1066/1506] PEP8 fix and add comments --- server/web.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/web.py b/server/web.py index d51f3762a6f..222121efa19 100644 --- a/server/web.py +++ b/server/web.py @@ -130,14 +130,16 @@ def run(self): ssl_context = self.__load_ssl_certs() self.__listen_func = functools.partial( reactor.listenSSL, - contextFactory = ssl_context) + contextFactory=ssl_context) else: self.__listen_func = reactor.listenTCP try: + # web and static content self.__listen_func( self.__listen_port, site, interface=self.__bind_address) + # websockets listenWS(self.__build_websockets_resource(), interface=self.__bind_address) reactor.run() except error.CannotListenError as e: From f72e2c80f27ce5128ae82417107447a8d61318be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 13 Apr 2018 13:00:25 -0300 Subject: [PATCH 1067/1506] Fix horrible interceptor bug with horrible stack trace This happened on pink but could also happen in white, so put it here --- server/www/scripts/auth/services/interceptor.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/www/scripts/auth/services/interceptor.js b/server/www/scripts/auth/services/interceptor.js index 99b72dd82df..3091f7cf9ab 100644 --- a/server/www/scripts/auth/services/interceptor.js +++ b/server/www/scripts/auth/services/interceptor.js @@ -18,10 +18,15 @@ angular.module('faradayApp'). $cookies.currentComponent; } return deferred.reject(response); - }); + }.error(function(auth){ + // I don't know what i'm doing. But appeareantly if i don't put this + // angular may fail with a horrible stack trace + return deferred.reject(response); + })); return deferred.promise; }else if (response.status === 403) { $location.path('/forbidden'); + return $q.reject(response); }else{ return $q.reject(response); } From e11d281a87e7ded67d971ffc7eff6840d091533c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 13 Apr 2018 13:23:20 -0300 Subject: [PATCH 1068/1506] Fix a bug when on bulk edit of reference. due to context switch some references could be already persisted on db and not in session --- server/models.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/models.py b/server/models.py index e2e7a8eab2b..329d782205a 100644 --- a/server/models.py +++ b/server/models.py @@ -467,12 +467,12 @@ def _create(self, value): # other process/thread won us on the commit # we need to fetch already created objs. session.rollback() - conflict_obj_names = [obj.name for obj in conflict_objs if obj.name != value] - for conflict_obj_name in conflict_obj_names: - conclict_obj = session.query(Reference).filter_by(name=conflict_obj_name).first() - if not conclict_obj: - raise Exception('This should not happend. AssocProxy could not find a conflict obj.') - self.col.add(conclict_obj) + for conflict_obj in conflict_objs: + if conflict_obj.name == value: + continue + persisted_conclict_obj = session.query(conflict_obj.__class__).filter_by(name=conflict_obj.name).first() + if persisted_conclict_obj: + self.col.add(persisted_conclict_obj) yield self.creator(value, parent_instance) def add(self, value): From 510f77826ea3189ade974090a4c5ae2dd433f5a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 13 Apr 2018 13:25:25 -0300 Subject: [PATCH 1069/1506] Fix bad parenthesis position in last commit --- server/www/scripts/auth/services/interceptor.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/auth/services/interceptor.js b/server/www/scripts/auth/services/interceptor.js index 3091f7cf9ab..bedadc40efd 100644 --- a/server/www/scripts/auth/services/interceptor.js +++ b/server/www/scripts/auth/services/interceptor.js @@ -18,11 +18,11 @@ angular.module('faradayApp'). $cookies.currentComponent; } return deferred.reject(response); - }.error(function(auth){ + }).error(function(auth){ // I don't know what i'm doing. But appeareantly if i don't put this // angular may fail with a horrible stack trace return deferred.reject(response); - })); + }); return deferred.promise; }else if (response.status === 403) { $location.path('/forbidden'); From 2dafcb44799fe5aa8f5efeb04b9181cdbe74ff75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 13 Apr 2018 17:23:23 -0300 Subject: [PATCH 1070/1506] Remove recently added buggy interceptor stuff --- server/www/scripts/auth/services/interceptor.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/www/scripts/auth/services/interceptor.js b/server/www/scripts/auth/services/interceptor.js index bedadc40efd..ed6a0604111 100644 --- a/server/www/scripts/auth/services/interceptor.js +++ b/server/www/scripts/auth/services/interceptor.js @@ -18,10 +18,6 @@ angular.module('faradayApp'). $cookies.currentComponent; } return deferred.reject(response); - }).error(function(auth){ - // I don't know what i'm doing. But appeareantly if i don't put this - // angular may fail with a horrible stack trace - return deferred.reject(response); }); return deferred.promise; }else if (response.status === 403) { From e0519c75647a2c3a703a2aebdcb70aa20e5e94a8 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 16 Mar 2018 15:42:31 -0300 Subject: [PATCH 1071/1506] Fix tornado 5.0 bug with IOLoop.instance() --- apis/rest/api.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apis/rest/api.py b/apis/rest/api.py index 3810bab056f..ac0d4b2f92b 100644 --- a/apis/rest/api.py +++ b/apis/rest/api.py @@ -25,28 +25,31 @@ _plugin_controller_api = None _http_server = None - - +ioloop_instance = None def startServer(): global _http_server + global ioloop_instance if _http_server is not None: - IOLoop.instance().start() + ioloop_instance.start() def stopServer(): global _http_server + global ioloop_instance if _http_server is not None: - IOLoop.instance().stop() + ioloop_instance.stop() _http_server.stop() def startAPIs(plugin_controller, model_controller, hostname, port): global _rest_controllers global _http_server + global ioloop_instance _rest_controllers = [PluginControllerAPI(plugin_controller), ModelControllerAPI(model_controller)] app = Flask('APISController') + ioloop_instance = IOLoop.current() _http_server = HTTPServer(WSGIContainer(app)) while True: try: @@ -76,9 +79,6 @@ def startAPIs(plugin_controller, model_controller, hostname, port): logging.getLogger("tornado.access").propagate = False threading.Thread(target=startServer).start() -def stopAPIs(): - stopServer() - class RESTApi(object): """ Abstract class for REST Controllers From d9e5bd4834943502363f13a20a24932850670a89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 16 Apr 2018 15:30:44 -0300 Subject: [PATCH 1072/1506] Fix basic cascade deletes and add some tests to it --- server/models.py | 14 ++++-- test_cases/models/test_cascades.py | 74 ++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 5 deletions(-) diff --git a/server/models.py b/server/models.py index 329d782205a..ac38d9760a3 100644 --- a/server/models.py +++ b/server/models.py @@ -208,6 +208,7 @@ class Host(Metadata): services = relationship( 'Service', order_by='Service.protocol,Service.port', + cascade="all, delete-orphan" ) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, @@ -1078,14 +1079,17 @@ class Credential(Metadata): name = BlankColumn(Text) host_id = Column(Integer, ForeignKey(Host.id), index=True, nullable=True) - host = relationship('Host', backref='credentials', foreign_keys=[host_id]) + host = relationship( + 'Host', + backref=backref("credentials", cascade="all, delete-orphan"), + foreign_keys=[host_id]) service_id = Column(Integer, ForeignKey(Service.id), index=True, nullable=True) service = relationship( - 'Service', - backref='credentials', - foreign_keys=[service_id], - ) + 'Service', + backref=backref('credentials', cascade="all, delete-orphan"), + foreign_keys=[service_id], + ) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) workspace = relationship( diff --git a/test_cases/models/test_cascades.py b/test_cases/models/test_cascades.py index af89c040a90..90ba4bab0c9 100644 --- a/test_cases/models/test_cascades.py +++ b/test_cases/models/test_cascades.py @@ -1,3 +1,6 @@ +import pytest +from contextlib import contextmanager +from server.models import CommandObject def test_delete_user(workspace, session): assert workspace.creator @@ -6,3 +9,74 @@ def test_delete_user(workspace, session): session.delete(user) session.commit() assert workspace.creator is None + + +class TestCascadeDelete: + @pytest.fixture(autouse=True) + def populate(self, workspace, service, session, + vulnerability_factory, credential_factory, + empty_command_factory): + session.commit() + self.session = session + assert service.workspace_id == workspace.id + + workspace.set_scope(['*.infobytesec.com', '192.168.1.0/24']) + self.workspace = workspace + self.host = service.host + self.host.set_hostnames(['a.com', 'b.com']) + self.service = service + + self.host_cred = credential_factory.create( + host=self.host, + service=None, + workspace=workspace + ) + + self.service_cred = credential_factory.create( + host=None, + service=service, + workspace=workspace + ) + + self.host_vuln = vulnerability_factory.create( + host=self.host, + service=None, + workspace=workspace, + ) + + self.service_vuln = vulnerability_factory.create( + host=None, + service=service, + workspace=workspace, + ) + + session.flush() + for vuln in [self.host_vuln, self.service_vuln]: + vuln.references = ['CVE-1234', 'CVE-4331'] + vuln.policy_violations = ["PCI-DSS"] + + self.command = empty_command_factory.create(workspace=workspace) + CommandObject.create(self.host_vuln, self.command) + CommandObject.create(self.service_vuln, self.command) + + session.commit() + + @contextmanager + def assert_deletes(self, *objs): + assert all(obj.id is not None for obj in objs) + ids = [(obj.__table__, obj.id) for obj in objs] + yield + self.session.commit() + for (table, id_) in ids: + assert self.session.query(table).filter( + table.columns['id'] == id_).count() == 0 + + def test_delete_host_cascade(self): + with self.assert_deletes(self.host, self.service, + self.host_vuln, self.service_vuln, + self.host_cred, self.service_cred): + self.session.delete(self.host) + + def test_delete_workspace(self): + self.session.delete(self.workspace) + self.session.commit() From 521bc3b62baa497e33c0a76f582223a6c684ace9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 16 Apr 2018 15:59:49 -0300 Subject: [PATCH 1073/1506] Fix misteriously broken tests Appearently the delete-orphan cascade doesn't behave well with the create() method of the factories --- test_cases/test_api_credentials.py | 1 + test_cases/test_model_controller.py | 1 + 2 files changed, 2 insertions(+) diff --git a/test_cases/test_api_credentials.py b/test_cases/test_api_credentials.py index 26aabe36e7e..3e4ee44a534 100644 --- a/test_cases/test_api_credentials.py +++ b/test_cases/test_api_credentials.py @@ -129,6 +129,7 @@ def _generate_raw_update_data(self, name, username, password, parent_id): def test_update_credentials_with_invalid_parent(self, test_client, session): credential = self.factory.create() + session.add(credential) session.commit() raw_data = self._generate_raw_update_data('Name1', 'Username2', 'Password3', parent_id=43) diff --git a/test_cases/test_model_controller.py b/test_cases/test_model_controller.py index a768b7733e8..1b2cac7e4f2 100644 --- a/test_cases/test_model_controller.py +++ b/test_cases/test_model_controller.py @@ -164,6 +164,7 @@ def test_find(get, url_endpoint, test_data, session): workspace = WorkspaceFactory.create() mappers_manager.createMappers(workspace.name) obj = test_data['factory'].create(workspace=workspace) + session.add(obj) session.commit() result = controller.find(test_data['class_signature'], obj.id) assert get.called From b0af80993b3a3ca154f87530011cdf2e713574f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 16 Apr 2018 16:20:28 -0300 Subject: [PATCH 1074/1506] Fix file cascade deletion --- server/models.py | 1 + test_cases/models/test_cascades.py | 28 +++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index ac38d9760a3..e167fb1949b 100644 --- a/server/models.py +++ b/server/models.py @@ -751,6 +751,7 @@ class VulnerabilityGeneric(VulnerabilityABC): primaryjoin="and_(File.object_id==VulnerabilityGeneric.id, " "File.object_type=='vulnerability')", foreign_keys="File.object_id", + cascade="all, delete-orphan" ) tags = relationship( diff --git a/test_cases/models/test_cascades.py b/test_cases/models/test_cascades.py index 90ba4bab0c9..451d8af1af2 100644 --- a/test_cases/models/test_cascades.py +++ b/test_cases/models/test_cascades.py @@ -1,6 +1,6 @@ import pytest from contextlib import contextmanager -from server.models import CommandObject +from server.models import CommandObject, File def test_delete_user(workspace, session): assert workspace.creator @@ -55,6 +55,24 @@ def populate(self, workspace, service, session, vuln.references = ['CVE-1234', 'CVE-4331'] vuln.policy_violations = ["PCI-DSS"] + self.attachment = File( + name='test.png', + filename='test.png', + content='test', + object_type='vulnerability', + object_id=self.service_vuln.id, + ) + self.session.add(self.attachment) + + self.host_attachment = File( + name='test.png', + filename='test.png', + content='test', + object_type='host', + object_id=self.host.id, + ) + self.session.add(self.host_attachment) + self.command = empty_command_factory.create(workspace=workspace) CommandObject.create(self.host_vuln, self.command) CommandObject.create(self.service_vuln, self.command) @@ -80,3 +98,11 @@ def test_delete_host_cascade(self): def test_delete_workspace(self): self.session.delete(self.workspace) self.session.commit() + + def test_delete_vuln_attachments(self): + with self.assert_deletes(self.attachment): + self.session.delete(self.service_vuln) + + def test_delete_host_attachments(self): + with self.assert_deletes(self.host_attachment): + self.session.delete(self.host) From fd72cd5516417842a6196693b41c3cb19bdf220f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 16 Apr 2018 16:50:29 -0300 Subject: [PATCH 1075/1506] Add more cascade delete tests --- server/models.py | 2 +- test_cases/models/test_cascades.py | 37 ++++++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/server/models.py b/server/models.py index e167fb1949b..9580af7fbaa 100644 --- a/server/models.py +++ b/server/models.py @@ -1353,7 +1353,7 @@ class Methodology(Metadata): workspace = relationship( 'Workspace', - backref=backref('methodologies', cascade="all, delete-orphan") + backref=backref('methodologies', cascade="all, delete-orphan"), ) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) diff --git a/test_cases/models/test_cascades.py b/test_cases/models/test_cascades.py index 451d8af1af2..5147896b023 100644 --- a/test_cases/models/test_cascades.py +++ b/test_cases/models/test_cascades.py @@ -1,6 +1,12 @@ import pytest from contextlib import contextmanager -from server.models import CommandObject, File +from server.models import ( + CommandObject, + Comment, + File, + Methodology, + Task +) def test_delete_user(workspace, session): assert workspace.creator @@ -73,6 +79,22 @@ def populate(self, workspace, service, session, ) self.session.add(self.host_attachment) + self.comment = Comment( + text="test", + object_type='host', + object_id=self.host.id, + workspace=self.workspace + ) + self.session.add(self.comment) + + self.reply_comment = Comment( + text="ok", + object_type='host', + object_id=self.host.id, + workspace=self.workspace, + reply_to=self.comment, + ) + self.command = empty_command_factory.create(workspace=workspace) CommandObject.create(self.host_vuln, self.command) CommandObject.create(self.service_vuln, self.command) @@ -95,7 +117,14 @@ def test_delete_host_cascade(self): self.host_cred, self.service_cred): self.session.delete(self.host) - def test_delete_workspace(self): + def test_delete_workspace(self, user): + methodology = Methodology(name='test', workspace=self.workspace) + task = Task(methodology=methodology, assigned_to=[user], + name="test", + workspace=self.workspace) + self.session.add(task) + self.session.commit() + self.session.delete(self.workspace) self.session.commit() @@ -103,6 +132,10 @@ def test_delete_vuln_attachments(self): with self.assert_deletes(self.attachment): self.session.delete(self.service_vuln) + def test_host_comments(self): + with self.assert_deletes(self.comment, self.reply_comment): + self.session.delete(self.host) + def test_delete_host_attachments(self): with self.assert_deletes(self.host_attachment): self.session.delete(self.host) From 8fac6a00db8417be13c8484fc6957b4a2cb452e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 17 Apr 2018 12:11:11 -0300 Subject: [PATCH 1076/1506] Add delete user test It shouldn't cascade delete --- test_cases/models/test_cascades.py | 47 +++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/test_cases/models/test_cascades.py b/test_cases/models/test_cascades.py index 5147896b023..56c6424d20c 100644 --- a/test_cases/models/test_cascades.py +++ b/test_cases/models/test_cascades.py @@ -19,7 +19,7 @@ def test_delete_user(workspace, session): class TestCascadeDelete: @pytest.fixture(autouse=True) - def populate(self, workspace, service, session, + def populate(self, workspace, service, session, user, vulnerability_factory, credential_factory, empty_command_factory): session.commit() @@ -27,6 +27,7 @@ def populate(self, workspace, service, session, assert service.workspace_id == workspace.id workspace.set_scope(['*.infobytesec.com', '192.168.1.0/24']) + self.user = user self.workspace = workspace self.host = service.host self.host.set_hostnames(['a.com', 'b.com']) @@ -35,25 +36,29 @@ def populate(self, workspace, service, session, self.host_cred = credential_factory.create( host=self.host, service=None, - workspace=workspace + workspace=workspace, + creator=user, ) self.service_cred = credential_factory.create( host=None, service=service, - workspace=workspace + workspace=workspace, + creator=user, ) self.host_vuln = vulnerability_factory.create( host=self.host, service=None, workspace=workspace, + creator=user, ) self.service_vuln = vulnerability_factory.create( host=None, service=service, workspace=workspace, + creator=user, ) session.flush() @@ -67,6 +72,7 @@ def populate(self, workspace, service, session, content='test', object_type='vulnerability', object_id=self.service_vuln.id, + creator=user, ) self.session.add(self.attachment) @@ -76,6 +82,7 @@ def populate(self, workspace, service, session, content='test', object_type='host', object_id=self.host.id, + creator=user, ) self.session.add(self.host_attachment) @@ -83,7 +90,8 @@ def populate(self, workspace, service, session, text="test", object_type='host', object_id=self.host.id, - workspace=self.workspace + workspace=self.workspace, + creator=user, ) self.session.add(self.comment) @@ -93,23 +101,34 @@ def populate(self, workspace, service, session, object_id=self.host.id, workspace=self.workspace, reply_to=self.comment, + creator=user, ) - self.command = empty_command_factory.create(workspace=workspace) + self.command = empty_command_factory.create( + workspace=workspace, + creator=user) CommandObject.create(self.host_vuln, self.command) CommandObject.create(self.service_vuln, self.command) session.commit() @contextmanager - def assert_deletes(self, *objs): + # def assert_deletes(self, *objs, should_delete=True): + def assert_deletes(self, *objs, **kwargs): + # this could be better with python3 (like in the comment before the function + # definition) + should_delete = kwargs.get('should_delete', True) assert all(obj.id is not None for obj in objs) ids = [(obj.__table__, obj.id) for obj in objs] yield self.session.commit() for (table, id_) in ids: + if should_delete: + expected_count = 0 + else: + expected_count = 1 assert self.session.query(table).filter( - table.columns['id'] == id_).count() == 0 + table.columns['id'] == id_).count() == expected_count def test_delete_host_cascade(self): with self.assert_deletes(self.host, self.service, @@ -139,3 +158,17 @@ def test_host_comments(self): def test_delete_host_attachments(self): with self.assert_deletes(self.host_attachment): self.session.delete(self.host) + + def test_delete_user_does_not_delete_childs(self): + objs = [self.workspace, self.host, self.service, + self.host_cred, self.service_cred, + self.host_vuln, self.service_vuln, + self.attachment, self.host_attachment, + self.comment, self.reply_comment, + self.command] + for obj in objs: + assert obj.creator is not None + with self.assert_deletes(*objs, should_delete=False): + self.session.delete(self.user) + for obj in objs: + assert obj.creator is None From df5d837f90c735b88aa51aed0ce1c04808d423f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 17 Apr 2018 12:43:43 -0300 Subject: [PATCH 1077/1506] Add cascade delete to workspace permisssions --- server/models.py | 12 ++++++++++++ test_cases/models/test_cascades.py | 14 ++++++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/server/models.py b/server/models.py index 9580af7fbaa..679a16a5e4a 100644 --- a/server/models.py +++ b/server/models.py @@ -1168,6 +1168,10 @@ class Workspace(Metadata): vulnerability_standard_count = query_expression() vulnerability_total_count = query_expression() + workspace_permission_instances = relationship( + "WorkspacePermission", + cascade="all, delete-orphan") + @classmethod def query_with_count(cls, only_confirmed): """ @@ -1252,6 +1256,10 @@ class WorkspacePermission(db.Model): user = relationship('User', foreign_keys=[user_id]) + @property + def parent(self): + return + def is_valid_workspace(workspace_name): return db.session.query(server.models.Workspace).filter_by(name=workspace_name).first() is not None @@ -1283,6 +1291,10 @@ class User(db.Model, UserMixin): nullable=False, default='client') # TODO: add many to many relationship to add permission to workspace + workspace_permission_instances = relationship( + "WorkspacePermission", + cascade="all, delete-orphan") + def __init__(self, *args, **kwargs): # added for compatibility with flask security try: diff --git a/test_cases/models/test_cascades.py b/test_cases/models/test_cascades.py index 56c6424d20c..57890d88402 100644 --- a/test_cases/models/test_cascades.py +++ b/test_cases/models/test_cascades.py @@ -5,7 +5,8 @@ Comment, File, Methodology, - Task + Task, + WorkspacePermission ) def test_delete_user(workspace, session): @@ -29,6 +30,8 @@ def populate(self, workspace, service, session, user, workspace.set_scope(['*.infobytesec.com', '192.168.1.0/24']) self.user = user self.workspace = workspace + self.permission = WorkspacePermission(user=user, workspace=workspace) + session.add(self.permission) self.host = service.host self.host.set_hostnames(['a.com', 'b.com']) self.service = service @@ -144,8 +147,10 @@ def test_delete_workspace(self, user): self.session.add(task) self.session.commit() - self.session.delete(self.workspace) - self.session.commit() + with self.assert_deletes(self.permission): + with self.assert_deletes(self.user, should_delete=False): + self.session.delete(self.workspace) + self.session.commit() def test_delete_vuln_attachments(self): with self.assert_deletes(self.attachment): @@ -169,6 +174,7 @@ def test_delete_user_does_not_delete_childs(self): for obj in objs: assert obj.creator is not None with self.assert_deletes(*objs, should_delete=False): - self.session.delete(self.user) + with self.assert_deletes(self.permission): + self.session.delete(self.user) for obj in objs: assert obj.creator is None From b206e1954aac8a08c468a48107502dd6ef8e1d37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 17 Apr 2018 17:55:14 -0300 Subject: [PATCH 1078/1506] Fix service cascade delete bug (no rock solid yet) Deleting a service was deleting its parent host --- server/models.py | 1 - test_cases/models/test_cascades.py | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index 9580af7fbaa..0d1fbd3e8f2 100644 --- a/server/models.py +++ b/server/models.py @@ -353,7 +353,6 @@ class Service(Metadata): host = relationship( 'Host', foreign_keys=[host_id], - cascade='all' ) workspace_id = Column(Integer, ForeignKey('workspace.id'), index=True, nullable=False) diff --git a/test_cases/models/test_cascades.py b/test_cases/models/test_cascades.py index 56c6424d20c..adc477a7fc4 100644 --- a/test_cases/models/test_cascades.py +++ b/test_cases/models/test_cascades.py @@ -172,3 +172,8 @@ def test_delete_user_does_not_delete_childs(self): self.session.delete(self.user) for obj in objs: assert obj.creator is None + + def test_delete_service_keeps_parents(self): + with self.assert_deletes(self.host, self.user, self.workspace, + should_delete=False): + self.session.delete(self.service) From c774b8a70d2e4895fda74323229774896674c72c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 17 Apr 2018 18:35:17 -0300 Subject: [PATCH 1079/1506] Fix task/methodology cascade deletes --- server/models.py | 9 ++++--- test_cases/models/test_cascades.py | 38 +++++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/server/models.py b/server/models.py index 0d1fbd3e8f2..7c414389a68 100644 --- a/server/models.py +++ b/server/models.py @@ -1341,11 +1341,12 @@ class Methodology(Metadata): template = relationship( 'MethodologyTemplate', - backref=backref('methodologies', cascade="all, delete-orphan") + backref=backref('methodologies') ) template_id = Column( Integer, - ForeignKey('methodology_template.id'), + ForeignKey('methodology_template.id', + ondelete="SET NULL"), index=True, nullable=True, ) @@ -1377,7 +1378,9 @@ class TaskTemplate(TaskABC): 'concrete': True } - template = relationship('MethodologyTemplate', backref='tasks') + template = relationship( + 'MethodologyTemplate', + backref=backref('tasks', cascade="all, delete-orphan")) template_id = Column( Integer, ForeignKey('methodology_template.id'), diff --git a/test_cases/models/test_cascades.py b/test_cases/models/test_cascades.py index adc477a7fc4..2bf8283dab7 100644 --- a/test_cases/models/test_cascades.py +++ b/test_cases/models/test_cascades.py @@ -5,7 +5,9 @@ Comment, File, Methodology, - Task + MethodologyTemplate, + Task, + TaskTemplate ) def test_delete_user(workspace, session): @@ -110,6 +112,31 @@ def populate(self, workspace, service, session, user, CommandObject.create(self.host_vuln, self.command) CommandObject.create(self.service_vuln, self.command) + self.methodology_template = MethodologyTemplate( + name="test", + ) + session.add(self.methodology_template) + + self.methodology_template_task = TaskTemplate( + name="aaaa", + template=self.methodology_template + ) + session.add(self.methodology_template) + + self.methodology = Methodology( + name="test", + template=self.methodology_template, + workspace=self.workspace) + session.add(self.methodology) + + self.methodology_task = Task( + name="aaaa", + workspace=self.workspace, + template=self.methodology_template_task, + methodology=self.methodology + ) + session.add(self.methodology_template_task) + session.commit() @contextmanager @@ -177,3 +204,12 @@ def test_delete_service_keeps_parents(self): with self.assert_deletes(self.host, self.user, self.workspace, should_delete=False): self.session.delete(self.service) + + def test_delete_methodology_template_keeps_child(self): + with self.assert_deletes(self.methodology, self.methodology_task, + should_delete=False): + self.session.delete(self.methodology_template) + + def test_delete_methodology_template_deletes_task(self): + with self.assert_deletes(self.methodology_template_task): + self.session.delete(self.methodology_template) From 2735a5cdab92f2fc42cbbd2bfe3115b2b9500b54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 17 Apr 2018 18:44:46 -0300 Subject: [PATCH 1080/1506] Fix task assigned to cascade deletes --- server/models.py | 6 ++++-- test_cases/models/test_cascades.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/server/models.py b/server/models.py index 7c414389a68..c5f6e84f151 100644 --- a/server/models.py +++ b/server/models.py @@ -1401,8 +1401,10 @@ class TaskAssignedTo(db.Model): task = relationship('Task') user_id = Column(Integer, ForeignKey('faraday_user.id'), nullable=False) - user = relationship('User', - foreign_keys=[user_id]) + user = relationship( + 'User', + foreign_keys=[user_id], + backref=backref('assigned_tasks', cascade="all, delete-orphan")) class Task(TaskABC): diff --git a/test_cases/models/test_cascades.py b/test_cases/models/test_cascades.py index 2bf8283dab7..7c31f429de8 100644 --- a/test_cases/models/test_cascades.py +++ b/test_cases/models/test_cascades.py @@ -7,6 +7,7 @@ Methodology, MethodologyTemplate, Task, + TaskAssignedTo, TaskTemplate ) @@ -137,6 +138,12 @@ def populate(self, workspace, service, session, user, ) session.add(self.methodology_template_task) + self.methodology_task_assigned = TaskAssignedTo( + task=self.methodology_task, + user=self.user, + ) + session.add(self.methodology_task_assigned) + session.commit() @contextmanager @@ -213,3 +220,15 @@ def test_delete_methodology_template_keeps_child(self): def test_delete_methodology_template_deletes_task(self): with self.assert_deletes(self.methodology_template_task): self.session.delete(self.methodology_template) + + def test_delete_task_deletes_assignations(self): + with self.assert_deletes(self.methodology_task_assigned): + self.session.delete(self.methodology_task) + + def test_delete_task_assignation_keeps_user(self): + with self.assert_deletes(self.user, should_delete=False): + self.session.delete(self.methodology_task_assigned) + + def test_delete_user_deletes_assignations(self): + with self.assert_deletes(self.methodology_task_assigned): + self.session.delete(self.user) From cc6d5e6f617341bc6abb8f7ffa0f7177fc9a7a3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 18 Apr 2018 16:10:38 -0300 Subject: [PATCH 1081/1506] Use ES5 compatible code in some scripts --- server/www/scripts/commons/providers/server.js | 2 +- server/www/scripts/statusReport/controllers/statusReport.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index 6225452f17e..a92b99b5a58 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -56,7 +56,7 @@ angular.module("faradayApp") return APIURL + objectType + "/" + objectId + "/"; }; - var createDbUrl = function(wsName = "") { + var createDbUrl = function(wsName) { return APIURL + "ws/" + wsName + (wsName ? "/" : ""); } diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 37f5db2f6e9..ea73db5eb31 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -479,7 +479,7 @@ angular.module('faradayApp') return obj.data; }); - return response.filter(x => !angular.equals(x, {})) + return response.filter(function(x){!angular.equals(x, {})}) }, function(failed) { commonsFact.showMessage("Something failed searching vulnerability exploits."); From 696422d848b9ac0c74a41fa7da081d3ad512cef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 18 Apr 2018 16:26:35 -0300 Subject: [PATCH 1082/1506] Remove wrong copypasted line --- server/www/scripts/commons/controllers/commercialCtrl.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/www/scripts/commons/controllers/commercialCtrl.js b/server/www/scripts/commons/controllers/commercialCtrl.js index ed1352b3cb9..ad942705d8e 100644 --- a/server/www/scripts/commons/controllers/commercialCtrl.js +++ b/server/www/scripts/commons/controllers/commercialCtrl.js @@ -10,7 +10,6 @@ angular.module('faradayApp') if(!auth) { $location.path('/login'); } - return deferred.reject(response); }); if ($location.path().split("/")[1] === "executive") { $scope.header = "executive report"; @@ -23,4 +22,4 @@ angular.module('faradayApp') } else { $scope.header = $location.path().split("/")[1]; } - }]); \ No newline at end of file + }]); From 2fbb58b65d3a0dba365e61c68e52e1c07c6cc646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 18 Apr 2018 16:34:15 -0300 Subject: [PATCH 1083/1506] Show licenses and users menu options in community --- server/www/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/index.html b/server/www/index.html index abcafbed6c3..fd8aa205963 100644 --- a/server/www/index.html +++ b/server/www/index.html @@ -74,8 +74,8 @@
    diff --git a/server/www/scripts/dashboard/controllers/dashboard.js b/server/www/scripts/dashboard/controllers/dashboard.js index cd4ca361d51..e8e6c3a8f1f 100644 --- a/server/www/scripts/dashboard/controllers/dashboard.js +++ b/server/www/scripts/dashboard/controllers/dashboard.js @@ -4,8 +4,8 @@ angular.module('faradayApp') .controller('dashboardCtrl', - ['$scope', '$filter', '$route', '$routeParams', '$location', 'dashboardSrv', 'workspacesFact', - function($scope, $filter, $route, $routeParams, $location, dashboardSrv, workspacesFact) { + ['$scope', '$filter', '$route', '$routeParams', '$location', 'dashboardSrv', 'workspacesFact', 'vulnsManager', + function($scope, $filter, $route, $routeParams, $location, dashboardSrv, workspacesFact, vulnsManager) { $scope.props = dashboardSrv.props; init = function() { @@ -16,6 +16,12 @@ angular.module('faradayApp') workspacesFact.list().then(function(wss) { $scope.workspaces = wss; }); + + vulnsManager.getVulns($scope.workspace, null, null, null, null, null) + .then(function(response) { + $scope.totalItems = response.count; + }); + dashboardSrv.setConfirmedFromCookie(); dashboardSrv.startTimer(); }; diff --git a/server/www/scripts/dashboard/partials/dashboard.html b/server/www/scripts/dashboard/partials/dashboard.html index 3a70f6b8727..e862e5335dd 100644 --- a/server/www/scripts/dashboard/partials/dashboard.html +++ b/server/www/scripts/dashboard/partials/dashboard.html @@ -4,48 +4,80 @@
    -
    -

    - - Dashboard for {{ workspace }} ({{props["confirmed"] === false ? 'all vulns' : 'confirmed' }}) - -
    - - - - -
    -

    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +
    +
    + Dashboard +
    +
    +
    + + +
    + + {{totalItems}} vulns total + +
    + +
    + + +
    +
    +
    +
    +
    + + +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    diff --git a/server/www/scripts/navigation/partials/leftBar.html b/server/www/scripts/navigation/partials/leftBar.html index 0aabd6333af..06c9995dd3e 100644 --- a/server/www/scripts/navigation/partials/leftBar.html +++ b/server/www/scripts/navigation/partials/leftBar.html @@ -2,6 +2,11 @@
    -
    -

    Forgot password?

    {{errorMessage}}

    diff --git a/server/www/styles/material-input.css b/server/www/styles/material-input.css index fc039a42e71..3bfc0903426 100644 --- a/server/www/styles/material-input.css +++ b/server/www/styles/material-input.css @@ -55,4 +55,5 @@ .form-input textarea:valid + .placeholder { cursor: text; opacity: 0; + z-index: -1; } From 661aa181222ebe4b332b6c579896b04915f70dc3 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 21 May 2018 15:08:53 -0300 Subject: [PATCH 1146/1506] Add empty status check command --- manage.py | 7 +++++++ server/commands/status_check.py | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 server/commands/status_check.py diff --git a/manage.py b/manage.py index 7bdce489079..16b02b7899f 100755 --- a/manage.py +++ b/manage.py @@ -16,6 +16,7 @@ from server.commands.app_urls import show_all_urls from server.commands.reset_db import reset_db_all from server.commands.reports import import_external_reports +from server.commands.status_check import full_status_check from server.models import db, User from server.importer import ImportCouchDB @@ -87,6 +88,11 @@ def sql_shell(): pgcli.run_cli() +@click.command() +def status_check(): + full_status_check() + + def validate_user_unique_field(ctx, param, value): with app.app_context(): if User.query.filter_by(**{param.name: value}).count(): @@ -132,6 +138,7 @@ def createsuperuser(username, email, password): cli.add_command(database_schema) cli.add_command(createsuperuser) cli.add_command(sql_shell) +cli.add_command(status_check) if __name__ == '__main__': diff --git a/server/commands/status_check.py b/server/commands/status_check.py new file mode 100644 index 00000000000..89121e9e0f3 --- /dev/null +++ b/server/commands/status_check.py @@ -0,0 +1,18 @@ +from colorama import init +from colorama import Fore, Back, Style + +init() + + +def check_server_running(): + pass + + +def check_open_ports(): + pass + + +def full_status_check(): + print('{red} Incomplete Check {white}.'.format(red=Fore.RED, white=Fore.WHITE)) + check_server_running() + check_open_ports() From 927fe243020d02c23e6a5b52855fbfc3d56682e6 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Mon, 21 May 2018 16:17:28 -0300 Subject: [PATCH 1147/1506] Add psycopg2 to install.sh --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 74e69e3ed03..6e44a1c343b 100755 --- a/install.sh +++ b/install.sh @@ -13,7 +13,7 @@ fi apt-get update #Install community dependencies -for pkg in build-essential python-setuptools python-pip python-dev libpq-dev libffi-dev gir1.2-gtk-3.0 gir1.2-vte-2.91 python-gobject zsh curl; do +for pkg in build-essential python-setuptools python-pip python-dev libpq-dev libffi-dev gir1.2-gtk-3.0 gir1.2-vte-2.91 python-gobject zsh curl python-psycopg2 ; do apt-get install -y $pkg done From 58e3399789b5d088ccc9b675b75b58af41423ebf Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Mon, 21 May 2018 16:20:40 -0300 Subject: [PATCH 1148/1506] Fix requirements versions --- requirements_extras.txt | 6 +++--- requirements_server.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements_extras.txt b/requirements_extras.txt index 5389e69f17d..d86012fb40b 100644 --- a/requirements_extras.txt +++ b/requirements_extras.txt @@ -1,4 +1,4 @@ -beautifulsoup4>=4.6.0 -psycopg2>=2.7.3 -w3af_api_client>=1.1.7 +beautifulsoup4==4.6.0 +psycopg2==2.7.3 +w3af_api_client==1.1.7 selenium==3.9.0 diff --git a/requirements_server.txt b/requirements_server.txt index 273050b796a..ba89fcebf4e 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -10,7 +10,7 @@ IPy==0.83 marshmallow Pillow==4.2.1 psycopg2==2.7.3.2 -pyasn1-modules>=0.0.11 +pyasn1-modules==0.0.11 pyopenssl==17.2.0 python-dateutil==2.6.0 python-slugify==1.2.4 From 324957fe7f31468208bb0de6a56d30e284b4c52a Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Mon, 21 May 2018 16:26:46 -0300 Subject: [PATCH 1149/1506] Fix requirements version, == in a few modules --- requirements_dev.txt | 6 +++--- requirements_server.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 3038dcac155..5410db11497 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,6 +1,6 @@ factory-boy==2.10.0 -pytest -pytest-factoryboy -responses +pytest==3.5.1 +pytest-factoryboy==2.0.1 +responses==0.9.0 hypothesis==3.48.0 beautifulsoup4==4.6.0 \ No newline at end of file diff --git a/requirements_server.txt b/requirements_server.txt index ba89fcebf4e..070182cc2c9 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -21,7 +21,7 @@ SQLAlchemy==1.2.0b2 sqlalchemy_schemadisplay==1.3 tqdm==4.15.0 twisted==17.5.0 -webargs +webargs==3.0.0 marshmallow-sqlalchemy git+https://github.com/sh4r3m4n/filteralchemy@dev#egg=filteralchemy filedepot==0.5.0 From 6e4c10c8a8e7d5f9e9e24286b620079f1d351f89 Mon Sep 17 00:00:00 2001 From: Winna_Z Date: Mon, 21 May 2018 18:22:05 -0300 Subject: [PATCH 1150/1506] check_server_status() done --- server/commands/status_check.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/server/commands/status_check.py b/server/commands/status_check.py index 89121e9e0f3..c91931f50b8 100644 --- a/server/commands/status_check.py +++ b/server/commands/status_check.py @@ -1,18 +1,25 @@ from colorama import init from colorama import Fore, Back, Style - +import socket +from server.utils.daemonize import is_server_running init() -def check_server_running(): - pass +def check_server_running(): + pid = is_server_running() + if pid is not None: + print('Faraday Server is Running. PID:{PID} {green} Running. \ + '.format(green=Fore.GREEN, PID=pid)) + return True + else: + print('Faraday Server is not running {red} Not Running. \ + '.format(red=Fore.RED)) + return True def check_open_ports(): pass - def full_status_check(): - print('{red} Incomplete Check {white}.'.format(red=Fore.RED, white=Fore.WHITE)) check_server_running() - check_open_ports() + From 7bdf2156c2e3ca9e3c870bf65ed3661349710194 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Mon, 21 May 2018 19:38:46 -0300 Subject: [PATCH 1151/1506] Goodbye Notes... --- apis/rest/api.py | 4 +- bin/create_note.py | 43 ---------------------- model/api.py | 10 ----- model/guiapi.py | 57 ++++++++++++----------------- plugins/plugin.py | 46 +++-------------------- test_cases/plugins/test_acunetix.py | 4 +- test_cases/plugins/test_burp.py | 4 +- test_cases/plugins/test_nessus.py | 5 +-- 8 files changed, 32 insertions(+), 141 deletions(-) delete mode 100644 bin/create_note.py diff --git a/apis/rest/api.py b/apis/rest/api.py index 5db283a3082..64b2c20fdf4 100644 --- a/apis/rest/api.py +++ b/apis/rest/api.py @@ -270,9 +270,7 @@ def createVulnWeb(self): 'params', 'query', 'category', 'parent_id']) def createNote(self): - return self._create( - self.controller.newNote, - ['name', 'text', 'parent_id']) + return jsonify(code=200) def createCred(self): return self._create( diff --git a/bin/create_note.py b/bin/create_note.py deleted file mode 100644 index 04fc441c5c6..00000000000 --- a/bin/create_note.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python2.7 -# -*- coding: utf-8 -*- -""" -Faraday Penetration Test IDE -Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) -See the file 'doc/LICENSE' for the license information -""" -import logging - -from model.common import factory -from persistence.server import models - -__description__ = 'Creates a new note' -__prettyname__ = 'Create Note' - -logger = logging.getLogger(__name__) - - -def main(workspace='', args=None, parser=None): - logger.warn('Create note will create a comment. fplugin name will be changed to create_comment') - parser.add_argument('parent', help='Parent ID') - parser.add_argument('--parent_type', - help='Vulnerability severity', - choices=['Host', 'Service'], - default='unclassified') - parser.add_argument('name', help='Note name') - parser.add_argument('text', help='Note content') - - parser.add_argument('--dry-run', action='store_true', help='Do not touch the database. Only print the object ID') - - parsed_args = parser.parse_args(args) - - obj = factory.createModelObject(models.Note.class_signature, - parsed_args.name, - workspace, - text=parsed_args.text, - object_id=parsed_args.parent, - object_type=parsed_args.parent_type.lower() - ) - - models.create_note(workspace, obj) - - return 0, 1 diff --git a/model/api.py b/model/api.py index f2d3fd6b0db..91d0afb57b0 100644 --- a/model/api.py +++ b/model/api.py @@ -195,22 +195,12 @@ def createAndAddVulnWebToService(host_id, service_id, name, desc, ref, severity, # Note def createAndAddNoteToHost(host_id, name, text): - note = newNote(name, text, parent_id=host_id, parent_type='host') - if addNoteToHost(host_id, note): - return note.getID() return None - def createAndAddNoteToService(host_id, service_id, name, text): - note = newNote(name, text, parent_id=service_id, parent_type='service') - if addNoteToService(host_id, service_id, note): - return note.getID() return None def createAndAddNoteToNote(host_id, service_id, note_id, name, text): - note = newNote(name, text, parent_id=note_id, parent_type='comment') - if addNoteToNote(host_id, service_id, note_id, note): - return note.getID() return None def createAndAddCredToService(host_id, service_id, username, password): diff --git a/model/guiapi.py b/model/guiapi.py index 93e29cf49b6..6f671389bb5 100644 --- a/model/guiapi.py +++ b/model/guiapi.py @@ -24,7 +24,7 @@ def setMainApp(ref): global __the_mainapp __the_mainapp = ref notification_center.setUiApp(__the_mainapp) - + def getMainApp(): global __the_mainapp return __the_mainapp @@ -32,12 +32,12 @@ def getMainApp(): def getMainWindow(): global __the_mainapp return __the_mainapp.getMainWindow() - + def postCustomEvent(event, receiver=None): if receiver is None: receiver = getMainWindow() __the_mainapp.postEvent(receiver, event) - + def sendCustomEvent(event, receiver=None): if receiver is None: receiver = getMainWindow() @@ -84,7 +84,7 @@ def createAndAddInterface(host_id, name = "", mac = "00:00:00:00:00:00", return None -def createAndAddServiceToInterface(host_id, interface_id, name, protocol = "tcp?", +def createAndAddServiceToInterface(host_id, interface_id, name, protocol = "tcp?", ports = [], status = "running", version = "unknown", description = ""): service = model.api.newService(name, protocol, ports, status, version, description, parent_id=interface_id) if addServiceToInterface(host_id, interface_id, service): @@ -157,33 +157,22 @@ def createAndAddVulnWeb(model_object, name, desc, website, path, ref=None, def createAndAddNoteToHost(host_id, name, text): - note = model.api.newNote(name, text, parent_id=host_id) - if addNoteToHost(host_id, note): - return note.getID() + return None def createAndAddNoteToInterface(host_id, interface_id, name, text): - note = model.api.newNote(name, text, parent_id=interface_id) - if addNoteToInterface(host_id, interface_id, note): - return note.getID() + return None def createAndAddNoteToService(host_id, service_id, name, text): - note = model.api.newNote(name, text, parent_id=service_id) - if addNoteToService(host_id, service_id, note): - return note.getID() + return None def createAndAddNote(model_object, name, text): - note = model.api.newNote(name, text, parent_id=model_object.getID()) - if addNote(model_object.getID(), note): - return note.getID() return None - - def createAndAddCred(model_object, username, password): cred = model.api.newCred(username, password, parent_id=model_object.getID()) if addCred(model_object.getID(), cred): @@ -255,7 +244,7 @@ def addVuln(model_object_id, vuln): return True return False - + def addNoteToHost(host_id, note): if note is not None: @@ -287,7 +276,7 @@ def addNote(model_object_id, note): return True return False - + def addCred(model_object_id, cred): if cred is not None: __model_controller.addCredSYNC(model_object_id, cred) @@ -336,16 +325,16 @@ def delServiceFromApplication(host_id, application_id, service_id): def delVulnFromApplication(vuln, hostname, appname): __model_controller.delVulnFromApplicationSYNC(hostname, appname, vuln) return True - + def delVulnFromInterface(vuln, hostname, intname): __model_controller.delVulnFromInterfaceSYNC(hostname,intname, vuln) return True - + def delVulnFromHost(vuln, hostname): __model_controller.delVulnFromHostSYNC(hostname,vuln) return True - + def delVulnFromService(vuln, hostname, srvname): __model_controller.delVulnFromServiceSYNC(hostname,srvname, vuln) return True @@ -354,21 +343,21 @@ def delVuln(model_object_id, vuln_id): __model_controller.delVulnSYNC(model_object_id, vuln_id) return True - - + + def delNoteFromApplication(note, hostname, appname): __model_controller.delNoteFromApplicationSYNC(hostname, appname, note) return True - + def delNoteFromInterface(note, hostname, intname): __model_controller.delNoteFromInterfaceSYNC(hostname,intname, note) return True - + def delNoteFromHost(note, hostname): __model_controller.delNoteFromHostSYNC(hostname, note) return True - + def delNoteFromService(note, hostname, srvname): __model_controller.delNoteFromServiceSYNC(hostname,srvname, note) return True @@ -377,7 +366,7 @@ def delNote(model_object_id, note_id): __model_controller.delNoteSYNC(model_object_id, note_id) return True - + def delCred(model_object_id, cred_id): __model_controller.delCredSYNC(model_object_id, cred_id) return True @@ -386,14 +375,14 @@ def delCredFromHost(cred, hostname): __model_controller.delCredFromHostSYNC(hostname, cred) return True - + def delCredFromService(cred, hostname, srvname): __model_controller.delCredFromServiceSYNC(hostname,srvname, cred) return True - - + + def editHost(host, name=None, description=None, os=None, owned=None): __model_controller.editHostSYNC(host, name, description, os, owned) @@ -407,9 +396,9 @@ def editApplication(application, name, description, status, version, owned): __model_controller.editApplicationSYNC(application, name, description, status, version, owned) return True -def editInterface(interface, name=None, description=None, hostnames=None, mac=None, ipv4=None, ipv6=None, network_segment=None, +def editInterface(interface, name=None, description=None, hostnames=None, mac=None, ipv4=None, ipv6=None, network_segment=None, amount_ports_opened=None, amount_ports_closed=None, amount_ports_filtered=None, owned=None): - __model_controller.editInterfaceSYNC(interface, name, description, hostnames, mac, ipv4, ipv6, network_segment, + __model_controller.editInterfaceSYNC(interface, name, description, hostnames, mac, ipv4, ipv6, network_segment, amount_ports_opened, amount_ports_closed, amount_ports_filtered, owned) return True diff --git a/plugins/plugin.py b/plugins/plugin.py index 9be3eb8ff9f..473d36aa2a1 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -26,7 +26,7 @@ Vuln, VulnWeb, Credential, - Note, + Note ) from model import Modelactions #from plugins.modelactions import modelactions @@ -312,52 +312,16 @@ def createAndAddVulnWebToService(self, host_id, service_id, name, desc="", return vulnweb_obj.getID() def createAndAddNoteToHost(self, host_id, name, text): + return None - note_obj = model.common.factory.createModelObject( - Note.class_signature, - name, text=text, object_id=host_id, object_type='host', - workspace_name=self.workspace) - - note_obj._metadata.creator = self.id - self.__addPendingAction(Modelactions.ADDNOTEHOST, note_obj) - return note_obj.getID() - - @deprecation.deprecated(deprecated_in="3.0", removed_in="3.5", - current_version=VERSION, - details="Interface object removed. Use host or service instead. Note will be added to Host") def createAndAddNoteToInterface(self, host_id, interface_id, name, text): - - note_obj = model.common.factory.createModelObject( - Note.class_signature, - name, text=text, object_id=host_id, object_type='host', - workspace_name=self.workspace) - - note_obj._metadata.creator = self.id - self.__addPendingAction(Modelactions.ADDNOTEHOST, note_obj) - return note_obj.getID() + return None def createAndAddNoteToService(self, host_id, service_id, name, text): - - note_obj = model.common.factory.createModelObject( - Note.class_signature, - name, text=text, object_id=service_id, object_type='service', - workspace_name=self.workspace) - - note_obj._metadata.creator = self.id - self.__addPendingAction(Modelactions.ADDNOTESRV, note_obj) - return note_obj.getID() + return None def createAndAddNoteToNote(self, host_id, service_id, note_id, name, text): - - note_obj = model.common.factory.createModelObject( - Note.class_signature, - name, text=text, object_id=note_id, object_type='comment', - workspace_name=self.workspace) - - note_obj._metadata.creator = self.id - - self.__addPendingAction(Modelactions.ADDNOTENOTE, note_obj) - return note_obj.getID() + return None def createAndAddCredToService(self, host_id, service_id, username, password): diff --git a/test_cases/plugins/test_acunetix.py b/test_cases/plugins/test_acunetix.py index 29541f14ed3..115b09538a0 100644 --- a/test_cases/plugins/test_acunetix.py +++ b/test_cases/plugins/test_acunetix.py @@ -57,12 +57,10 @@ def test_Plugin_creates_apropiate_objects(self, monkeypatch): action = self.plugin._pending_actions.get(block=True) actions[action[0]].append(action[1]) - assert actions.keys() == [2000, 20008, 2027, 2040, 2038] + assert actions.keys() == [2000, 20008, 2038] assert len(actions[2000]) == 1 assert actions[2000][0].name == "5.175.17.140" assert len(actions[20008]) == 1 - assert len(actions[2027]) == 1 - assert len(actions[2040]) == 1 assert len(actions[2038]) == 52 assert actions[20008][0].ports == [80] diff --git a/test_cases/plugins/test_burp.py b/test_cases/plugins/test_burp.py index f70214adcf2..26e53bbc900 100644 --- a/test_cases/plugins/test_burp.py +++ b/test_cases/plugins/test_burp.py @@ -50,10 +50,8 @@ def test_Plugin_creates_adecuate_objects(self, monkeypatch): actions[action[0]].append(action[1]) assert actions[2000][0].name == "200.20.20.201" - assert actions.keys() == [2000, 20008, 2027, 2040, 2038] + assert actions.keys() == [2000, 20008, 2038] assert len(actions[20008]) == 14 - assert len(actions[2027]) == 14 - assert len(actions[2040]) == 14 assert len(actions[2038]) == 14 assert all('http' == name for name in map(lambda service: service.name, actions[20008])) diff --git a/test_cases/plugins/test_nessus.py b/test_cases/plugins/test_nessus.py index 6c5ca3e0da6..b64e4ce3d43 100644 --- a/test_cases/plugins/test_nessus.py +++ b/test_cases/plugins/test_nessus.py @@ -48,13 +48,10 @@ def test_Plugin_Calls_createAndAddHost(self, monkeypatch): actions[action[0]].append(action[1]) assert actions[2000][0].name == "12.233.108.201" - assert actions.keys() == [2017, 20008, 2027, 2000, 2038, 2040] + assert actions.keys() == [2000, 2017, 2038, 20008] assert len(actions[20008]) == 1 - assert len(actions[2027]) == 1 assert len(actions[2038]) == 1 - assert len(actions[2040]) == 1 - assert actions[2040][0].name == "preprod.boardvantage.net" assert actions[2038][0].name == "Nessus SYN scanner" assert actions[20008][0].ports == [443] From ef42ff5916ac815bcd53147c9356dfa8a421237d Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Tue, 22 May 2018 10:11:42 -0300 Subject: [PATCH 1152/1506] 4725 - Update input and input placeholder color --- server/www/styles/material-input.css | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/www/styles/material-input.css b/server/www/styles/material-input.css index 3bfc0903426..80b6e78d8f1 100644 --- a/server/www/styles/material-input.css +++ b/server/www/styles/material-input.css @@ -18,7 +18,7 @@ transition: all 100ms ease-in-out; width: 100%; cursor: text; - color: #ddddd; + color: #a1a1a1; } .form-input input, .form-input textarea { @@ -29,11 +29,12 @@ font-size: 16px; font-weight: 300; border: 0; - border-bottom: 1px dashed #A1A1A1; + border-bottom: 1px dashed #d4d4d4; transition: border-color 100ms ease-in-out; outline: none; padding: 0; margin: 0; + color: #a1a1a1; } .form-input textarea { min-height: 30px; From a987c3b372dd1197e85f4865bd7278a00a2043e9 Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Tue, 22 May 2018 10:11:46 -0300 Subject: [PATCH 1153/1506] 4725 - Update password input lock size --- server/www/estilos-v3.css | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/www/estilos-v3.css b/server/www/estilos-v3.css index 63de724c953..e7c152cac84 100644 --- a/server/www/estilos-v3.css +++ b/server/www/estilos-v3.css @@ -122,13 +122,13 @@ body { .frd-icon-lock { background-image: url("images/icon-login-password-lock.svg"); - width: 24px; - height: 24px; + width: 20px; + height: 20px; } #form-signin .placeholder .frd-icon-lock { position: relative; - bottom: 10px; + bottom: 5px; } #footer-login { From 81bbb8bdf2d226ab3f66a75ec08baebad4ebb257 Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Tue, 22 May 2018 10:12:01 -0300 Subject: [PATCH 1154/1506] 4725 - Update password input lock image --- server/www/images/icon-login-password-lock.svg | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/www/images/icon-login-password-lock.svg b/server/www/images/icon-login-password-lock.svg index 496c2b5c105..c9b623d5c3a 100644 --- a/server/www/images/icon-login-password-lock.svg +++ b/server/www/images/icon-login-password-lock.svg @@ -3,15 +3,15 @@ - - + - From 3efcb994b28e405eb4438042006ad4d7fc35f354 Mon Sep 17 00:00:00 2001 From: Nicolas Ronchi Cesarini Date: Tue, 22 May 2018 10:33:16 -0300 Subject: [PATCH 1155/1506] fix - showHeader method --- server/www/scripts/commons/controllers/headerCtrl.js | 6 ++++++ server/www/scripts/commons/partials/header.html | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/server/www/scripts/commons/controllers/headerCtrl.js b/server/www/scripts/commons/controllers/headerCtrl.js index 8bb6a7a4854..427701c66bc 100644 --- a/server/www/scripts/commons/controllers/headerCtrl.js +++ b/server/www/scripts/commons/controllers/headerCtrl.js @@ -7,6 +7,12 @@ angular.module('faradayApp') ['$scope', '$routeParams', '$location', 'dashboardSrv', 'workspacesFact', 'vulnsManager', function($scope, $routeParams, $location, dashboardSrv, workspacesFact, vulnsManager) { + + $scope.showHeader = function() { + var noNav = ["", "home", "login", "index"]; + return noNav.indexOf($scope.component) < 0; + }; + init = function(name) { $scope.location = $location.path().split('/')[1]; $scope.workspace = $routeParams.wsId; diff --git a/server/www/scripts/commons/partials/header.html b/server/www/scripts/commons/partials/header.html index f10bf3de5c8..51fe3347f84 100644 --- a/server/www/scripts/commons/partials/header.html +++ b/server/www/scripts/commons/partials/header.html @@ -1,4 +1,4 @@ -
    +
    From 46a0c94ec2a27c5422043df39f78a4a73e63b416 Mon Sep 17 00:00:00 2001 From: Nicolas Ronchi Cesarini Date: Tue, 22 May 2018 10:37:19 -0300 Subject: [PATCH 1156/1506] (fix) removal of header on index page --- server/www/index.html | 1 - 1 file changed, 1 deletion(-) diff --git a/server/www/index.html b/server/www/index.html index 9c0d053e44c..485a6b7af30 100644 --- a/server/www/index.html +++ b/server/www/index.html @@ -70,7 +70,6 @@
    -
    From 4edf7bdebb88df43c01eac34b0f7adefc928c1f0 Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Tue, 22 May 2018 12:34:58 -0300 Subject: [PATCH 1157/1506] 4721 - Starting new dashboard --- server/www/dashboard-v3.css | 89 +++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 server/www/dashboard-v3.css diff --git a/server/www/dashboard-v3.css b/server/www/dashboard-v3.css new file mode 100644 index 00000000000..5c3e1104fea --- /dev/null +++ b/server/www/dashboard-v3.css @@ -0,0 +1,89 @@ +.bg-white { + background-color: #FFF; +} + +.border-right { + border-right: 1px solid #e7e7e7; +} + +.dashboard-label-wrapper { + text-align: left; + margin-top: 10px; + margin-bottom: 10px; +} + +.dashboard-label-wrapper > .label { + padding: 5px; + width: 100%; + display: block; +} + +.label-gray { + background-color: #a1a1a1; +} + +.m0 { + margin: 0px !important; +} + +.mt-md { + margin-top: 10px; +} + +.p0 { + padding: 0px !important; +} + +.pl0 { + padding-left: 0px !important; +} + +.p-md { + padding: 10px; +} + +.ph-sm { + padding-left: 5px; + padding-right: 5px; +} + +.ph-md { + padding-left: 10px; + padding-right: 10px; +} + +.ph-xxl { + padding-left: 45px; + padding-right: 45px; +} + +.pl-sm { + padding-left: 5px; +} + +.pr-sm { + padding-right: 5px; +} + +.progress-bar-tall { + height: 57px; +} + +.progress-bar-tall > .progress-bar > span { + top: 20px; + position: relative; +} + +.progress-bar-light-blue > .progress-bar { + background-color: #409ccd; +} + +.panel { + -webkit-box-shadow: none; + box-shadow: none; + border-radius: 0px; +} + +.text-left { + text-align: left !important; +} From 318dbf48fcc4da9a754211361aae0e72ecfc51b4 Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Tue, 22 May 2018 12:34:58 -0300 Subject: [PATCH 1158/1506] 4721 - Starting new dashboard --- server/www/dashboard-v3.css | 89 +++++++++++++++++++ server/www/index.html | 3 +- .../dashboard/partials/activityFeed.html | 2 +- .../scripts/dashboard/partials/dashboard.html | 68 +++++++++----- .../dashboard/partials/last-vulns.html | 14 +-- .../scripts/dashboard/partials/services.html | 56 ++++++------ .../dashboard/partials/summarized.html | 4 +- .../scripts/dashboard/partials/top-hosts.html | 6 +- .../dashboard/partials/top-services.html | 6 +- .../dashboard/partials/vulns-by-severity.html | 10 +-- .../dashboard/partials/vulns-piechart.html | 6 +- .../partials/workspace-progress.html | 48 +++++----- 12 files changed, 215 insertions(+), 97 deletions(-) create mode 100644 server/www/dashboard-v3.css diff --git a/server/www/dashboard-v3.css b/server/www/dashboard-v3.css new file mode 100644 index 00000000000..5c3e1104fea --- /dev/null +++ b/server/www/dashboard-v3.css @@ -0,0 +1,89 @@ +.bg-white { + background-color: #FFF; +} + +.border-right { + border-right: 1px solid #e7e7e7; +} + +.dashboard-label-wrapper { + text-align: left; + margin-top: 10px; + margin-bottom: 10px; +} + +.dashboard-label-wrapper > .label { + padding: 5px; + width: 100%; + display: block; +} + +.label-gray { + background-color: #a1a1a1; +} + +.m0 { + margin: 0px !important; +} + +.mt-md { + margin-top: 10px; +} + +.p0 { + padding: 0px !important; +} + +.pl0 { + padding-left: 0px !important; +} + +.p-md { + padding: 10px; +} + +.ph-sm { + padding-left: 5px; + padding-right: 5px; +} + +.ph-md { + padding-left: 10px; + padding-right: 10px; +} + +.ph-xxl { + padding-left: 45px; + padding-right: 45px; +} + +.pl-sm { + padding-left: 5px; +} + +.pr-sm { + padding-right: 5px; +} + +.progress-bar-tall { + height: 57px; +} + +.progress-bar-tall > .progress-bar > span { + top: 20px; + position: relative; +} + +.progress-bar-light-blue > .progress-bar { + background-color: #409ccd; +} + +.panel { + -webkit-box-shadow: none; + box-shadow: none; + border-radius: 0px; +} + +.text-left { + text-align: left !important; +} diff --git a/server/www/index.html b/server/www/index.html index 485a6b7af30..6e9f780eda7 100644 --- a/server/www/index.html +++ b/server/www/index.html @@ -22,7 +22,6 @@ - @@ -30,6 +29,8 @@ + + diff --git a/server/www/scripts/dashboard/partials/activityFeed.html b/server/www/scripts/dashboard/partials/activityFeed.html index 57c545a264b..8aced40ea16 100644 --- a/server/www/scripts/dashboard/partials/activityFeed.html +++ b/server/www/scripts/dashboard/partials/activityFeed.html @@ -1,4 +1,4 @@ -
    +

    Activity Feed diff --git a/server/www/scripts/dashboard/partials/dashboard.html b/server/www/scripts/dashboard/partials/dashboard.html index 0c842672141..43d9f9337a4 100644 --- a/server/www/scripts/dashboard/partials/dashboard.html +++ b/server/www/scripts/dashboard/partials/dashboard.html @@ -4,28 +4,54 @@
    -
    -
    -
    +
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    diff --git a/server/www/scripts/dashboard/partials/last-vulns.html b/server/www/scripts/dashboard/partials/last-vulns.html index 2e3d13c1a49..1d2508b37a3 100644 --- a/server/www/scripts/dashboard/partials/last-vulns.html +++ b/server/www/scripts/dashboard/partials/last-vulns.html @@ -1,4 +1,4 @@ -
    +

    Last Vulnerabilities @@ -11,14 +11,14 @@

    Last Vulnerabilities

    No vulnerabilities found yet.

    - +
    - - - - - + + + + + diff --git a/server/www/scripts/dashboard/partials/services.html b/server/www/scripts/dashboard/partials/services.html index be25240d430..11543f208c7 100644 --- a/server/www/scripts/dashboard/partials/services.html +++ b/server/www/scripts/dashboard/partials/services.html @@ -3,34 +3,32 @@
    -
    -
    -
    -

    Services report - -

    -
    -
    - -

    No services found yet

    -
    -
    -
    -
    -
    - -
    -
    {{srv.count}}
    -
    {{srv.name}}
    -
    -     -
    -
    +
    +
    +

    Services report + +

    +
    +
    + +

    No services found yet

    +
    +
    +
    +
    +
    + +
    +
    {{srv.count}}
    +
    {{srv.name}}
    +
    +     +
    -
    -
    -
    \ No newline at end of file +
    + +
    diff --git a/server/www/scripts/dashboard/partials/summarized.html b/server/www/scripts/dashboard/partials/summarized.html index f3824898d7e..61f0fc5b1e6 100644 --- a/server/www/scripts/dashboard/partials/summarized.html +++ b/server/www/scripts/dashboard/partials/summarized.html @@ -3,8 +3,8 @@
    -
    -
    +
    +

    Workspace summarized report diff --git a/server/www/scripts/dashboard/partials/top-hosts.html b/server/www/scripts/dashboard/partials/top-hosts.html index 04474c8225c..a8eaabbe71e 100644 --- a/server/www/scripts/dashboard/partials/top-hosts.html +++ b/server/www/scripts/dashboard/partials/top-hosts.html @@ -2,8 +2,8 @@ -
    -
    +
    +

    Top Hosts @@ -19,4 +19,4 @@

    Top Hosts

    -
    \ No newline at end of file +

    diff --git a/server/www/scripts/dashboard/partials/top-services.html b/server/www/scripts/dashboard/partials/top-services.html index 3ee58df5c40..7c12011d6f8 100644 --- a/server/www/scripts/dashboard/partials/top-services.html +++ b/server/www/scripts/dashboard/partials/top-services.html @@ -2,8 +2,8 @@ -
    - -
    \ No newline at end of file +
    diff --git a/server/www/scripts/dashboard/partials/vulns-by-severity.html b/server/www/scripts/dashboard/partials/vulns-by-severity.html index 80b76f394aa..cc4681f0dba 100644 --- a/server/www/scripts/dashboard/partials/vulns-by-severity.html +++ b/server/www/scripts/dashboard/partials/vulns-by-severity.html @@ -1,10 +1,10 @@ -
    -
    +
    +
    -
    -
    +
    +
    {{count}}
    {{severity}}
    diff --git a/server/www/scripts/dashboard/partials/vulns-piechart.html b/server/www/scripts/dashboard/partials/vulns-piechart.html index 76a793dc6df..48860eedbf8 100644 --- a/server/www/scripts/dashboard/partials/vulns-piechart.html +++ b/server/www/scripts/dashboard/partials/vulns-piechart.html @@ -2,8 +2,8 @@ -
    -
    +
    +

    Vulnerabilities @@ -22,4 +22,4 @@

    Vulnerabilities chart-colours="data.colors" chart-options="data.options">

    -
    \ No newline at end of file +
    diff --git a/server/www/scripts/dashboard/partials/workspace-progress.html b/server/www/scripts/dashboard/partials/workspace-progress.html index 4535344058a..9301d513930 100644 --- a/server/www/scripts/dashboard/partials/workspace-progress.html +++ b/server/www/scripts/dashboard/partials/workspace-progress.html @@ -1,24 +1,28 @@ -
    -
    -

    - Workspace progress - -

    -
    -
    -
    - {{wsProgress}}% -
      -
    • Start date: {{wsStart | amUtc | amDateFormat:'MM/DD/YYYY'}}
      • -
    • End date: {{wsEnd | amUtc | amDateFormat:'MM/DD/YYYY'}}
      • -
    -
    -
    - -

    Start date and end date are required

    -
    +
    +
    +

    + Workspace progress + +

    +
    +
    +
    + {{wsProgress}}% +
    +
    +
    + Start date: {{wsStart | amUtc | amDateFormat:'MM/DD/YYYY'}} +
    +
    + End date: {{wsEnd | amUtc | amDateFormat:'MM/DD/YYYY'}} +
    +
    + +

    Start date and end date are required

    +
    +
    From 18419bfbbe78479e690a6a4c81f13bf51c44964c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 22 May 2018 12:51:43 -0300 Subject: [PATCH 1159/1506] Add basic free text search in vulns API Now it only uses name and description fields, it should be improved --- server/api/modules/vulns.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index cefaf85c60c..b5ffa676482 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -453,6 +453,17 @@ def _get_eagerloaded_query(self, *args, **kwargs): [Vulnerability, VulnerabilityWeb] ), *joinedloads) + def _filter_query(self, query): + query = super(VulnerabilityView, self)._filter_query(query) + search_term = flask.request.args.get('search', None) + if search_term is not None: + # TODO migration: add more fields to free text search + like_term = '%' + search_term + '%' + match_name = VulnerabilityGeneric.name.ilike(like_term) + match_desc = VulnerabilityGeneric.description.ilike(like_term) + query = query.filter(match_name | match_desc) + return query + @property def model_class(self): if request.method == 'POST': From c5ae9c433ee5bb85d9e931e5c81460438995fb72 Mon Sep 17 00:00:00 2001 From: montive Date: Tue, 22 May 2018 14:38:38 -0300 Subject: [PATCH 1160/1506] Update status_check --- server/commands/status_check.py | 82 +++++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 5 deletions(-) diff --git a/server/commands/status_check.py b/server/commands/status_check.py index c91931f50b8..b97223fda90 100644 --- a/server/commands/status_check.py +++ b/server/commands/status_check.py @@ -1,7 +1,20 @@ +import requests from colorama import init from colorama import Fore, Back, Style import socket from server.utils.daemonize import is_server_running +import subprocess +import sqlalchemy +from config.configuration import getInstanceConfiguration +from utils import dependencies + + +CONF = getInstanceConfiguration() + +import server.utils.logger + +logger = server.utils.logger.get_logger() + init() @@ -9,17 +22,76 @@ def check_server_running(): pid = is_server_running() if pid is not None: - print('Faraday Server is Running. PID:{PID} {green} Running. \ - '.format(green=Fore.GREEN, PID=pid)) + print('Faraday Server is Running. PID:{PID} {green} Running. {white}\ + '.format(green=Fore.GREEN, PID=pid, white=Fore.WHITE)) return True else: - print('Faraday Server is not running {red} Not Running. \ - '.format(red=Fore.RED)) + print('Faraday Server is not running {red} Not Running. {white} \ + '.format(red=Fore.RED, white=Fore.WHITE)) return True def check_open_ports(): pass +def check_postgres(): + + try: + if db.sessions.query(Workspace).count(): + print("PostgreSQL is running") + except: + print('Could not connect to postgresql, please check if database is running') + + + +def check_client(): + port_rest = CONF.getApiRestfulConInfoPort() + + try: + response_rest = requests.get('http://localhost:%s/status/check' % port_rest) + print "Faraday GTK is running" + except: + print('WARNING. Faraday GTK is not running') + +def check_server_dependencies(): + + installed_deps, missing_deps, conflict_deps = dependencies.check_dependencies( + requirements_file='requirements_server.txt') + + print("Checking server's dependencies") + + + if conflict_deps: + print("Some dependencies are old. Update them with \"pip install -r requirements_server.txt -U\"") + print("Dependecies to update: ",", ".join(conflict_deps)) + + + if missing_deps: + print("Dependencies not met. Please refer to the documentation in order to install them. ", + ", ".join(missing_deps)) + +def check_gtk_dependencies(): + installed_deps, missing_deps, conflict_deps = dependencies.check_dependencies( + requirements_file='requirements.txt') + + print("Checking GTK's dependencies") + + + if conflict_deps: + print("Some dependencies are old. Update them with \"pip install -r requirements_server.txt -U\"") + print("Dependecies to update: ",", ".join(conflict_deps)) + + + if missing_deps: + print("Dependencies not met. Please refer to the documentation in order to install them. ", + ", ".join(missing_deps)) + + + def full_status_check(): + print('{red} Incomplete Check {white}.'.format(red=Fore.RED, white=Fore.WHITE)) check_server_running() - + check_open_ports() + check_postgres() + check_client() + check_server_dependencies() + check_gtk_dependencies() \ No newline at end of file From 5ea7bddbcec7bd8f6b89930d413f8c7536510c54 Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Wed, 23 May 2018 10:07:18 -0300 Subject: [PATCH 1161/1506] 4721 - Update activity feed and last vulnerabilities table --- server/www/dashboard-v3.css | 16 ++++++++++++++++ .../scripts/dashboard/partials/activityFeed.html | 8 ++++---- .../scripts/dashboard/partials/dashboard.html | 2 +- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/server/www/dashboard-v3.css b/server/www/dashboard-v3.css index 5c3e1104fea..c3e4392ddfd 100644 --- a/server/www/dashboard-v3.css +++ b/server/www/dashboard-v3.css @@ -87,3 +87,19 @@ .text-left { text-align: left !important; } + +.dashboard .progress-bar span { + font-weight: bold; +} + +.dashboard table thead th span { + color: #3393d4; +} + +.fg-blue { + color: #3393d4; +} + +.fg-light-gray { + color: #a1a1a1; +} diff --git a/server/www/scripts/dashboard/partials/activityFeed.html b/server/www/scripts/dashboard/partials/activityFeed.html index 8aced40ea16..94196379865 100644 --- a/server/www/scripts/dashboard/partials/activityFeed.html +++ b/server/www/scripts/dashboard/partials/activityFeed.html @@ -20,9 +20,9 @@

    Activity Feed

    diff --git a/server/www/scripts/dashboard/partials/dashboard.html b/server/www/scripts/dashboard/partials/dashboard.html index 43d9f9337a4..8ba9a14c848 100644 --- a/server/www/scripts/dashboard/partials/dashboard.html +++ b/server/www/scripts/dashboard/partials/dashboard.html @@ -2,7 +2,7 @@ -
    +
    From d81ed585cf66707e288f5f8a971ec9a9e4e1ba21 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 23 May 2018 12:25:27 -0300 Subject: [PATCH 1162/1506] [FIX] persistance.server.models.update_vuln_web [ADD] Test for update_vuln --- persistence/server/models.py | 2 +- test_cases/test_persistence_server_models.py | 44 ++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/persistence/server/models.py b/persistence/server/models.py index f05d4df3299..3feb41d453c 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -414,7 +414,7 @@ def create_vuln_web(workspace_name, vuln_web, command_id=None): @_ignore_in_changes -def update_vuln_web(workspace_name, vuln_web, command_id): +def update_vuln_web(workspace_name, vuln_web, command_id=None): """Take a workspace_name and a VulnWeb object and update it in the sever. Return the server's json response as a dictionary. diff --git a/test_cases/test_persistence_server_models.py b/test_cases/test_persistence_server_models.py index e69de29bb2d..f0740eb9551 100644 --- a/test_cases/test_persistence_server_models.py +++ b/test_cases/test_persistence_server_models.py @@ -0,0 +1,44 @@ + +import persistence.server.models as models +import pytest +import responses + +from test_api_workspaced_base import GenericAPITest + +from test_cases.factories import ServiceFactory, CommandFactory, \ + CommandObjectFactory, HostFactory, EmptyCommandFactory, \ + UserFactory, VulnerabilityWebFactory, VulnerabilityFactory, \ + ReferenceFactory, PolicyViolationFactory + +@pytest.mark.usefixtures('logged_user') +class TestModelsFuncions(GenericAPITest): + + factory = VulnerabilityFactory + + def setUp(self): + pass + + @responses.activate + def test_persistence_server_update_vuln(self): + + fo = self.first_object + + vuln = {} + vuln['desc'] = fo.description + vuln['data'] = fo.data + vuln['severity'] = fo.severity + vuln['refs'] = list(fo.references) + vuln['confirmed'] = fo.confirmed + vuln['resolution'] = fo.resolution + vuln['status'] = fo.status + vuln['policyviolations'] = list(fo.policy_violations) + + v = models.Vuln(vuln, self.workspace.name) + v.id = fo.id + + responses.add(responses.PUT, 'http://localhost:5985/_api/v2/ws/{0}/vulns/{1}/'.format(self.workspace.name, v.id), + json={}, status=200) + + models.update_vuln(self.workspace.name, v) + + From b7148c9a84a297c4a8b32f3dd9124121ec05503a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 23 May 2018 12:30:37 -0300 Subject: [PATCH 1163/1506] Improve start log messages and excepition logging in the server ./faraday-server.py now shows the URL of the server instead of the websocket one --- server/web.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/server/web.py b/server/web.py index 222121efa19..af78e5de6e3 100644 --- a/server/web.py +++ b/server/web.py @@ -74,10 +74,10 @@ class WebServer(object): WEB_UI_LOCAL_PATH = os.path.join(server.config.FARADAY_BASE, 'server/www') def __init__(self, enable_ssl=False): - logger.info('Starting web server at port {0} with bind address {1}. SSL {2}'.format( - server.config.faraday_server.port, + logger.info('Starting web server at {}://{}:{}/'.format( + 'https' if enable_ssl else 'http', server.config.faraday_server.bind_address, - enable_ssl)) + server.config.faraday_server.port)) self.__ssl_enabled = enable_ssl self.__config_server() self.__build_server_tree() @@ -118,7 +118,13 @@ def __build_websockets_resource(self): url = 'wss://' + url else: url = 'ws://' + url - logger.info(u"Websocket listening at {url}".format(url=url)) + # logger.info(u"Websocket listening at {url}".format(url=url)) + logger.info('Starting websocket server at port {0} with bind address {1}. ' + 'SSL {2}'.format( + websocket_port, + self.__bind_address, + self.__ssl_enabled + )) factory = WorkspaceServerFactory(url=url) factory.protocol = BroadcastServerProtocol @@ -146,6 +152,6 @@ def run(self): logger.error(str(e)) sys.exit(1) except Exception as e: - logger.debug(e) logger.error('Something went wrong when trying to setup the Web UI') + logger.exception(e) sys.exit(1) From be017fed74fb0ae69fe7e7307c09a2caa6124ac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 23 May 2018 13:37:41 -0300 Subject: [PATCH 1164/1506] Move filteralchemy fork to infobyte account --- requirements_server.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_server.txt b/requirements_server.txt index 070182cc2c9..2eb36716da7 100644 --- a/requirements_server.txt +++ b/requirements_server.txt @@ -23,7 +23,7 @@ tqdm==4.15.0 twisted==17.5.0 webargs==3.0.0 marshmallow-sqlalchemy -git+https://github.com/sh4r3m4n/filteralchemy@dev#egg=filteralchemy +git+https://github.com/infobyte/filteralchemy@dev#egg=filteralchemy filedepot==0.5.0 nplusone==0.8.1 deprecation==1.0.1 From 47c0236528d97025b85ffbd576569ce1718f4b39 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 23 May 2018 16:25:07 -0300 Subject: [PATCH 1165/1506] [ADD] persistance.server.models.update_vuln/update_veln_web testing --- test_cases/test_persistence_server_models.py | 149 ++++++++++++++++--- 1 file changed, 126 insertions(+), 23 deletions(-) diff --git a/test_cases/test_persistence_server_models.py b/test_cases/test_persistence_server_models.py index f0740eb9551..8d626acab10 100644 --- a/test_cases/test_persistence_server_models.py +++ b/test_cases/test_persistence_server_models.py @@ -1,44 +1,147 @@ - import persistence.server.models as models import pytest import responses +import requests from test_api_workspaced_base import GenericAPITest -from test_cases.factories import ServiceFactory, CommandFactory, \ - CommandObjectFactory, HostFactory, EmptyCommandFactory, \ - UserFactory, VulnerabilityWebFactory, VulnerabilityFactory, \ - ReferenceFactory, PolicyViolationFactory +from test_cases.factories import VulnerabilityWebFactory, VulnerabilityFactory -@pytest.mark.usefixtures('logged_user') -class TestModelsFuncions(GenericAPITest): +@pytest.mark.usefixtures('logged_user') +class TestVulnPersistanceModelsFuncions(GenericAPITest): factory = VulnerabilityFactory - def setUp(self): - pass - @responses.activate def test_persistence_server_update_vuln(self): - fo = self.first_object - vuln = {} - vuln['desc'] = fo.description - vuln['data'] = fo.data - vuln['severity'] = fo.severity - vuln['refs'] = list(fo.references) - vuln['confirmed'] = fo.confirmed - vuln['resolution'] = fo.resolution - vuln['status'] = fo.status - vuln['policyviolations'] = list(fo.policy_violations) + vuln = {'desc': fo.description, 'data': fo.data, 'severity': fo.severity, 'refs': list(fo.references), + 'confirmed': fo.confirmed, 'resolution': fo.resolution, 'status': fo.status, + 'policyviolations': list(fo.policy_violations)} v = models.Vuln(vuln, self.workspace.name) v.id = fo.id - responses.add(responses.PUT, 'http://localhost:5985/_api/v2/ws/{0}/vulns/{1}/'.format(self.workspace.name, v.id), - json={}, status=200) + resp = {u'status': u'closed', + u'_rev': u'', + u'parent_type': v.getParentType(), + u'owned': v.isOwned(), + u'owner': v.getParent(), + u'query': u'', + u'refs': v.getRefs(), + u'impact': {u'integrity': False, u'confidentiality': False, u'availability': False, + u'accountability': False}, + u'confirmed': v.getConfirmed(), + u'severity': v.getSeverity(), + u'service': None, + u'policyviolations': v.getPolicyViolations(), + u'params': u'', + u'type': u'Vulnerability', + u'method': u'', + u'metadata': {u'update_time': u'2018-05-23T17:03:27.880196+00:00', u'update_user': u'', + u'update_action': 0, u'creator': u'Nmap', + u'create_time': u'2018-05-18T16:30:26.011851+00:00', + u'update_controller_action': u'', u'owner': u'faraday', u'command_id': 22}, + u'website': u'', + u'issuetracker': {}, + u'description': v.getDesc(), + u'tags': [], + u'easeofresolution': None, + u'hostnames': [], + u'pname': u'', + u'date': u'2018-05-18T16:30:26.011851+00:00', + u'path': u'', + u'data': v.getData(), + u'response': u'', + u'desc': v.getDesc(), + u'name': v.getName(), + u'obj_id': str(v.getID()), + u'request': u'', + u'_attachments': {}, + u'target': u'192.168.10.103', + u'_id': v.getID(), + u'resolution': v.getResolution() + } + responses.add(responses.PUT, + 'http://localhost:5985/_api/v2/ws/{0}/vulns/{1}/'.format(self.workspace.name, v.id), + json=resp, status=200) + + a = requests.put('http://localhost:5985/_api/v2/ws/{0}/vulns/{1}/'.format(self.workspace.name, v.id)) + + try: + models.update_vuln(self.workspace.name, v) + except TypeError as e: + pytest.fail(e.message) + + +@pytest.mark.usefixtures('logged_user') +class TestVulnWebPersistanceModelsFuncions(GenericAPITest): + factory = VulnerabilityWebFactory + + @responses.activate + def test_persistence_server_update_vuln_web(self): + fo = self.first_object + + vuln_web = {'desc': fo.description, 'data': fo.data, 'severity': fo.severity, 'refs': list(fo.references), + 'confirmed': fo.confirmed, 'resolution': fo.resolution, 'status': fo.status, + 'policyviolations': list(fo.policy_violations), 'path': fo.path, 'website': fo.website, + 'request': fo.request, 'response': fo.response, 'method': fo.method, 'params': fo.parameters, + 'pname': fo.parameter_name, 'query': str(fo.query), '_attachments': fo.attachments, + 'hostnames': list(fo.hostnames), + 'impact': {'accountability': fo.impact_accountability, 'availability': fo.impact_availability, + 'confidentiality': fo.impact_confidentiality, 'integrity': fo.impact_integrity}, + 'service': fo.service_id, 'tags': list(fo.tags), 'target': fo.target_host_ip} + + v = models.VulnWeb(vuln_web, self.workspace.name) + v.id = fo.id - models.update_vuln(self.workspace.name, v) + resp = {u'status': u'closed', + u'_rev': u'', + u'parent_type': v.getParentType(), + u'owned': v.isOwned(), + u'owner': v.getParent(), + u'query': str(v.getQuery()), + u'refs': v.getRefs(), + u'impact': v.getImpact(), + u'confirmed': v.getConfirmed(), + u'severity': v.getSeverity(), + u'service': v.getService(), + u'policyviolations': v.getPolicyViolations(), + u'params': v.getParams(), + u'type': u'VulnerabilityWeb', + u'method': v.getMethod(), + u'metadata': {u'update_time': u'2018-05-23T17:03:27.880196+00:00', u'update_user': u'', + u'update_action': 0, u'creator': u'Nmap', + u'create_time': u'2018-05-18T16:30:26.011851+00:00', + u'update_controller_action': u'', u'owner': u'faraday', u'command_id': 22}, + u'website': v.getWebsite(), + u'issuetracker': {}, + u'description': v.getDesc(), + u'tags': v.getTags(), + u'easeofresolution': None, + u'hostnames': v.getHostnames(), + u'pname': v.getPname(), + u'date': u'2018-05-18T16:30:26.011851+00:00', + u'path': v.getPath(), + u'data': v.getData(), + u'response': v.getResponse(), + u'desc': v.getDesc(), + u'name': v.getName(), + u'obj_id': str(v.getID()), + u'request': v.getRequest(), + u'_attachments': str(v.getAttachments()), + u'target': v.getTarget(), + u'_id': v.getID(), + u'resolution': v.getResolution() + } + responses.add(responses.PUT, + 'http://localhost:5985/_api/v2/ws/{0}/vulns/{1}/'.format(self.workspace.name, v.id), + json=resp, status=200) + a = requests.put('http://localhost:5985/_api/v2/ws/{0}/vulns/{1}/'.format(self.workspace.name, v.id)) + try: + models.update_vuln_web(self.workspace.name, v) + except TypeError as e: + pytest.fail(e.message) From 30c99641a76ad1bf8dc95e304c12dd93c506bd05 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 23 May 2018 16:57:34 -0300 Subject: [PATCH 1166/1506] [FIX] Port gotten from config --- test_cases/test_persistence_server_models.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/test_cases/test_persistence_server_models.py b/test_cases/test_persistence_server_models.py index 8d626acab10..6f6ad78c6d6 100644 --- a/test_cases/test_persistence_server_models.py +++ b/test_cases/test_persistence_server_models.py @@ -3,6 +3,8 @@ import responses import requests +import server.config + from test_api_workspaced_base import GenericAPITest from test_cases.factories import VulnerabilityWebFactory, VulnerabilityFactory @@ -63,11 +65,14 @@ def test_persistence_server_update_vuln(self): u'_id': v.getID(), u'resolution': v.getResolution() } + + port = server.config.faraday_server.port + responses.add(responses.PUT, - 'http://localhost:5985/_api/v2/ws/{0}/vulns/{1}/'.format(self.workspace.name, v.id), + 'http://localhost:{0}/_api/v2/ws/{1}/vulns/{2}/'.format(port,self.workspace.name, v.id), json=resp, status=200) - a = requests.put('http://localhost:5985/_api/v2/ws/{0}/vulns/{1}/'.format(self.workspace.name, v.id)) + a = requests.put('http://localhost:{0}/_api/v2/ws/{1}/vulns/{2}/'.format(port,self.workspace.name, v.id)) try: models.update_vuln(self.workspace.name, v) @@ -135,11 +140,14 @@ def test_persistence_server_update_vuln_web(self): u'_id': v.getID(), u'resolution': v.getResolution() } + + port = server.config.faraday_server.port + responses.add(responses.PUT, - 'http://localhost:5985/_api/v2/ws/{0}/vulns/{1}/'.format(self.workspace.name, v.id), + 'http://localhost:{0}/_api/v2/ws/{1}/vulns/{2}/'.format(port,self.workspace.name, v.id), json=resp, status=200) - a = requests.put('http://localhost:5985/_api/v2/ws/{0}/vulns/{1}/'.format(self.workspace.name, v.id)) + a = requests.put('http://localhost:{0}/_api/v2/ws/{1}/vulns/{2}/'.format(port,self.workspace.name, v.id)) try: models.update_vuln_web(self.workspace.name, v) From 6ea01b953d1941804f018422de57957eda056af8 Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Tue, 22 May 2018 10:11:42 -0300 Subject: [PATCH 1167/1506] 4725 - Update input and input placeholder color --- server/www/styles/material-input.css | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/www/styles/material-input.css b/server/www/styles/material-input.css index 3bfc0903426..80b6e78d8f1 100644 --- a/server/www/styles/material-input.css +++ b/server/www/styles/material-input.css @@ -18,7 +18,7 @@ transition: all 100ms ease-in-out; width: 100%; cursor: text; - color: #ddddd; + color: #a1a1a1; } .form-input input, .form-input textarea { @@ -29,11 +29,12 @@ font-size: 16px; font-weight: 300; border: 0; - border-bottom: 1px dashed #A1A1A1; + border-bottom: 1px dashed #d4d4d4; transition: border-color 100ms ease-in-out; outline: none; padding: 0; margin: 0; + color: #a1a1a1; } .form-input textarea { min-height: 30px; From 5bb3db5b9f388cc66552b92743e2e67f66147825 Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Tue, 22 May 2018 10:11:46 -0300 Subject: [PATCH 1168/1506] 4725 - Update password input lock size --- server/www/estilos-v3.css | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/www/estilos-v3.css b/server/www/estilos-v3.css index 63de724c953..e7c152cac84 100644 --- a/server/www/estilos-v3.css +++ b/server/www/estilos-v3.css @@ -122,13 +122,13 @@ body { .frd-icon-lock { background-image: url("images/icon-login-password-lock.svg"); - width: 24px; - height: 24px; + width: 20px; + height: 20px; } #form-signin .placeholder .frd-icon-lock { position: relative; - bottom: 10px; + bottom: 5px; } #footer-login { From 2a6ab606c0ccc2d5651fe799fdb20f5910f568e5 Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Tue, 22 May 2018 10:12:01 -0300 Subject: [PATCH 1169/1506] 4725 - Update password input lock image --- server/www/images/icon-login-password-lock.svg | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/www/images/icon-login-password-lock.svg b/server/www/images/icon-login-password-lock.svg index 496c2b5c105..c9b623d5c3a 100644 --- a/server/www/images/icon-login-password-lock.svg +++ b/server/www/images/icon-login-password-lock.svg @@ -3,15 +3,15 @@ - - + - From 3bc49a91c92d9ca88f295fb457f331a3a3398e23 Mon Sep 17 00:00:00 2001 From: montive Date: Wed, 23 May 2018 18:15:48 -0300 Subject: [PATCH 1170/1506] Update status_check.py --- server/commands/status_check.py | 53 ++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/server/commands/status_check.py b/server/commands/status_check.py index b97223fda90..02fd8369dc0 100644 --- a/server/commands/status_check.py +++ b/server/commands/status_check.py @@ -1,24 +1,24 @@ import requests +import subprocess +import sqlalchemy +import socket +import server.utils.logger from colorama import init from colorama import Fore, Back, Style -import socket from server.utils.daemonize import is_server_running -import subprocess -import sqlalchemy +from server.models import db +from server.web import app from config.configuration import getInstanceConfiguration from utils import dependencies CONF = getInstanceConfiguration() -import server.utils.logger - logger = server.utils.logger.get_logger() init() - def check_server_running(): pid = is_server_running() if pid is not None: @@ -34,14 +34,12 @@ def check_open_ports(): pass def check_postgres(): - - try: - if db.sessions.query(Workspace).count(): + with app.app_context(): + try: + result = str(db.engine.execute("SELECT version()")) print("PostgreSQL is running") - except: - print('Could not connect to postgresql, please check if database is running') - - + except sqlalchemy.exc.OperationalError: + print('Could not connect to postgresql, please check if database is running') def check_client(): port_rest = CONF.getApiRestfulConInfoPort() @@ -49,7 +47,7 @@ def check_client(): try: response_rest = requests.get('http://localhost:%s/status/check' % port_rest) print "Faraday GTK is running" - except: + except requests.exceptions.ConnectionError: print('WARNING. Faraday GTK is not running') def check_server_dependencies(): @@ -57,14 +55,10 @@ def check_server_dependencies(): installed_deps, missing_deps, conflict_deps = dependencies.check_dependencies( requirements_file='requirements_server.txt') - print("Checking server's dependencies") - - if conflict_deps: print("Some dependencies are old. Update them with \"pip install -r requirements_server.txt -U\"") print("Dependecies to update: ",", ".join(conflict_deps)) - if missing_deps: print("Dependencies not met. Please refer to the documentation in order to install them. ", ", ".join(missing_deps)) @@ -73,19 +67,29 @@ def check_gtk_dependencies(): installed_deps, missing_deps, conflict_deps = dependencies.check_dependencies( requirements_file='requirements.txt') - print("Checking GTK's dependencies") - - if conflict_deps: print("Some dependencies are old. Update them with \"pip install -r requirements_server.txt -U\"") print("Dependecies to update: ",", ".join(conflict_deps)) - if missing_deps: print("Dependencies not met. Please refer to the documentation in order to install them. ", ", ".join(missing_deps)) - +def check_credentials(): + api_username = CONF.getAPIUsername() + api_password = CONF.getAPIPassword() + + values = {'email': api_username , 'password': api_password} + + r = requests.post('http://localhost:5985/_api/login', json=values) + + if r.status_code == 200 and 'user' in r.json()['response']: + print('Credentials matched') + elif r.status_code == 400: + print('Error. Credentials does not match') + elif r.status_code == 500: + print('Server failed with unexpected error. check if databaseservice is working') + def full_status_check(): print('{red} Incomplete Check {white}.'.format(red=Fore.RED, white=Fore.WHITE)) @@ -94,4 +98,5 @@ def full_status_check(): check_postgres() check_client() check_server_dependencies() - check_gtk_dependencies() \ No newline at end of file + check_gtk_dependencies() + check_credentials() \ No newline at end of file From 797320f2531be7296b9992e4d77ee80f4ef79246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 23 May 2018 19:20:34 -0300 Subject: [PATCH 1171/1506] Add authentication and authorization to websocket server Changes to the GTK client are really horrible, but they (appearently) work! --- persistence/server/changes_stream.py | 7 ++++++ server/api/modules/websocket_auth.py | 23 ++++++++++++++++++ server/app.py | 2 ++ server/websocket_factories.py | 35 +++++++++++++++++++++++++-- test_cases/test_api_websocket_auth.py | 26 ++++++++++++++++++++ 5 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 server/api/modules/websocket_auth.py create mode 100644 test_cases/test_api_websocket_auth.py diff --git a/persistence/server/changes_stream.py b/persistence/server/changes_stream.py index 1ce4367a618..f7f0c9db46f 100644 --- a/persistence/server/changes_stream.py +++ b/persistence/server/changes_stream.py @@ -92,9 +92,16 @@ def stop(self): super(WebsocketsChangesStream, self).stop() def on_open(self, ws): + from persistence.server.server import _create_server_api_url, _post + r = _post( + _create_server_api_url() + + '/ws/{}/websocket_token/'.format(self.workspace_name), + expected_response=200) + token = r['token'] self.ws.send(json.dumps({ 'action': 'JOIN_WORKSPACE', 'workspace': self.workspace_name, + 'token': token, })) def on_message(self, ws, message): diff --git a/server/api/modules/websocket_auth.py b/server/api/modules/websocket_auth.py new file mode 100644 index 00000000000..5a38148ce29 --- /dev/null +++ b/server/api/modules/websocket_auth.py @@ -0,0 +1,23 @@ +# Faraday Penetration Test IDE +# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) +# See the file 'doc/LICENSE' for the license information + +from flask import Blueprint +from flask import current_app as app +from itsdangerous import TimestampSigner +from server.api.base import GenericWorkspacedView + +websocket_auth_api = Blueprint('websocket_auth_api', __name__) + + +class WebsocketAuthView(GenericWorkspacedView): + route_base = 'websocket_token' + + def post(self, workspace_name): + workspace = self._get_workspace(workspace_name) + signer = TimestampSigner(app.config['SECRET_KEY'], salt="websocket") + token = signer.sign(str(workspace.id)) + return {"token": token} + + +WebsocketAuthView.register(websocket_auth_api) diff --git a/server/app.py b/server/app.py index 4bd204cc075..8fe93eb005c 100644 --- a/server/app.py +++ b/server/app.py @@ -64,6 +64,7 @@ def register_blueprints(app): from server.api.modules.workspaces import workspace_api from server.api.modules.handlers import handlers_api from server.api.modules.comments import comment_api + from server.api.modules.websocket_auth import websocket_auth_api app.register_blueprint(commandsrun_api) app.register_blueprint(credentials_api) app.register_blueprint(host_api) @@ -76,6 +77,7 @@ def register_blueprints(app): app.register_blueprint(workspace_api) app.register_blueprint(handlers_api) app.register_blueprint(comment_api) + app.register_blueprint(websocket_auth_api) def check_testing_configuration(testing, app): diff --git a/server/websocket_factories.py b/server/websocket_factories.py index fb4d2169e0a..75ba8732595 100644 --- a/server/websocket_factories.py +++ b/server/websocket_factories.py @@ -1,7 +1,9 @@ import json import logging +import itsdangerous import Cookie +import server.utils from collections import defaultdict from Queue import Queue, Empty @@ -13,7 +15,9 @@ WebSocketServerProtocol ) -logger =logging.getLogger(__name__) +from server.models import Workspace + +logger = server.utils.logger.get_logger(__name__) changes_queue = Queue() @@ -32,6 +36,7 @@ def onConnect(self, request): return (protocol, headers) def onMessage(self, payload, is_binary): + from server.web import app """ We only support JOIN and LEAVE workspace messages. When authentication is implemented we need to verify @@ -42,7 +47,33 @@ def onMessage(self, payload, is_binary): if not is_binary: message = json.loads(payload) if message['action'] == 'JOIN_WORKSPACE': - self.factory.join_workspace(self, message['workspace']) + if 'workspace' not in message or 'token' not in message: + logger.warning('Invalid join workspace message: ' + '{}'.format(message)) + self.sendClose() + return + signer = itsdangerous.TimestampSigner(app.config['SECRET_KEY'], + salt="websocket") + try: + workspace_id = signer.unsign(message['token'], max_age=60) + except itsdangerous.BadData as e: + self.sendClose() + logger.warning('Invalid websocket token for workspace ' + '{}'.format(message['workspace'])) + logger.exception(e) + else: + with app.app_context(): + workspace = Workspace.query.get(int(workspace_id)) + if workspace.name != message['workspace']: + logger.warning( + 'Trying to join workspace {} with token of ' + 'workspace {}. Rejecting.'.format( + message['workspace'], workspace.name + )) + self.sendClose() + else: + self.factory.join_workspace( + self, message['workspace']) if message['action'] == 'LEAVE_WORKSPACE': self.factory.leave_workspace(self, message['workspace']) diff --git a/test_cases/test_api_websocket_auth.py b/test_cases/test_api_websocket_auth.py new file mode 100644 index 00000000000..3cc6d28ebcd --- /dev/null +++ b/test_cases/test_api_websocket_auth.py @@ -0,0 +1,26 @@ +import pytest + + +class TestWebsocketAuthEndpoint: + + def test_not_logged_in_request_fail(self, test_client, workspace): + res = test_client.post('/v2/ws/{}/websocket_token/'.format( + workspace.name)) + assert res.status_code == 401 + + @pytest.mark.usefixtures('logged_user') + def test_get_method_not_allowed(self, test_client, workspace): + res = test_client.get('/v2/ws/{}/websocket_token/'.format( + workspace.name)) + assert res.status_code == 405 + + @pytest.mark.usefixtures('logged_user') + def test_succeeds(self, test_client, workspace): + res = test_client.post('/v2/ws/{}/websocket_token/'.format( + workspace.name)) + assert res.status_code == 200 + + # A token for that workspace should be generated, + # This will break if we change the token generation + # mechanism. + assert res.json['token'].startswith(str(workspace.id)) From fd1d42e0512554ef8861a0cf9fea7d278945e641 Mon Sep 17 00:00:00 2001 From: Nicolas Ronchi Cesarini Date: Mon, 21 May 2018 15:21:25 -0300 Subject: [PATCH 1172/1506] Cambios en status report --- server/www/estilos.css | 3 +- server/www/images/icon-list-confirmed.svg | 11 ++ server/www/images/icon-list-notconfirmed.svg | 15 ++ .../www/images/icon-toolbar-confirmed-off.svg | 19 +++ .../www/images/icon-toolbar-confirmed-on.svg | 13 ++ server/www/images/icon-toolbar-delete.svg | 18 ++ server/www/images/icon-toolbar-download.svg | 20 +++ server/www/images/icon-toolbar-edit.svg | 19 +++ .../images/icon-toolbar-searchbar-loupe.svg | 13 ++ server/www/index.html | 1 + .../statusReport/controllers/statusReport.js | 47 +++--- .../statusReport/partials/statusReport.html | 151 ++++++++--------- .../partials/ui-grid/columns/datecolumn.html | 2 +- .../partials/ui-grid/columns/idcolumn.html | 5 +- .../ui-grid/columns/severitycolumn.html | 12 +- .../ui-grid/columns/statuscolumn.html | 51 +++++- .../partials/ui-grid/confirmbutton.html | 4 +- .../scripts/statusReport/styles/status.css | 158 ++++++++++++++++++ .../www/scripts/vulndb/partials/vulndb.html | 4 +- 19 files changed, 446 insertions(+), 120 deletions(-) create mode 100644 server/www/images/icon-list-confirmed.svg create mode 100644 server/www/images/icon-list-notconfirmed.svg create mode 100644 server/www/images/icon-toolbar-confirmed-off.svg create mode 100644 server/www/images/icon-toolbar-confirmed-on.svg create mode 100644 server/www/images/icon-toolbar-delete.svg create mode 100644 server/www/images/icon-toolbar-download.svg create mode 100644 server/www/images/icon-toolbar-edit.svg create mode 100644 server/www/images/icon-toolbar-searchbar-loupe.svg create mode 100644 server/www/scripts/statusReport/styles/status.css diff --git a/server/www/estilos.css b/server/www/estilos.css index 8c7998a6c2c..e126ec7f255 100644 --- a/server/www/estilos.css +++ b/server/www/estilos.css @@ -727,8 +727,7 @@ text.host-name{font-size: 13px;font-weight:bold;} color: #000000 !important; } } - /* Workspaces list - Open*/ - div.button-control.col-md-6.col-sm-6.col-xs-12 {display: block; float: right} + #sec { float: right; margin-top: -34px; diff --git a/server/www/images/icon-list-confirmed.svg b/server/www/images/icon-list-confirmed.svg new file mode 100644 index 00000000000..03f543428bc --- /dev/null +++ b/server/www/images/icon-list-confirmed.svg @@ -0,0 +1,11 @@ + + + + + + + diff --git a/server/www/images/icon-list-notconfirmed.svg b/server/www/images/icon-list-notconfirmed.svg new file mode 100644 index 00000000000..72f0f7261be --- /dev/null +++ b/server/www/images/icon-list-notconfirmed.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + diff --git a/server/www/images/icon-toolbar-confirmed-off.svg b/server/www/images/icon-toolbar-confirmed-off.svg new file mode 100644 index 00000000000..55343e5fcd9 --- /dev/null +++ b/server/www/images/icon-toolbar-confirmed-off.svg @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + diff --git a/server/www/images/icon-toolbar-confirmed-on.svg b/server/www/images/icon-toolbar-confirmed-on.svg new file mode 100644 index 00000000000..4566210c364 --- /dev/null +++ b/server/www/images/icon-toolbar-confirmed-on.svg @@ -0,0 +1,13 @@ + + + + + + + + + diff --git a/server/www/images/icon-toolbar-delete.svg b/server/www/images/icon-toolbar-delete.svg new file mode 100644 index 00000000000..db7d415472d --- /dev/null +++ b/server/www/images/icon-toolbar-delete.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + diff --git a/server/www/images/icon-toolbar-download.svg b/server/www/images/icon-toolbar-download.svg new file mode 100644 index 00000000000..c9c846f31f0 --- /dev/null +++ b/server/www/images/icon-toolbar-download.svg @@ -0,0 +1,20 @@ + + + + + + + + + + + diff --git a/server/www/images/icon-toolbar-edit.svg b/server/www/images/icon-toolbar-edit.svg new file mode 100644 index 00000000000..99b782229de --- /dev/null +++ b/server/www/images/icon-toolbar-edit.svg @@ -0,0 +1,19 @@ + + + + + + + + + + diff --git a/server/www/images/icon-toolbar-searchbar-loupe.svg b/server/www/images/icon-toolbar-searchbar-loupe.svg new file mode 100644 index 00000000000..8dccf1b48a1 --- /dev/null +++ b/server/www/images/icon-toolbar-searchbar-loupe.svg @@ -0,0 +1,13 @@ + + + + + + + diff --git a/server/www/index.html b/server/www/index.html index 485a6b7af30..e75faa521a2 100644 --- a/server/www/index.html +++ b/server/www/index.html @@ -31,6 +31,7 @@ + diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index d90eb8dbb17..e66f8a43a5a 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -180,13 +180,13 @@ angular.module("faradayApp") "target": true, "desc": true, "resolution": false, - "data": true, + "data": false, "easeofresolution": false, - "status": false, + "status": true, "website": false, "path": false, "request": false, - "refs": true, + "refs": false, "evidence": false, "hostnames": true, "impact": false, @@ -208,9 +208,9 @@ angular.module("faradayApp") $scope.columnsWidths = { "name": "120", "service": "110", - "hostnames": "100", + "hostnames": "130", "target": "100", - "desc": "300", + "desc": "200", "resolution": "170", "data": "170", "easeofresolution": "140", @@ -261,8 +261,7 @@ angular.module("faradayApp") }; var defineColumns = function() { - $scope.gridOptions.columnDefs.push({ name: "selectAll", width: "20", enableColumnResizing: false,headerCellTemplate: "", pinnedLeft:true }); - $scope.gridOptions.columnDefs.push({ name: "confirmVuln", width: "23", enableColumnResizing: false, headerCellTemplate: "
    ", cellTemplate: "scripts/statusReport/partials/ui-grid/confirmbutton.html" }); + $scope.gridOptions.columnDefs.push({ name: "confirmVuln", width: "23", enableColumnResizing: false, headerCellTemplate: "", cellTemplate: "scripts/statusReport/partials/ui-grid/confirmbutton.html" }); function getColumnSort(columnName){ if($cookies.get("SRsortColumn") === columnName){ @@ -278,7 +277,7 @@ angular.module("faradayApp") var header = '
    '+ '
    {{ col.displayName CUSTOM_FILTERS }}'+ ' '+ - ' '+ + ' '+ ' '+ '  '+ '
    '+ @@ -307,15 +306,6 @@ angular.module("faradayApp") sortingAlgorithm: compareSeverities, enableColumnResizing: false }); - $scope.gridOptions.columnDefs.push({ name : 'date', - displayName : "date", - cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/datecolumn.html', - headerCellTemplate: header, - width: '80', - enableColumnResizing: false, - sort: getColumnSort('date'), - visible: $scope.columns["date"] - }); $scope.gridOptions.columnDefs.push({ name : 'name', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/namecolumn.html', headerCellTemplate: header, @@ -375,13 +365,6 @@ angular.module("faradayApp") visible: $scope.columns["easeofresolution"], width: $scope.columnsWidths['easeofresolution'], }); - $scope.gridOptions.columnDefs.push({ name : 'status', - cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/statuscolumn.html', - headerCellTemplate: header, - width: $scope.columnsWidths['status'], - sort: getColumnSort('status'), - visible: $scope.columns["status"] - }); $scope.gridOptions.columnDefs.push({ name : 'website', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/defaultcolumn.html', headerCellTemplate: header, @@ -403,6 +386,22 @@ angular.module("faradayApp") visible: $scope.columns["request"], width: $scope.columnsWidths['request'], }); + $scope.gridOptions.columnDefs.push({ name : 'date', + displayName : "date", + cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/datecolumn.html', + headerCellTemplate: header, + width: '80', + enableColumnResizing: false, + sort: getColumnSort('date'), + visible: $scope.columns["date"] + }); + $scope.gridOptions.columnDefs.push({ name : 'status', + cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/statuscolumn.html', + headerCellTemplate: header, + width: $scope.columnsWidths['status'], + sort: getColumnSort('status'), + visible: $scope.columns["status"] + }); $scope.gridOptions.columnDefs.push({ name : 'refs', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/refscolumn.html', headerCellTemplate: header, diff --git a/server/www/scripts/statusReport/partials/statusReport.html b/server/www/scripts/statusReport/partials/statusReport.html index 9138d004fb8..f3eae20e93e 100644 --- a/server/www/scripts/statusReport/partials/statusReport.html +++ b/server/www/scripts/statusReport/partials/statusReport.html @@ -2,60 +2,24 @@ -
    +
    - -
    - - - -
    - - -
    -
    - - -
    -
    - - - -
    - -
    -
    -
    - - -
    +
    + -
    - -
    - -
    -
    -
    -
    -
    -
    - - - - - - -
    -
    - +
    +
    +
    + + + + +
    +
    + + +
    - -
    +
    +
    + +
    Total diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/datecolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/datecolumn.html index 1a5ab831bca..429862c4e54 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/datecolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/datecolumn.html @@ -1,5 +1,5 @@
    -
    {{row.entity.metadata.create_time | date:"MM/dd/yyyy"}} +
    {{row.entity.metadata.create_time | amTimeAgo}}
    {{row.entity.metadata.create_time | amUtc | amDateFormat:'MM/DD/YYYY'}}
    diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/idcolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/idcolumn.html index 5b975a4c1b3..06b45b4051b 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/idcolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/idcolumn.html @@ -1,5 +1,4 @@ -
    -
    {{COL_FIELD CUSTOM_FILTERS}} -
    +
    + {{COL_FIELD CUSTOM_FILTERS}}
    diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html index 193d29028a5..7fc5bc2a9d1 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html @@ -1 +1,11 @@ -
    {{COL_FIELD | uppercase}}
    \ No newline at end of file + +
    + {{COL_FIELD | uppercase}} +
    \ No newline at end of file diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html index 654f2f50783..3125f48468c 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html @@ -1 +1,50 @@ -
    {{COL_FIELD | uppercase}}
    \ No newline at end of file + + \ No newline at end of file diff --git a/server/www/scripts/statusReport/partials/ui-grid/confirmbutton.html b/server/www/scripts/statusReport/partials/ui-grid/confirmbutton.html index f6f79e1fc24..d502ddb0b5a 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/confirmbutton.html +++ b/server/www/scripts/statusReport/partials/ui-grid/confirmbutton.html @@ -1,3 +1,5 @@
    - + + +
    \ No newline at end of file diff --git a/server/www/scripts/statusReport/styles/status.css b/server/www/scripts/statusReport/styles/status.css new file mode 100644 index 00000000000..66866b5e694 --- /dev/null +++ b/server/www/scripts/statusReport/styles/status.css @@ -0,0 +1,158 @@ +.status-report-grid .ui-grid-header{ + border-top: solid #E5E5E5 6px; + border-bottom: 2px solid #d8d8d8; + background: none; + color: #E9E9E9; + padding: 0px; + height: 45px; +} + +.status-report-grid .ui-grid-header .ui-grid-cell-contents { + color: #7c7c7c; + padding: 9px 5px 9px 5px !important; + text-align: center; +} + +.status-report-grid .ui-grid-cell-contents { + text-align: center; +} + +.status-report-grid .ui-grid-header-cell{ + border-right: solid #E9E9E9 1px; + text-align: center; +} + +.status-report-grid .ui-grid-header-cell i, .status-report-grid .grid_id{ + + padding: 9px 5px 9px 5px !important; +} + +.status-report-grid .ui-grid-row { + height: 45px; +} + +.status-report-grid .grid_id .ui-grid-cell-contents{ + position: relative; + top: 3px; +} + +.status-report-grid .ui-grid-row-selected .ui-grid-cell{ + background-color: #007bff !important; +} +.control-edit .edit-icon, .download-wrapper .download-icon { + height: 20px; + color: #000; + position: relative; + bottom: 2px; +} + +.status-report-grid .ui-grid-header-cell{ + text-align: center; +} + +.download-wrapper { + bottom: 5px; +} + +.status-report-grid .ui-grid-header-cell .glyphicon-remove{ + color: #ccc; + font-size: 8px; + font-weight: lighter; + top: 0px; + left: 4px; +} + +.search-wrapper .form-group{ + margin: 0px; +} + +.status-report-grid .ui-grid-contents-wrapper{ + clear: both; +} + +.status-report-grid .ui-grid-cell { + border: none ; +} + +.button-control { + display:flex; + flex-direction: row; + padding-right: 20px; + padding-left: 0px; + padding-bottom: 17px; + justify-content: space-between; + align-items: center; +} + +#reports-main { + padding: 40px 0 0 24px; +} + +#reports-main .reports{ + padding: 0px; +} + +.search-wrapper { + width: 385px; + padding: 0px; + margin-right: 16px; + margin-left: 16px; +} + +.search-wrapper input{ + height: 40px !important; + font-size: 14px; +} + +.search-wrapper .input-group input, .search-wrapper .input-group span { + border: none; + border-radius: 0px; + background-color: #FFFFFF +} + +.control-wrapper { + font-size: 14px; +} + +.space-wrapper{ + flex:1 +} + +.control-wrapper button{ + border-radius: 0px; +} + +.control-wrapper > button{ + border: none; + background: none; + width: 47px; + height: 33px; +} + +.control-edit { + padding-right: 0px; +} + +.control-edit .edit { + width: 30px; +} + +.control-edit .dropdown-toggle { + padding-right: 0px; + padding-left: 0px; + width: 10px; +} + +button.btn-new { + width:95px; + height:33px; + background-color: #5cb85c; +} + +.control-wrap-right { + justify-self: flex-end; +} + +.control-new{ + margin-right: 16px; +} \ No newline at end of file diff --git a/server/www/scripts/vulndb/partials/vulndb.html b/server/www/scripts/vulndb/partials/vulndb.html index 3821a4c4a0a..b58ef5dc8a7 100644 --- a/server/www/scripts/vulndb/partials/vulndb.html +++ b/server/www/scripts/vulndb/partials/vulndb.html @@ -4,10 +4,8 @@
    +
    -
    -

    Vulnerability Templates

    -
    From 7fee86e5b873087507b7cc2e8c8ca79ce2bb07b9 Mon Sep 17 00:00:00 2001 From: Nicolas Ronchi Cesarini Date: Tue, 22 May 2018 17:58:06 -0300 Subject: [PATCH 1173/1506] fixes to status report after first feedback --- server/www/estilos.css | 6 +++ server/www/header.css | 5 ++- .../statusReport/controllers/statusReport.js | 7 +-- .../statusReport/partials/statusReport.html | 19 ++++---- .../ui-grid/columns/statuscolumn.html | 16 +++---- .../partials/ui-grid/confirmbutton.html | 2 +- .../scripts/statusReport/styles/status.css | 45 +++++++++++++++---- 7 files changed, 66 insertions(+), 34 deletions(-) diff --git a/server/www/estilos.css b/server/www/estilos.css index e126ec7f255..5a006d6b869 100644 --- a/server/www/estilos.css +++ b/server/www/estilos.css @@ -112,6 +112,7 @@ header.head { display: block; padding: 8px 0; color: #fff; + opacity: .4; text-align: center; transition: all .3s ease; -ms-transition: all .3s ease; @@ -120,12 +121,17 @@ header.head { } aside nav a:hover { background-color: #d9d9d9; + opacity: 1; } aside nav a.active { background-color: #d9d9d9; } + .left-nav ul{ + margin-top: 10px; + } + .left-nav { width: 64px; z-index: 1; diff --git a/server/www/header.css b/server/www/header.css index caeadb75d05..2e1d14e8f3b 100644 --- a/server/www/header.css +++ b/server/www/header.css @@ -26,7 +26,7 @@ } .faraday-page-header .subtitle { - font-size: 14px; + font-size: 12px; font-weight: 300; color: #a3a3a3; float: left; @@ -90,7 +90,7 @@ } .controls-wrapper button { - border-left: solid 1px #a3a3a3; + border-left: solid 1px #dddddd; border-radius: 0px; width:55px; } @@ -117,6 +117,7 @@ .user-menu-wrapper { align-self: flex-end; + margin-right: 30px; } .flex-spacer { diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index e66f8a43a5a..2621c15edc1 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -218,7 +218,7 @@ angular.module("faradayApp") "website": "90", "path": "90", "request": "90", - "refs": "120", + "refs": "20", "_attachments": "100", "impact": "90", "method": "90", @@ -261,7 +261,7 @@ angular.module("faradayApp") }; var defineColumns = function() { - $scope.gridOptions.columnDefs.push({ name: "confirmVuln", width: "23", enableColumnResizing: false, headerCellTemplate: "", cellTemplate: "scripts/statusReport/partials/ui-grid/confirmbutton.html" }); + $scope.gridOptions.columnDefs.push({ name: "confirmVuln", width: "45", enableColumnResizing: false, headerCellTemplate: "", cellTemplate: "scripts/statusReport/partials/ui-grid/confirmbutton.html" }); function getColumnSort(columnName){ if($cookies.get("SRsortColumn") === columnName){ @@ -291,7 +291,6 @@ angular.module("faradayApp") displayName : "_id", cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/idcolumn.html', headerCellTemplate: header, - width: '50', sort: getColumnSort('_id'), visible: $scope.columns["_id"] }); @@ -300,7 +299,6 @@ angular.module("faradayApp") headerCellTemplate: header, displayName : "sev", type: 'string', - width: '70', visible: $scope.columns["severity"], sort: getColumnSort('severity'), sortingAlgorithm: compareSeverities, @@ -390,7 +388,6 @@ angular.module("faradayApp") displayName : "date", cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/datecolumn.html', headerCellTemplate: header, - width: '80', enableColumnResizing: false, sort: getColumnSort('date'), visible: $scope.columns["date"] diff --git a/server/www/scripts/statusReport/partials/statusReport.html b/server/www/scripts/statusReport/partials/statusReport.html index f3eae20e93e..6772a763820 100644 --- a/server/www/scripts/statusReport/partials/statusReport.html +++ b/server/www/scripts/statusReport/partials/statusReport.html @@ -12,11 +12,6 @@ New
    -
    - -
    +
    + +
    @@ -66,21 +66,20 @@ - - +
    -
    diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html index 3125f48468c..d72b5d28372 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html @@ -1,22 +1,22 @@
    -
    +
    {{COL_FIELD | uppercase}}
    -
    +
    {{COL_FIELD | uppercase}}
    -
    +
    {{COL_FIELD | uppercase}}
    -
    +
    {{COL_FIELD | uppercase}} @@ -26,22 +26,22 @@
    -
    +
    {{COL_FIELD | uppercase}}
    -
    +
    {{COL_FIELD | uppercase}}
    -
    +
    {{COL_FIELD | uppercase}}
    -
    +
    {{COL_FIELD | uppercase}} diff --git a/server/www/scripts/statusReport/partials/ui-grid/confirmbutton.html b/server/www/scripts/statusReport/partials/ui-grid/confirmbutton.html index d502ddb0b5a..bf5807f9e4c 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/confirmbutton.html +++ b/server/www/scripts/statusReport/partials/ui-grid/confirmbutton.html @@ -1,5 +1,5 @@
    - +
    \ No newline at end of file diff --git a/server/www/scripts/statusReport/styles/status.css b/server/www/scripts/statusReport/styles/status.css index 66866b5e694..301a3fca4e7 100644 --- a/server/www/scripts/statusReport/styles/status.css +++ b/server/www/scripts/statusReport/styles/status.css @@ -9,8 +9,8 @@ .status-report-grid .ui-grid-header .ui-grid-cell-contents { color: #7c7c7c; - padding: 9px 5px 9px 5px !important; - text-align: center; + padding: 9px 9px 9px 5px !important; + text-align: left; } .status-report-grid .ui-grid-cell-contents { @@ -23,8 +23,7 @@ } .status-report-grid .ui-grid-header-cell i, .status-report-grid .grid_id{ - - padding: 9px 5px 9px 5px !important; + padding: 11px 9px 9px 5px !important } .status-report-grid .ui-grid-row { @@ -37,15 +36,33 @@ } .status-report-grid .ui-grid-row-selected .ui-grid-cell{ - background-color: #007bff !important; + background-color: #8DA2B7 !important; + color: #FFFFFF; +} + +.status-report-grid .ui-grid-row-selected a{ + color: #FFFFFF; +} + +.status-report-grid .no-background{ + background: none; +} + +.control-edit { + width:44px; } -.control-edit .edit-icon, .download-wrapper .download-icon { + +.control-edit .edit-icon, .download-wrapper .download-icon, .control-wrapper .delete-icon, .search-wrapper .search-icon { height: 20px; color: #000; position: relative; bottom: 2px; } +.status-report-grid .confirm-icon { + width: 40%; +} + .status-report-grid .ui-grid-header-cell{ text-align: center; } @@ -85,7 +102,7 @@ } #reports-main { - padding: 40px 0 0 24px; + padding: 0px 0 0 24px; } #reports-main .reports{ @@ -102,6 +119,7 @@ .search-wrapper input{ height: 40px !important; font-size: 14px; + box-shadow: none; } .search-wrapper .input-group input, .search-wrapper .input-group span { @@ -114,8 +132,13 @@ font-size: 14px; } +.filter-wrapper .confirm-button{ + position: relative; + bottom: 1px; +} + .space-wrapper{ - flex:1 + flex:1; } .control-wrapper button{ @@ -131,6 +154,12 @@ .control-edit { padding-right: 0px; + position: relative; +} + +.control-edit .dropdown-menu-right{ + left: 0px !important; + width: 245px; } .control-edit .edit { From 15009965cf78f300315bfa668df4dca88cd89538 Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Thu, 24 May 2018 12:39:30 -0300 Subject: [PATCH 1174/1506] 4721 - Adding vulnerabilities by status donut visualization --- server/www/dashboard-v3.css | 11 ++++ server/www/index.html | 1 + .../www/scripts/commons/providers/server.js | 7 +++ .../controllers/vulnsByStatusCtrl.js | 34 ++++++++++ .../dashboard/partials/activityFeed.html | 63 +++++++++---------- .../scripts/dashboard/partials/dashboard.html | 7 +-- .../dashboard/partials/last-vulns.html | 13 ++-- .../dashboard/partials/vulnsByStatus.html | 18 ++++++ .../partials/workspace-progress.html | 6 +- .../scripts/dashboard/providers/dashboard.js | 17 ++++- 10 files changed, 129 insertions(+), 48 deletions(-) create mode 100644 server/www/scripts/dashboard/controllers/vulnsByStatusCtrl.js create mode 100644 server/www/scripts/dashboard/partials/vulnsByStatus.html diff --git a/server/www/dashboard-v3.css b/server/www/dashboard-v3.css index c3e4392ddfd..687a4115589 100644 --- a/server/www/dashboard-v3.css +++ b/server/www/dashboard-v3.css @@ -52,6 +52,11 @@ padding-right: 10px; } +.ph-xl { + padding-left: 30px; + padding-right: 30px; +} + .ph-xxl { padding-left: 45px; padding-right: 45px; @@ -103,3 +108,9 @@ .fg-light-gray { color: #a1a1a1; } + +.seccion.dashboard article header h2{ + margin: 0px; + padding-left: 20px; + padding-right: 20px; +} diff --git a/server/www/index.html b/server/www/index.html index 6e9f780eda7..dc2fc2f898d 100644 --- a/server/www/index.html +++ b/server/www/index.html @@ -151,6 +151,7 @@ + diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index 2801b1858b1..144a4abf0c2 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -344,6 +344,13 @@ angular.module("faradayApp") return get(url, payload) } + ServerAPI.getVulnsGroupedBy = function(wsName, groupBy) { + var url = createGetUrl(wsName, 'vulns') + 'count/'; + var payload = {'group_by': groupBy} + + return get(url, payload) + } + ServerAPI.createHost = function(wsName, host) { return modHost(createObject, wsName, host); } diff --git a/server/www/scripts/dashboard/controllers/vulnsByStatusCtrl.js b/server/www/scripts/dashboard/controllers/vulnsByStatusCtrl.js new file mode 100644 index 00000000000..72c95e3e540 --- /dev/null +++ b/server/www/scripts/dashboard/controllers/vulnsByStatusCtrl.js @@ -0,0 +1,34 @@ +// Faraday Penetration Test IDE +// Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +// See the file 'doc/LICENSE' for the license information + +angular.module('faradayApp') + .controller('vulnsByStatusCtrl', + ['$scope', '$routeParams', 'dashboardSrv', + function($scope, $routeParams, dashboardSrv) { + + $scope.workspace = null; + + init = function() { + if($routeParams.wsId != undefined) { + $scope.workspace = $routeParams.wsId; + + dashboardSrv.getVulnerabilitiesGroupedBy($scope.workspace, 'status') + .then(function(vulnsByStatus) { + $scope.data = {key: [], value: [], colors: [], options: {maintainAspectRatio: false, animateRotate: true}}; + $scope.loaded = true; + vulnsByStatus.forEach(function(vuln, index) { + $scope.data.value.push(vuln.count); + $scope.data.key.push(vuln.status); + $scope.data.colors.push(dashboardSrv.vulnColors[index]); + }); + + $scope.loaded = true; + }); + } + }; + + dashboardSrv.registerCallback(init); + + init(); + }]); diff --git a/server/www/scripts/dashboard/partials/activityFeed.html b/server/www/scripts/dashboard/partials/activityFeed.html index 94196379865..6168d8c9ec1 100644 --- a/server/www/scripts/dashboard/partials/activityFeed.html +++ b/server/www/scripts/dashboard/partials/activityFeed.html @@ -1,11 +1,10 @@
    - -
    -

    Activity Feed - -

    -
    - +
    +

    Activity Feed + +

    +
    +

    No activities found yet.

    - -
    SeverityTargetNameWebDateSeverityTargetNameWebDate
    - {{cmd.user || Anonymous}} - ran {{cmd.command}} - imported {{cmd.tool == 'unknown' ? cmd.command : cmd.tool}} report + {{cmd.user || Anonymous}} + ran {{cmd.command}} + imported {{cmd.tool == 'unknown' ? cmd.command : cmd.tool}} report and found nothing : @@ -33,7 +33,7 @@

    Activity Feed & {{cmd.vulnerabilities_count}} {{cmd.vulnerabilities_count == 1 ? 'vulnerability' : 'vulnerabilities'}} - {{cmd.criticalIssue}} {{cmd.criticalIssue == 1 ? 'is' : 'are'}} rated as Critical. - +

    - - - - - -
    - - - {{cmd.user || Anonymous}} - ran {{cmd.command}} - imported {{cmd.tool == 'unknown' ? cmd.command : cmd.tool}} report - and found - nothing - : - {{cmd.hosts_count}} {{cmd.hosts_count == 1 ? 'host' : 'hosts'}} - , - & - {{cmd.services_count}} {{cmd.services_count == 1 ? 'service' : 'services'}} - & - {{cmd.vulnerabilities_count}} {{cmd.vulnerabilities_count == 1 ? 'vulnerability' : 'vulnerabilities'}} - - {{cmd.criticalIssue}} {{cmd.criticalIssue == 1 ? 'is' : 'are'}} rated as Critical. - -
    + + + + + + +
    + + + {{cmd.user || Anonymous}} + ran {{cmd.command}} + imported {{cmd.tool == 'unknown' ? cmd.command : cmd.tool}} report + and found + nothing + : + {{cmd.hosts_count}} {{cmd.hosts_count == 1 ? 'host' : 'hosts'}} + , + & + {{cmd.services_count}} {{cmd.services_count == 1 ? 'service' : 'services'}} + & + {{cmd.vulnerabilities_count}} {{cmd.vulnerabilities_count == 1 ? 'vulnerability' : 'vulnerabilities'}} + - {{cmd.criticalIssue}} {{cmd.criticalIssue == 1 ? 'is' : 'are'}} rated as Critical. + +
    +
    diff --git a/server/www/scripts/dashboard/partials/dashboard.html b/server/www/scripts/dashboard/partials/dashboard.html index 8ba9a14c848..eb245935ad4 100644 --- a/server/www/scripts/dashboard/partials/dashboard.html +++ b/server/www/scripts/dashboard/partials/dashboard.html @@ -16,7 +16,7 @@
    -
    +
    @@ -45,11 +45,6 @@
    -
    -
    -
    -
    -
    diff --git a/server/www/scripts/dashboard/partials/last-vulns.html b/server/www/scripts/dashboard/partials/last-vulns.html index 1d2508b37a3..dc3586f4aed 100644 --- a/server/www/scripts/dashboard/partials/last-vulns.html +++ b/server/www/scripts/dashboard/partials/last-vulns.html @@ -1,9 +1,11 @@
    -
    -

    Last Vulnerabilities - -

    -
    +
    +

    Last Vulnerabilities + +

    +
    + +
    diff --git a/server/www/scripts/dashboard/partials/vulnsByStatus.html b/server/www/scripts/dashboard/partials/vulnsByStatus.html new file mode 100644 index 00000000000..f86e97072c3 --- /dev/null +++ b/server/www/scripts/dashboard/partials/vulnsByStatus.html @@ -0,0 +1,18 @@ + + + + +
    + +
    diff --git a/server/www/scripts/dashboard/partials/workspace-progress.html b/server/www/scripts/dashboard/partials/workspace-progress.html index 9301d513930..f8de6b7b084 100644 --- a/server/www/scripts/dashboard/partials/workspace-progress.html +++ b/server/www/scripts/dashboard/partials/workspace-progress.html @@ -5,11 +5,11 @@

    -
    -
    +
    +
    {{wsProgress}}%
    -
    +
    Start date: {{wsStart | amUtc | amDateFormat:'MM/DD/YYYY'}}
    diff --git a/server/www/scripts/dashboard/providers/dashboard.js b/server/www/scripts/dashboard/providers/dashboard.js index 228498b9deb..ff1e6020a3f 100644 --- a/server/www/scripts/dashboard/providers/dashboard.js +++ b/server/www/scripts/dashboard/providers/dashboard.js @@ -208,9 +208,22 @@ angular.module('faradayApp') return deferred.promise; }; + dashboardSrv.getVulnerabilitiesGroupedBy = function(ws, groupBy) { + var deferred = $q.defer(); + + ServerAPI.getVulnsGroupedBy(ws, groupBy) + .then(function(res) { + deferred.resolve(res.data.groups); + }, function() { + deferred.reject("Unable to get Vulnerabilities"); + }); + + return deferred.promise; + }; + dashboardSrv.getObjectsCount = function(ws) { var deferred = $q.defer(); - // Confirmed empty = All vulns + // Confirmed empty = All vulns var confirmed = undefined; if (dashboardSrv.props['confirmed']) { @@ -290,7 +303,7 @@ angular.module('faradayApp') dashboardSrv.getHostsCountByCommandId = function(ws, command_id) { var deferred = $q.defer(); - + ServerAPI.getHosts(ws, {"command_id": command_id }) .then(function(res) { deferred.resolve(res.data); From 1159fabb3b4ac5c140d8127a1e441b520f964f48 Mon Sep 17 00:00:00 2001 From: Nicolas Ronchi Cesarini Date: Thu, 24 May 2018 16:53:22 -0300 Subject: [PATCH 1175/1506] fixes --- .../statusReport/controllers/statusReport.js | 25 +------------------ .../partials/ui-grid/confirmbutton.html | 4 +-- .../scripts/statusReport/styles/status.css | 11 ++++++-- 3 files changed, 12 insertions(+), 28 deletions(-) diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 2621c15edc1..95009806ae3 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -302,19 +302,16 @@ angular.module("faradayApp") visible: $scope.columns["severity"], sort: getColumnSort('severity'), sortingAlgorithm: compareSeverities, - enableColumnResizing: false }); $scope.gridOptions.columnDefs.push({ name : 'name', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/namecolumn.html', headerCellTemplate: header, - width: $scope.columnsWidths['name'], sort: getColumnSort('name'), visible: $scope.columns["name"] }); $scope.gridOptions.columnDefs.push({ name : 'service', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/servicecolumn.html', headerCellTemplate: header, - width: $scope.columnsWidths['service'], visible: $scope.columns["service"], sort: getColumnSort('service'), }); @@ -324,21 +321,18 @@ angular.module("faradayApp") minWidth: '100', maxWidth: '200', enableSorting: false, - width: $scope.columnsWidths['hostname'], sort: getColumnSort('hostnames'), visible: $scope.columns["hostnames"] }); $scope.gridOptions.columnDefs.push({ name : 'target', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/targetcolumn.html', headerCellTemplate: header, - width: $scope.columnsWidths['target'], sort: getColumnSort('target'), visible: $scope.columns["target"] }); $scope.gridOptions.columnDefs.push({ name : 'desc', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/desccolumn.html', headerCellTemplate: header, - width: $scope.columnsWidths['desc'], sort: getColumnSort('desc'), visible: $scope.columns["desc"] }); @@ -347,55 +341,48 @@ angular.module("faradayApp") headerCellTemplate: header, sort: getColumnSort('resolution'), visible: $scope.columns["resolution"], - width: $scope.columnsWidths['resolution'], }); $scope.gridOptions.columnDefs.push({ name : 'data', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/resolutioncolumn.html', headerCellTemplate: header, sort: getColumnSort('data'), visible: $scope.columns["data"], - width: $scope.columnsWidths['data'], }); $scope.gridOptions.columnDefs.push({ name : 'easeofresolution', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/defaultcolumn.html', headerCellTemplate: header, + minWidth: '150', sort: getColumnSort('easeofresolution'), visible: $scope.columns["easeofresolution"], - width: $scope.columnsWidths['easeofresolution'], }); $scope.gridOptions.columnDefs.push({ name : 'website', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/defaultcolumn.html', headerCellTemplate: header, sort: getColumnSort('website'), visible: $scope.columns["website"], - width: $scope.columnsWidths['website'], }); $scope.gridOptions.columnDefs.push({ name : 'path', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/defaultcolumn.html', headerCellTemplate: header, sort: getColumnSort('path'), visible: $scope.columns["path"], - width: $scope.columnsWidths['path'], }); $scope.gridOptions.columnDefs.push({ name : 'request', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/resolutioncolumn.html', headerCellTemplate: header, sort: getColumnSort('request'), visible: $scope.columns["request"], - width: $scope.columnsWidths['request'], }); $scope.gridOptions.columnDefs.push({ name : 'date', displayName : "date", cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/datecolumn.html', headerCellTemplate: header, - enableColumnResizing: false, sort: getColumnSort('date'), visible: $scope.columns["date"] }); $scope.gridOptions.columnDefs.push({ name : 'status', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/statuscolumn.html', headerCellTemplate: header, - width: $scope.columnsWidths['status'], sort: getColumnSort('status'), visible: $scope.columns["status"] }); @@ -405,7 +392,6 @@ angular.module("faradayApp") sort: getColumnSort('refs'), visible: $scope.columns["refs"], enableSorting: false, - width: $scope.columnsWidths['refs'], }); $scope.gridOptions.columnDefs.push({ name : '_attachments', displayName: "evidence", @@ -413,7 +399,6 @@ angular.module("faradayApp") headerCellTemplate: header, sort: getColumnSort('_attachments'), visible: $scope.columns["evidence"], - width: $scope.columnsWidths['_attachments'], }); $scope.gridOptions.columnDefs.push({ name : 'impact', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/impactcolumn.html', @@ -421,35 +406,30 @@ angular.module("faradayApp") sort: getColumnSort('impact'), enableSorting: false, visible: $scope.columns["impact"], - width: $scope.columnsWidths['impact'], }); $scope.gridOptions.columnDefs.push({ name : 'method', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/defaultcolumn.html', headerCellTemplate: header, sort: getColumnSort('method'), visible: $scope.columns["method"], - width: $scope.columnsWidths['method'], }); $scope.gridOptions.columnDefs.push({ name : 'params', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/defaultcolumn.html', headerCellTemplate: header, sort: getColumnSort('params'), visible: $scope.columns["params"], - width: $scope.columnsWidths['params'], }); $scope.gridOptions.columnDefs.push({ name : 'pname', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/defaultcolumn.html', headerCellTemplate: header, sort: getColumnSort('pname'), visible: $scope.columns["pname"], - width: $scope.columnsWidths['pname'], }); $scope.gridOptions.columnDefs.push({ name : 'query', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/defaultcolumn.html', headerCellTemplate: header, sort: getColumnSort('query'), visible: $scope.columns["query"], - width: $scope.columnsWidths['query'], }); $scope.gridOptions.columnDefs.push({ name : 'response', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/resolutioncolumn.html', @@ -461,7 +441,6 @@ angular.module("faradayApp") $scope.gridOptions.columnDefs.push({ name : 'web', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/webcolumn.html', headerCellTemplate: header, - width: $scope.columnsWidths['web'], sort: getColumnSort('web'), visible: $scope.columns["web"] }); @@ -469,7 +448,6 @@ angular.module("faradayApp") displayName : "creator", cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/creatorcolumn.html', headerCellTemplate: header, - width: $scope.columnsWidths['metadata.creator'], sort: getColumnSort('metadata.creator'), visible: $scope.columns["creator"] }); @@ -481,7 +459,6 @@ angular.module("faradayApp") // displayName : "policy violations", cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/policyviolationscolumn.html', headerCellTemplate: header, - width: $scope.columnsWidths['policyviolations'], sort: getColumnSort('policyviolations'), visible: $scope.columns["policyviolations"], enableSorting: false, diff --git a/server/www/scripts/statusReport/partials/ui-grid/confirmbutton.html b/server/www/scripts/statusReport/partials/ui-grid/confirmbutton.html index bf5807f9e4c..b65ef0f538b 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/confirmbutton.html +++ b/server/www/scripts/statusReport/partials/ui-grid/confirmbutton.html @@ -1,5 +1,5 @@
    - - + +
    \ No newline at end of file diff --git a/server/www/scripts/statusReport/styles/status.css b/server/www/scripts/statusReport/styles/status.css index 301a3fca4e7..0c48d179537 100644 --- a/server/www/scripts/statusReport/styles/status.css +++ b/server/www/scripts/statusReport/styles/status.css @@ -9,7 +9,7 @@ .status-report-grid .ui-grid-header .ui-grid-cell-contents { color: #7c7c7c; - padding: 9px 9px 9px 5px !important; + padding: 9px 3px 9px 5px !important; text-align: left; } @@ -132,6 +132,8 @@ font-size: 14px; } +.control-wrapper button:focus {outline:0;} + .filter-wrapper .confirm-button{ position: relative; bottom: 1px; @@ -182,6 +184,11 @@ button.btn-new { justify-self: flex-end; } +.control-wrap-right button{ + font-size: 12px; +} + .control-new{ margin-right: 16px; -} \ No newline at end of file +} + From e366533a4ee5cded083c1e5105dc28adbfad4f01 Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Fri, 25 May 2018 10:09:09 -0300 Subject: [PATCH 1176/1506] 4725 - Fix login input red box shadow in Firefox --- server/www/styles/material-input.css | 1 + 1 file changed, 1 insertion(+) diff --git a/server/www/styles/material-input.css b/server/www/styles/material-input.css index 80b6e78d8f1..d5079486f14 100644 --- a/server/www/styles/material-input.css +++ b/server/www/styles/material-input.css @@ -35,6 +35,7 @@ padding: 0; margin: 0; color: #a1a1a1; + box-shadow: none; /* Firefox defaults to red so we clean it */ } .form-input textarea { min-height: 30px; From 454622c0f8a89732796b3d32731d2167088b51a2 Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Fri, 25 May 2018 11:08:51 -0300 Subject: [PATCH 1177/1506] 4725 - Fix left sidebar displaying login background on welcome screen --- server/www/estilos-v3.css | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/www/estilos-v3.css b/server/www/estilos-v3.css index e7c152cac84..04aefb252b3 100644 --- a/server/www/estilos-v3.css +++ b/server/www/estilos-v3.css @@ -1,10 +1,14 @@ -body { +#login-main { background-image: url(images/fondofaraday4k.jpg) !important; + width: 100%; + height: 100%; + display: inline-block; + position: absolute; } #login-container { width: 410px; - margin-top: 180px; + margin-top: 160px; background-color: #FFF; margin-left: auto; margin-right: auto; From 01ff6a065505a1ac14936bacd3100373184a3d9c Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Fri, 25 May 2018 12:19:31 -0300 Subject: [PATCH 1178/1506] 4721 - Update vulnerabilities by status donut colors and add top 5 hosts --- server/www/dashboard-v3.css | 6 +++++- .../dashboard/controllers/vulnsByStatusCtrl.js | 13 ++++++++++--- server/www/scripts/dashboard/partials/compound.html | 6 +++--- .../www/scripts/dashboard/partials/dashboard.html | 7 ++++++- server/www/scripts/dashboard/partials/services.html | 2 +- .../www/scripts/dashboard/partials/summarized.html | 2 +- .../{vulnsByStatus.html => vulns-by-status.html} | 5 ++--- .../dashboard/partials/workspace-progress.html | 2 +- 8 files changed, 29 insertions(+), 14 deletions(-) rename server/www/scripts/dashboard/partials/{vulnsByStatus.html => vulns-by-status.html} (71%) diff --git a/server/www/dashboard-v3.css b/server/www/dashboard-v3.css index 687a4115589..f64bece4b45 100644 --- a/server/www/dashboard-v3.css +++ b/server/www/dashboard-v3.css @@ -6,6 +6,10 @@ border-right: 1px solid #e7e7e7; } +.small-widget-height { + height: 195px; +} + .dashboard-label-wrapper { text-align: left; margin-top: 10px; @@ -54,7 +58,7 @@ .ph-xl { padding-left: 30px; - padding-right: 30px; + padding-right: 30px; } .ph-xxl { diff --git a/server/www/scripts/dashboard/controllers/vulnsByStatusCtrl.js b/server/www/scripts/dashboard/controllers/vulnsByStatusCtrl.js index 72c95e3e540..01c4f126c24 100644 --- a/server/www/scripts/dashboard/controllers/vulnsByStatusCtrl.js +++ b/server/www/scripts/dashboard/controllers/vulnsByStatusCtrl.js @@ -17,10 +17,19 @@ angular.module('faradayApp') .then(function(vulnsByStatus) { $scope.data = {key: [], value: [], colors: [], options: {maintainAspectRatio: false, animateRotate: true}}; $scope.loaded = true; + + vulnerabilityColors = { + 'open': '#DF3936', + 'close': '#A1CE31', + 're-open': '#DFBF35', + 'risk-accept': '#2e97bd' + }; + vulnsByStatus.forEach(function(vuln, index) { $scope.data.value.push(vuln.count); $scope.data.key.push(vuln.status); - $scope.data.colors.push(dashboardSrv.vulnColors[index]); + + $scope.data.colors.push(vulnerabilityColors[vuln.status]); }); $scope.loaded = true; @@ -28,7 +37,5 @@ angular.module('faradayApp') } }; - dashboardSrv.registerCallback(init); - init(); }]); diff --git a/server/www/scripts/dashboard/partials/compound.html b/server/www/scripts/dashboard/partials/compound.html index 5664fb0cd90..b950f990b01 100644 --- a/server/www/scripts/dashboard/partials/compound.html +++ b/server/www/scripts/dashboard/partials/compound.html @@ -20,7 +20,7 @@

    Hosts - + {{host.name}} @@ -33,7 +33,7 @@

    Hosts -
    + diff --git a/server/www/scripts/dashboard/partials/dashboard.html b/server/www/scripts/dashboard/partials/dashboard.html index eb245935ad4..bbcdb7605ac 100644 --- a/server/www/scripts/dashboard/partials/dashboard.html +++ b/server/www/scripts/dashboard/partials/dashboard.html @@ -16,7 +16,7 @@
    -
    +

    @@ -45,6 +45,11 @@
    +
    +
    +
    +
    +
    diff --git a/server/www/scripts/dashboard/partials/services.html b/server/www/scripts/dashboard/partials/services.html index 11543f208c7..269b16e6559 100644 --- a/server/www/scripts/dashboard/partials/services.html +++ b/server/www/scripts/dashboard/partials/services.html @@ -3,7 +3,7 @@
    -
    +

    Services report diff --git a/server/www/scripts/dashboard/partials/summarized.html b/server/www/scripts/dashboard/partials/summarized.html index 61f0fc5b1e6..cfc9e5cd751 100644 --- a/server/www/scripts/dashboard/partials/summarized.html +++ b/server/www/scripts/dashboard/partials/summarized.html @@ -4,7 +4,7 @@
    -
    +

    Workspace summarized report diff --git a/server/www/scripts/dashboard/partials/vulnsByStatus.html b/server/www/scripts/dashboard/partials/vulns-by-status.html similarity index 71% rename from server/www/scripts/dashboard/partials/vulnsByStatus.html rename to server/www/scripts/dashboard/partials/vulns-by-status.html index f86e97072c3..8f66396ba82 100644 --- a/server/www/scripts/dashboard/partials/vulnsByStatus.html +++ b/server/www/scripts/dashboard/partials/vulns-by-status.html @@ -3,11 +3,10 @@
    -
    +

    - Vulnerabilities by status - + Vulnerabilities by status

    diff --git a/server/www/scripts/dashboard/partials/workspace-progress.html b/server/www/scripts/dashboard/partials/workspace-progress.html index f8de6b7b084..6bbc9501544 100644 --- a/server/www/scripts/dashboard/partials/workspace-progress.html +++ b/server/www/scripts/dashboard/partials/workspace-progress.html @@ -1,4 +1,4 @@ -
    +

    Workspace progress From cd3e4b9cc5381d4924e4f9d490f3774ca7c670f3 Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Mon, 28 May 2018 08:53:41 -0300 Subject: [PATCH 1179/1506] 4721 - Update dashboard hosts widget table margins --- .../scripts/dashboard/partials/compound.html | 48 ++++++++++--------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/server/www/scripts/dashboard/partials/compound.html b/server/www/scripts/dashboard/partials/compound.html index b950f990b01..d97e0c7f147 100644 --- a/server/www/scripts/dashboard/partials/compound.html +++ b/server/www/scripts/dashboard/partials/compound.html @@ -1,4 +1,4 @@ -
    +

    Hosts @@ -11,28 +11,30 @@

    Hosts

    No hosts found yet.

    - - - - - - - - - - - - - - - -
    HostServicesOS
    - {{host.name}} - - {{host.services}} - - -
    +
    + + + + + + + + + + + + + + + +
    HostServicesOS
    + {{host.name}} + + {{host.services}} + + +
    +
    diff --git a/server/www/scripts/dashboard/partials/dashboard.html b/server/www/scripts/dashboard/partials/dashboard.html index bbcdb7605ac..c6225102343 100644 --- a/server/www/scripts/dashboard/partials/dashboard.html +++ b/server/www/scripts/dashboard/partials/dashboard.html @@ -6,7 +6,7 @@
    -
    +
    @@ -26,7 +26,7 @@
    -
    +
    diff --git a/server/www/scripts/dashboard/partials/vulns-by-severity.html b/server/www/scripts/dashboard/partials/vulns-by-severity.html index cc4681f0dba..4c164501e0b 100644 --- a/server/www/scripts/dashboard/partials/vulns-by-severity.html +++ b/server/www/scripts/dashboard/partials/vulns-by-severity.html @@ -13,7 +13,7 @@

    No vulnerabilities found yet.

    -
    +
    {{count}}
    diff --git a/server/www/scripts/dashboard/providers/dashboard.js b/server/www/scripts/dashboard/providers/dashboard.js index ff1e6020a3f..24615e15a59 100644 --- a/server/www/scripts/dashboard/providers/dashboard.js +++ b/server/www/scripts/dashboard/providers/dashboard.js @@ -45,10 +45,10 @@ angular.module('faradayApp') dashboardSrv.vulnColors = [ "#932EBE", // critical - "#DF3936", // high - "#DFBF35", // med - "#A1CE31", // low - "#428BCA", // info + "#e77273", // high + "#e7d174", // med + "#bddd72", // low + "#7aabd9", // info "#999999" // unclassified ]; From 1cb88a9b8481f547b1487d980f3b79d100daea3d Mon Sep 17 00:00:00 2001 From: montive Date: Tue, 29 May 2018 18:58:38 -0300 Subject: [PATCH 1198/1506] Arreglado bug de Postgresql --- server/commands/status_check.py | 118 ++++++++++++++++++-------------- 1 file changed, 65 insertions(+), 53 deletions(-) diff --git a/server/commands/status_check.py b/server/commands/status_check.py index 825e50d7478..c516b261c9a 100644 --- a/server/commands/status_check.py +++ b/server/commands/status_check.py @@ -18,34 +18,31 @@ init() -def check_server_running(): +def check_postgres(): + tmp = os.popen("ps -Af").read() + if "postgresql" in tmp: + return True + else: + return False - pid = is_server_running() - return pid - -def check_open_ports(): - address = server.config.faraday_server.bind_address - port = int(server.config.faraday_server.port) - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex((address,port)) - if result == 0: - return True - else: - return False - +def check_active_user(): + with app.app_context(): + try: + active_users = db.engine.execute('SELECT active FROM faraday_user') + for item in active_users: + if item[0] == True: + return True + except sqlalchemy.exc.OperationalError: + return 0 -def check_postgres(): - with app.app_context(): - try: - result = str(db.engine.execute("SELECT version()")) - return result - except sqlalchemy.exc.OperationalError: - return None - -def check_client(): +def check_server_running(): + pid = is_server_running() + return pid + +def check_client_running(): port_rest = CONF.getApiRestfulConInfoPort() try: @@ -58,7 +55,6 @@ def check_client(): def check_server_dependencies(): - installed_deps, missing_deps, conflict_deps = dependencies.check_dependencies( requirements_file='requirements_server.txt') @@ -73,7 +69,6 @@ def check_server_dependencies(): def check_client_dependencies(): - installed_deps, missing_deps, conflict_deps = dependencies.check_dependencies( requirements_file='requirements.txt') @@ -90,10 +85,7 @@ def check_client_dependencies(): return None, None - - def check_credentials(): - api_username = CONF.getAPIUsername() api_password = CONF.getAPIPassword() @@ -115,7 +107,6 @@ def check_credentials(): def check_storage_permission(): - home = os.path.expanduser("~") path = home+'/.faraday/storage/test' @@ -127,30 +118,42 @@ def check_storage_permission(): return None +def check_open_ports(): + address = server.config.faraday_server.bind_address + port = int(server.config.faraday_server.port) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex((address,port)) + if result == 0: + return True + else: + return False + + def full_status_check(): - #Prints the status of PostreSQL using check_postgres() - print('\n{white}Checking if postgreSQL is running...'.format(white=Fore.WHITE)) + #Checking PostgreSQL + print('\n{white}Checking if PostgreSQL is running...'.format(white=Fore.WHITE)) result = check_postgres() if result: print('[{green}+{white}] PostgreSQL is running'.\ format(green=Fore.GREEN, white=Fore.WHITE)) else: - print('[{red}-{white}] Could not connect to postgresql, please check if database is running'\ + print('[{red}-{white}] Could not connect to PostgreSQL, please check if database is running'\ .format(red=Fore.RED, white=Fore.WHITE)) return - print('\n{white}Checking if Faraday is running...'.format(white=Fore.WHITE)) - if check_client(): - print('[{green}+{white}] Faraday GTK is running'.\ - format(green=Fore.GREEN, white=Fore.WHITE)) + if check_active_user(): + print("[{green}+{white}] Active user exists".format(green=Fore.GREEN, white=Fore.WHITE)) + elif check_active_user() == 0: + print("[{red}-{white}] Faraday database non existant".format(red=Fore.RED, white=Fore.WHITE)) else: - print('[{yellow}-{white}] Faraday GTK is not running'\ - .format(yellow=Fore.YELLOW, white=Fore.WHITE)) + print("[{red}-{white}] Active user doesn't exists".format(red=Fore.RED, white=Fore.WHITE)) - #Prints Status of the server using check_server_running() + + #Checking status + print('\n{white}Checking if Faraday is running...'.format(white=Fore.WHITE)) pid = check_server_running() if pid is not None: print('[{green}+{white}] Faraday Server is Running. PID:{PID} \ @@ -159,47 +162,56 @@ def full_status_check(): print('[{red}-{white}] Faraday Server is not running {white} \ '.format(red=Fore.RED, white=Fore.WHITE)) + if check_client_running(): + print('[{green}+{white}] Faraday GTK is running'.\ + format(green=Fore.GREEN, white=Fore.WHITE)) + else: + print('[{yellow}-{white}] Faraday GTK is not running'\ + .format(yellow=Fore.YELLOW, white=Fore.WHITE)) + + #Checking dependencies print('\n{white}Checking Faraday dependencies...'.format(white=Fore.WHITE)) - status, server_dep = check_server_dependencies() if status == True: - print('[{red}-{white}] Some server dependencies are old. Update them with \"pip install -r requirements_server.txt -U\": (' + ','.join(server_dep) + ')') \ + print('[{red}-{white}] Some server\'s dependencies are old. Update them with \"pip install -r requirements_server.txt -U\": (' + ','.join(server_dep) + ')') \ .format(red=Fore.RED, white=Fore.WHITE) elif status == 0: - print('[{red}-{white}] Client dependencies not met. Install them with \"pip install -r requirements_server.txt -U\": (' + ','.join(server_dep) + ')')\ + print('[{red}-{white}] Server\'s dependencies not met. Install them with \"pip install -r requirements_server.txt -U\": (' + ','.join(server_dep) + ')')\ .format(red=Fore.RED, white=Fore.WHITE) else: - print('[{green}+{white}] Server dependencies met' \ + print('[{green}+{white}] Server\'s dependencies met' \ .format(green=Fore.GREEN, white=Fore.WHITE)) status, client_dep = check_client_dependencies() if status == True: - print('[{red}-{white}] Some client dependencies are old. Update them with \"pip install -r requirements.txt -U\": (' + ','.join(client_dep) + ')') \ + print('[{red}-{white}] Some client\'s dependencies are old. Update them with \"pip install -r requirements.txt -U\": (' + ','.join(client_dep) + ')') \ .format(red=Fore.RED, white=Fore.WHITE) elif status == 0: - print('[{red}-{white}] Client dependencies not met. Install them with \"pip install -r requirements.txt -U\": (' + ','.join(client_dep) + ')')\ + print('[{red}-{white}] Client\'s dependencies not met. Install them with \"pip install -r requirements.txt -U\": (' + ','.join(client_dep) + ')')\ .format(red=Fore.RED, white=Fore.WHITE) else: - print('[{green}+{white}] Client dependencies met'\ + print('[{green}+{white}] Client\'s dependencies met'\ .format(green=Fore.GREEN, white=Fore.WHITE)) - + + #Checking config print('\n{white}Checking Faraday config...{white}'.format(white=Fore.WHITE)) if pid and result: status_code = check_credentials() if status_code == 200: - print('[{green}+{white}] Credentials matched'.format(green=Fore.GREEN, white=Fore.WHITE)) + print('[{green}+{white}] User\'s credentials matched'.format(green=Fore.GREEN, white=Fore.WHITE)) elif status_code == 400: - print('[{red}-{white}] Error. Credentials does not match' \ + print('[{red}-{white}] Error. User\'s credentials do not match' \ .format(red=Fore.RED, white=Fore.WHITE)) else: - print('[{red}-{white}] Either Faraday Server not running or database not working'.format(red=Fore.RED, white=Fore.WHITE)) + print('[{red}-{white}] Failed checking user\'s credentials. Either Faraday Server is not running or database is not working'\ + .format(red=Fore.RED, white=Fore.WHITE)) if check_storage_permission(): print('[{green}+{white}] ~/.faraday/storage -> Permission accepted' \ @@ -209,8 +221,8 @@ def full_status_check(): .format(red=Fore.RED, white=Fore.WHITE)) if check_open_ports(): - print "[{green}+{white}]Port {PORT} in {ad} is open"\ + print "[{green}+{white}] Port {PORT} in {ad} is open"\ .format(PORT=server.config.faraday_server.port, green=Fore.GREEN,white=Fore.WHITE,ad=server.config.faraday_server.bind_address) else: - print "[{red}-{white}] in {ad} is not open"\ - .format(PORT=server.config.faraday_server.port,red=Fore.RED,white=Fore.WHITE,ad =server.config.faraday_server.bind_address) \ No newline at end of file + print "[{red}-{white}] Port {PORT} in {ad} is not open"\ + .format(PORT=server.config.faraday_server.port,red=Fore.RED,white=Fore.WHITE,ad =server.config.faraday_server.bind_address) From fc6656f233ba30a6773bb34604c77517edfd2f20 Mon Sep 17 00:00:00 2001 From: Nicolas Ronchi Cesarini Date: Wed, 30 May 2018 00:36:26 -0300 Subject: [PATCH 1199/1506] Cambios en look and feel de la vista de hosts emulando la de status report --- server/www/scripts/hosts/partials/list.html | 289 ++++++++++-------- .../scripts/statusReport/styles/status.css | 22 ++ 2 files changed, 187 insertions(+), 124 deletions(-) diff --git a/server/www/scripts/hosts/partials/list.html b/server/www/scripts/hosts/partials/list.html index 4b587e2e232..331ac36c680 100644 --- a/server/www/scripts/hosts/partials/list.html +++ b/server/www/scripts/hosts/partials/list.html @@ -6,168 +6,209 @@
    - -
    - - - -
    - - - - - New - -
    -
    -
    -
    -
    -
    - - - - - - -
    +
    +
    + +
    +
    + +
    +
    + +
    +
    + +
    +
    + + + + +
    - -
    -
    -

    Add columns

    -
      +
    + +
    +
    +
    +
    + + +
    - + +
    +
    - - - + + - - - - - - - - - - - - - - - - - - - + + + - - - - - - - - diff --git a/server/www/scripts/statusReport/styles/status.css b/server/www/scripts/statusReport/styles/status.css index e71ef1dcf86..01834b6ba50 100644 --- a/server/www/scripts/statusReport/styles/status.css +++ b/server/www/scripts/statusReport/styles/status.css @@ -60,6 +60,28 @@ width:44px; } +.status-report-grid .ui-grid-row:last-child .ui-grid-cell{ + border:none; +} + +.checkbox-select{ + text-align: center !important; +} + +.hosts-list .ui-grid-header .ui-grid-cell-contents{ + padding: 8px 3px 11px 5px !important +} + +.hosts-list .ui-grid-header .ui-grid-cell-contents span{ + color: #7c7c7c; +} + +.hosts-list .ui-grid-header .hosts-list-checkall{ + position: relative; + top: 3px; + left: 7px; +} + .status-report-grid .ui-grid-pager-panel { background-color: #f1f1f1; } From d9b1bc2e22c7b66c487d27eab15c088e2494d538 Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Wed, 30 May 2018 11:23:51 -0300 Subject: [PATCH 1200/1506] 4774 - Update workspace table styles --- server/www/index.html | 1 + .../www/scripts/workspaces/partials/list.html | 28 ++-- server/www/table-v3.css | 129 ++++++++++++++++++ 3 files changed, 144 insertions(+), 14 deletions(-) create mode 100644 server/www/table-v3.css diff --git a/server/www/index.html b/server/www/index.html index 0f481605b6c..c4b56ad5ca1 100644 --- a/server/www/index.html +++ b/server/www/index.html @@ -33,6 +33,7 @@ + diff --git a/server/www/scripts/workspaces/partials/list.html b/server/www/scripts/workspaces/partials/list.html index cfa3ca7c679..fc95802af3c 100644 --- a/server/www/scripts/workspaces/partials/list.html +++ b/server/www/scripts/workspaces/partials/list.html @@ -41,30 +41,30 @@

    -
    - ID - +
    + + + ID + - IP - + + IP + - OS - + + OS + - Services - + + SERVICES + - Hostnames - + + GOS + - MAC - + + MAC + - Open Services - + + OPEN SERVICES + - Vulns - + + VULNS + - Credentials - + + CREDENTIALS + - Owned - + + OWNED + - Description - + + DESCRIPTION + - Creation time - + + CREATION TIME + - Last modified - + + LAST MODIFIED +
    - {{host.id}} + +
    + +
    		- {{host.ip}} - + + - - - - + +
    + {{host.ip}} + +
    		- - {{summary}}
    -     +     		+ - - {{hostname}}
    -     +     		+
    + + {{summary}}
    +     +
    +     		+
    + + {{hostname}}
    +     +
    +     		+
    + {{host.mac}} +
    +     		+
    + +
    		- {{host.mac}} + +
    + +
    		+ {{host.credentials}} - - owned - not yet + +
    + + owned + not yet +
    		- {{host.description}} + +
    + {{host.description}} +
    		- + +
    + +
    		- + +
    + +
    +
    - - - - - - - + + + + + + + - - - - - - + + + + +
    NameStart DateEnd DateVulnsHostsServices
    NameStart DateEnd DateVulnsHostsServices
    + {{ws.name}} {{objects[ws.name]['total_vulns']}}{{objects[ws.name]['hosts']}}{{objects[ws.name]['total_vulns']}}{{objects[ws.name]['hosts']}}
    diff --git a/server/www/table-v3.css b/server/www/table-v3.css new file mode 100644 index 00000000000..ec83c0c8287 --- /dev/null +++ b/server/www/table-v3.css @@ -0,0 +1,129 @@ +.table-v3 .ui-grid-header { + border-top: solid #E5E5E5 2px; + border-bottom: 2px solid #d8d8d8; + background: none; + color: #E9E9E9; + padding: 0px; + height: 45px; +} + +.table-v3 .ui-grid-header .ui-grid-cell-contents { + color: #7c7c7c; + padding: 9px 3px 9px 5px !important; + text-align: left; +} + +.table-v3 .ui-grid-cell-contents { + text-align: left; +} + +.table-v3 .ui-grid-cell-contents.confirm-toggle{ + text-align: center; +} + +.table-v3 .ui-grid-header-cell{ + border-right: solid #E9E9E9 1px; + text-align: center; +} + +.table-v3 .ui-grid-header-cell i, .table-v3 .grid_id{ + padding: 11px 9px 9px 5px !important +} + +.table-v3 .ui-grid-row { + height: 45px; +} + +.table-v3 .date-cell { + opacity: 0.5; +} + +.table-v3 .grid_id .ui-grid-cell-contents{ + position: relative; + top: 3px; +} + +.table-v3 .ui-grid-row-selected .ui-grid-cell{ + background-color: #8DA2B7 !important; + color: #FFFFFF; +} + +.table-v3 .ui-grid-row-selected a{ + color: #FFFFFF; +} + +.table-v3 .no-background{ + background: none; +} + +.table-v3 .ui-grid-row:last-child .ui-grid-cell{ + border:none; +} + +.table-v3 .ui-grid-pager-panel { + background-color: #f1f1f1; +} + +.table-v3 .status.opened { + background-color: rgba(223, 57, 54, .6) !important; + text-align: center; +} + +.table-v3 .status-wrapper { + height: 100%; +} + +.table-v3 .status.opened span { + color: #DF3936; +} + +.table-v3 .status.risk-accepted { + background-color: rgba(46, 151, 189, 0.6) !important; + text-align: center; +} + +.table-v3 .status.re-opened span { + color: rgba(223, 191, 53, 1); +} + +.table-v3 .status.re-opened { + background-color: rgba(223, 191, 53, 0.6) !important; + text-align: center; +} + +.table-v3 .status.risk-accepted span { + color: rgba(46, 151, 189, 1); +} + +.table-v3 .status.closed { + background-color: rgba(161, 206, 49, 0.6) !important; + text-align: center; +} + +.table-v3 .status.closed span { + color: rgba(161, 206, 49, 1); +} + +.table-v3 .confirm-icon { + width: 40%; +} + +.table-v3 .ui-grid-header-cell{ + text-align: center; +} + +.table-v3 .ui-grid-header-cell .glyphicon-remove{ + color: #ccc; + font-size: 8px; + font-weight: lighter; + top: 0px; + left: 4px; +} + +.table-v3 .ui-grid-contents-wrapper{ + clear: both; +} + +.table-v3 .ui-grid-cell { + border: none ; +} From f82ebc07cbb2baf9f45618681225a4da2a3ef836 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 30 May 2018 11:53:48 -0300 Subject: [PATCH 1201/1506] [FIX] URL join from + to urlparse.urljoin --- persistence/server/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/persistence/server/server.py b/persistence/server/server.py index a5af42794f1..60c41db728f 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -1503,7 +1503,7 @@ def server_info(): def login_user(uri, uname, upass): auth = {"email": uname, "password": upass} try: - resp = requests.post(uri + "/_api/login", json=auth) + resp = requests.post(urlparse.urljoin(uri, "/_api/login"), json=auth) if resp.status_code == 401: return None else: @@ -1516,7 +1516,7 @@ def login_user(uri, uname, upass): def is_authenticated(uri, cookies): try: - resp = requests.get(uri + "/_api/session", cookies=cookies, timeout=1) + resp = requests.get(urlparse.urljoin(uri, "/_api/session"), cookies=cookies, timeout=1) if resp.status_code != 403: user_info = resp.json() return bool(user_info.get('username', {})) @@ -1555,7 +1555,7 @@ def test_server_url(url_to_test): def get_user_info(): try: - resp = requests.get(_get_base_server_url() + "/_api/session", cookies=_conf().getDBSessionCookies(), timeout=1) + resp = requests.get(urlparse.urljoin(_get_base_server_url(), "/_api/session"), cookies=_conf().getDBSessionCookies(), timeout=1) if resp.status_code != 403: return resp.json() else: From afb74e2e54784781713026c66d00e037bdfc1259 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Wed, 30 May 2018 13:02:59 -0300 Subject: [PATCH 1202/1506] [REF] Change 'test_server_url' to 'check_server_url' as it is not a test function --- gui/gtk/application.py | 2 +- gui/gtk/dialogs.py | 4 ++-- gui/gtk/server.py | 4 ++-- .../_build/html/_modules/persistence/server/server.html | 2 +- persistence/server/docs/_build/html/_modules/server.html | 2 +- persistence/server/docs/_build/html/genindex.html | 2 +- persistence/server/docs/_build/html/server.html | 2 +- persistence/server/models.py | 4 ++-- persistence/server/server.py | 2 +- test_cases/test_persistence_server_server.py | 8 ++++---- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/gui/gtk/application.py b/gui/gtk/application.py index 2b0ed76f889..e969e638ca9 100644 --- a/gui/gtk/application.py +++ b/gui/gtk/application.py @@ -344,7 +344,7 @@ def connect_to_couch(self, server_url, parent=None): if parent is None: parent = self.window - if not self.serverIO.test_server_url(server_url): + if not self.serverIO.check_server_url(server_url): errorDialog(parent, "Could not connect to Faraday Server.", ("Are you sure it is running and that you can " "connect to it? \n Make sure your username and " diff --git a/gui/gtk/dialogs.py b/gui/gtk/dialogs.py index 5546408782e..a4979099898 100644 --- a/gui/gtk/dialogs.py +++ b/gui/gtk/dialogs.py @@ -14,7 +14,7 @@ from gi.repository import Gtk, GdkPixbuf, Gdk from config.configuration import getInstanceConfiguration -from persistence.server.server import is_authenticated, login_user, get_user_info, test_server_url +from persistence.server.server import is_authenticated, login_user, get_user_info, check_server_url from model import guiapi from decorators import scrollable @@ -76,7 +76,7 @@ def on_click_ok(self, button=None): """ repourl = self.ip_entry.get_text() - if not test_server_url(repourl): + if not check_server_url(repourl): errorDialog(self, "Could not connect to Faraday Server.", ("Are you sure it is running and that the URL is correct?")) return False diff --git a/gui/gtk/server.py b/gui/gtk/server.py index d9bf5e396c5..6fc35e50bf6 100644 --- a/gui/gtk/server.py +++ b/gui/gtk/server.py @@ -86,8 +86,8 @@ def server_info(self): return models.server_info() @safe_io_with_server(False) - def test_server_url(self, url): - return models.test_server_url(url) + def check_server_url(self, url): + return models.check_server_url(url) @safe_io_with_server(None) def get_changes_stream(self): diff --git a/persistence/server/docs/_build/html/_modules/persistence/server/server.html b/persistence/server/docs/_build/html/_modules/persistence/server/server.html index cfdd503e825..bdd5d69fd92 100644 --- a/persistence/server/docs/_build/html/_modules/persistence/server/server.html +++ b/persistence/server/docs/_build/html/_modules/persistence/server/server.html @@ -1589,7 +1589,7 @@

    Source code for persistence.server.server

             is_server_up = False
     return is_server_up
    -
    [docs]def test_server_url(url_to_test): +
    [docs]def check_server_url(url_to_test): """Return True if the url_to_test is indeed a valid Faraday Server URL. False otherwise. """ diff --git a/persistence/server/docs/_build/html/_modules/server.html b/persistence/server/docs/_build/html/_modules/server.html index 0046eb14ed4..586cca00591 100644 --- a/persistence/server/docs/_build/html/_modules/server.html +++ b/persistence/server/docs/_build/html/_modules/server.html @@ -1509,7 +1509,7 @@

    Source code for server

             is_server_up = False
     return is_server_up
    -
    [docs]def test_server_url(url_to_test): +
    [docs]def check_server_url(url_to_test): """Return True if the url_to_test is indeed a valid Faraday Server URL. False otherwise. """ diff --git a/persistence/server/docs/_build/html/genindex.html b/persistence/server/docs/_build/html/genindex.html index 5f9e870b6ca..1c8be6868d0 100644 --- a/persistence/server/docs/_build/html/genindex.html +++ b/persistence/server/docs/_build/html/genindex.html @@ -279,7 +279,7 @@

    P

    T

    diff --git a/persistence/server/docs/_build/html/server.html b/persistence/server/docs/_build/html/server.html index 30157715dcf..e879bc595e1 100644 --- a/persistence/server/docs/_build/html/server.html +++ b/persistence/server/docs/_build/html/server.html @@ -1473,7 +1473,7 @@

    Submodules
    -persistence.server.server.test_server_url(url_to_test)[source]
    +persistence.server.server.check_server_url(url_to_test)[source]

    Return True if the url_to_test is indeed a valid Faraday Server URL. False otherwise.

    diff --git a/persistence/server/models.py b/persistence/server/models.py index f275dd6f413..72b61ea6e4a 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -679,9 +679,9 @@ def server_info(): return server.server_info() -def test_server_url(url_to_test): +def check_server_url(url_to_test): """Return True if url_to_test/_api/info is accessible, False otherwise""" - return server.test_server_url(url_to_test) + return server.check_server_url(url_to_test) # NOTE: the whole 'which arguments are mandatory and which type should they be" diff --git a/persistence/server/server.py b/persistence/server/server.py index 60c41db728f..753a7b0a02d 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -1541,7 +1541,7 @@ def check_faraday_version(): if info is not None and version != info['Version']: raise RuntimeError('Client and server versions do not match') -def test_server_url(url_to_test): +def check_server_url(url_to_test): """Return True if the url_to_test is indeed a valid Faraday Server URL. False otherwise. """ diff --git a/test_cases/test_persistence_server_server.py b/test_cases/test_persistence_server_server.py index 743e298bc81..3aae3555a9b 100644 --- a/test_cases/test_persistence_server_server.py +++ b/test_cases/test_persistence_server_server.py @@ -15,7 +15,7 @@ class TestServerFuncions: def test_test_server_url_server_runnimg(self, test_client): """ The name of the method is correct we are testing the function: - test_server_url + check_server_url """ mocked_response = { @@ -25,13 +25,13 @@ def test_test_server_url_server_runnimg(self, test_client): responses.add(responses.GET, 'http://localhost/_api/v2/info', json=mocked_response, status=200) - assert server.test_server_url('http://localhost') + assert server.check_server_url('http://localhost') @responses.activate def test_test_server_url_another_http_returns_404(self): responses.add(responses.GET, 'http://localhost/_api/v2/info', status=404) - assert not server.test_server_url('http://localhost') + assert not server.check_server_url('http://localhost') def test_test_server_url_aserver_down(self, test_client): - assert not server.test_server_url('http://localhost') + assert not server.check_server_url('http://localhost') From 68d2caa625211b7e97524b90bf47215d1a9563c0 Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Wed, 30 May 2018 14:03:48 -0300 Subject: [PATCH 1203/1506] 4773 - Update vulndb styles --- server/www/estilos-v3.css | 4 + .../scripts/vulndb/controllers/vulnModels.js | 5 + .../www/scripts/vulndb/partials/vulndb.html | 97 +++++++++---------- server/www/table-v3.css | 10 +- 4 files changed, 66 insertions(+), 50 deletions(-) diff --git a/server/www/estilos-v3.css b/server/www/estilos-v3.css index 04aefb252b3..006459f3e7f 100644 --- a/server/www/estilos-v3.css +++ b/server/www/estilos-v3.css @@ -140,3 +140,7 @@ bottom: 0px; right: 15px; } + +.justify-flex-start { + justify-content: flex-start !important; +} diff --git a/server/www/scripts/vulndb/controllers/vulnModels.js b/server/www/scripts/vulndb/controllers/vulnModels.js index 397b2299901..4b76d4d784e 100644 --- a/server/www/scripts/vulndb/controllers/vulnModels.js +++ b/server/www/scripts/vulndb/controllers/vulnModels.js @@ -340,6 +340,7 @@ angular.module('faradayApp') $scope.toggleSort = function(field) { $scope.toggleSortField(field); $scope.toggleReverse(); + $scope.sort(); }; // toggles column sort field @@ -352,6 +353,10 @@ angular.module('faradayApp') $scope.reverse = !$scope.reverse; }; + $scope.clearSearch = function() { + $scope.search = ''; + }; + var equalAsSets = function(a, b) { if(a.length != b.length) return false; diff --git a/server/www/scripts/vulndb/partials/vulndb.html b/server/www/scripts/vulndb/partials/vulndb.html index b58ef5dc8a7..0fa0f36b78c 100644 --- a/server/www/scripts/vulndb/partials/vulndb.html +++ b/server/www/scripts/vulndb/partials/vulndb.html @@ -6,56 +6,56 @@
    -
    -
    -
    - -
    - -
    - -
    -
    +
    + +
    +
    + -
    -
    -
    - - - - +
    +
    +
    +
    +
    + + + + + +
    +
    - + +
    - - - + - - - - + + @@ -64,12 +64,11 @@ selection-model-mode="multiple-additive" selection-model-selected-class="multi-selected" selection-model-on-change="selectedModels()"> - - - - - - + + + + +
    - Name +
    + - Description + + Name - Resolution + + Description - Exploitation + + Resolution + Exploitation +
    {{model.name}}{{model.description}}{{model.resolution}}{{model.exploitation}}{{model.name}}{{model.description}}{{model.resolution}}{{model.exploitation}}
    @@ -88,6 +87,6 @@
    -
    -

    +
    +
    diff --git a/server/www/table-v3.css b/server/www/table-v3.css index ec83c0c8287..e214ec2244b 100644 --- a/server/www/table-v3.css +++ b/server/www/table-v3.css @@ -7,7 +7,8 @@ height: 45px; } -.table-v3 .ui-grid-header .ui-grid-cell-contents { +.table-v3 .ui-grid-header .ui-grid-cell-contents, +.table-v3 .ui-grid-header .ui-grid-cell-contents span { color: #7c7c7c; padding: 9px 3px 9px 5px !important; text-align: left; @@ -15,6 +16,7 @@ .table-v3 .ui-grid-cell-contents { text-align: left; + border: none; } .table-v3 .ui-grid-cell-contents.confirm-toggle{ @@ -127,3 +129,9 @@ .table-v3 .ui-grid-cell { border: none ; } + +.table-v3 .ui-grid-header .hosts-list-checkall{ + position: relative; + top: 3px; + left: 7px; +} From 21522633987eabfb4c2b3900dc61d274a3b1f9cd Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Wed, 30 May 2018 14:26:26 -0300 Subject: [PATCH 1204/1506] 4774 - Update workspace view buttons styles --- .../workspaces/controllers/workspaces.js | 30 +++++++++------- .../www/scripts/workspaces/partials/list.html | 36 +++++++++++++++++-- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/server/www/scripts/workspaces/controllers/workspaces.js b/server/www/scripts/workspaces/controllers/workspaces.js index 082603ccade..fc7d7479962 100644 --- a/server/www/scripts/workspaces/controllers/workspaces.js +++ b/server/www/scripts/workspaces/controllers/workspaces.js @@ -16,7 +16,7 @@ angular.module('faradayApp') $scope.workspaces = []; $scope.wss = []; // $scope.newworkspace = {}; - + var hash_tmp = window.location.hash.split("/")[1]; switch (hash_tmp){ case "status": @@ -63,7 +63,7 @@ angular.module('faradayApp') }; $scope.onSuccessInsert = function(workspace){ - $scope.wss.push(workspace.name); + $scope.wss.push(workspace.name); workspace.scope = workspace.scope.map(function(scope){ return {key: scope} }); @@ -75,9 +75,9 @@ angular.module('faradayApp') for (var i = 0; i < $scope.workspaces.length; i++) { if ($scope.workspaces[i].name > workspace.name) break; } - $scope.workspaces.splice(i, 0, workspace); + $scope.workspaces.splice(i, 0, workspace); }; - + $scope.onFailInsert = function(error){ var modal = $uibModal.open({ templateUrl: 'scripts/commons/partials/modalKO.html', @@ -90,7 +90,7 @@ angular.module('faradayApp') return error; } } - }); + }); }; $scope.onSuccessEdit = function(workspace){ @@ -109,7 +109,7 @@ angular.module('faradayApp') }; }; - $scope.onSuccessDelete = function(workspace_name){ + $scope.onSuccessDelete = function(workspace_name){ remove = function(arr, item) { for(var i = arr.length; i--;) { if(arr[i] === item) { @@ -119,7 +119,7 @@ angular.module('faradayApp') return arr; }; - $scope.wss = remove($scope.wss, workspace_name); + $scope.wss = remove($scope.wss, workspace_name); for(var i = 0; i < $scope.workspaces.length; i++) { if($scope.workspaces[i].name == workspace_name){ $scope.workspaces.splice(i, 1); @@ -127,7 +127,7 @@ angular.module('faradayApp') } }; }; - + $scope.insert = function(workspace){ delete workspace.selected; workspacesFact.put(workspace).then(function(resp){ @@ -187,7 +187,7 @@ angular.module('faradayApp') }; // Modals methods - $scope.new = function(){ + $scope.new = function(){ $scope.modal = $uibModal.open({ templateUrl: 'scripts/workspaces/partials/modalNew.html', controller: 'workspacesModalNew', @@ -201,12 +201,12 @@ angular.module('faradayApp') }).filter(Boolean); workspace = $scope.create(workspace.name, workspace.description, workspace.start_date, workspace.end_date, api_scope); - $scope.insert(workspace); + $scope.insert(workspace); }); }; - $scope.edit = function(){ + $scope.edit = function(){ var workspace; $scope.workspaces.forEach(function(w) { if(w.selected) { @@ -236,7 +236,7 @@ angular.module('faradayApp') $scope.update(workspace, oldName, function(workspace){ workspace.scope = old_scope; - }); + }); }); } else { var modal = $uibModal.open({ @@ -278,7 +278,7 @@ angular.module('faradayApp') $scope.modal.result.then(function() { $scope.workspaces.forEach(function(w) { if(w.selected == true) - $scope.remove(w.name); + $scope.remove(w.name); }); }); } else { @@ -324,5 +324,9 @@ angular.module('faradayApp') $location.path("/dashboard/ws/"+path); }; + $scope.clearSearch = function() { + $scope.search = ''; + }; + $scope.init(); }]); diff --git a/server/www/scripts/workspaces/partials/list.html b/server/www/scripts/workspaces/partials/list.html index fc95802af3c..b1edcec9309 100644 --- a/server/www/scripts/workspaces/partials/list.html +++ b/server/www/scripts/workspaces/partials/list.html @@ -11,7 +11,7 @@

    Workspaces
    -
    +

    -
    +
    +
    +
    + +
    +
    + +
    +
    + +
    +
    +
    +
    +
    + + + + + +
    +
    +
    +
    +
    From 3230b88a49c44e94ee0bbdd27346f4d76889188f Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Wed, 30 May 2018 16:45:46 -0300 Subject: [PATCH 1205/1506] 4775 - Update licenses styles view --- server/www/scripts/hosts/partials/list.html | 12 +-- .../scripts/licenses/controllers/licenses.js | 6 +- .../www/scripts/licenses/partials/list.html | 98 ++++++++++--------- 3 files changed, 64 insertions(+), 52 deletions(-) diff --git a/server/www/scripts/hosts/partials/list.html b/server/www/scripts/hosts/partials/list.html index 331ac36c680..ef8950b6655 100644 --- a/server/www/scripts/hosts/partials/list.html +++ b/server/www/scripts/hosts/partials/list.html @@ -3,17 +3,17 @@
    -
    +
    -
    -
    @@ -129,7 +129,7 @@ {{host.id}}
    -
    - @@ -203,7 +203,7 @@ - - + @@ -25,10 +25,7 @@

    Last Vulnerabilities {{vuln.target}}

    - + diff --git a/server/www/scripts/dashboard/partials/modal-services-by-host.html b/server/www/scripts/dashboard/partials/modal-services-by-host.html index 0298a553bb0..1a6037903bb 100644 --- a/server/www/scripts/dashboard/partials/modal-services-by-host.html +++ b/server/www/scripts/dashboard/partials/modal-services-by-host.html @@ -3,42 +3,61 @@
    -

    Services for {{name}} ({{services.length}} total)

    +

    Services for {{name}} ({{services.length}} total)

    -
    {{host.ip}} @@ -179,7 +179,7 @@
    		{{host.credentials}}
    diff --git a/server/www/scripts/licenses/controllers/licenses.js b/server/www/scripts/licenses/controllers/licenses.js index e6857e3cf23..d3fa084805d 100644 --- a/server/www/scripts/licenses/controllers/licenses.js +++ b/server/www/scripts/licenses/controllers/licenses.js @@ -196,7 +196,11 @@ angular.module('faradayApp') // toggle column sort order $scope.toggleReverse = function() { $scope.reverse = !$scope.reverse; - } + }; + + $scope.clearSearch = function() { + $scope.search = ''; + }; init(); }]); diff --git a/server/www/scripts/licenses/partials/list.html b/server/www/scripts/licenses/partials/list.html index 39de606e741..fd4e013cd74 100644 --- a/server/www/scripts/licenses/partials/list.html +++ b/server/www/scripts/licenses/partials/list.html @@ -3,58 +3,65 @@
    -
    +
    +

    Licenses ({{licenses.length}}) - - -

    -
    -
    -
    - - - - -
    +
    +
    + +
    +
    + +
    +
    + +
    +
    +
    +
    +
    + + + + + +
    +
    +

    The licenses marked in dark red are about to expire

    - +
    - - - + + - - - - - + @@ -64,18 +71,19 @@

    The licenses m selection-model-selected-class="multi-selected" selection-model-on-change="selectedLicenses()" ng-class="{'almost-expired': almostExpired(license.end)}"> -

    - - - - - - + + + + + +
    - Product +
    + Product - Type + + Type - Notes + + Notes - Start date + + Start date - End date + + End date
    {{license.product}}{{license.lictype}}{{license.notes}}{{license.start | date:'MM/dd/yyyy'}}{{license.end|date:'MM/dd/yyyy'}}{{license.product}}{{license.lictype}}{{license.notes}}{{license.start | date:'MM/dd/yyyy'}}{{license.end|date:'MM/dd/yyyy'}}
    -
    +
    +
    From 2fa12cfd8e5ed68b28f89db9460eac14d7d20969 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 30 May 2018 17:53:37 -0300 Subject: [PATCH 1206/1506] [FIX] Return empty list if an object was deleted while the api was doing the serialization --- server/api/base.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/api/base.py b/server/api/base.py index 40612fd1719..001e54edbd8 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -5,7 +5,7 @@ from flask import abort, g from flask_classful import FlaskView from sqlalchemy.orm import joinedload, undefer -from sqlalchemy.orm.exc import NoResultFound +from sqlalchemy.orm.exc import NoResultFound, ObjectDeletedError from sqlalchemy.inspection import inspect from sqlalchemy import func from marshmallow import Schema @@ -150,7 +150,10 @@ def _get_object(self, object_id, eagerload=False, **kwargs): return obj def _dump(self, obj, route_kwargs, **kwargs): - return self._get_schema_instance(route_kwargs, **kwargs).dump(obj).data + try: + return self._get_schema_instance(route_kwargs, **kwargs).dump(obj).data + except ObjectDeletedError: + return [] def _parse_data(self, schema, request, *args, **kwargs): return FlaskParser().parse(schema, request, locations=('json',), From ecebae490979eaa9b004830b8913501b3f390e57 Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Wed, 30 May 2018 18:02:06 -0300 Subject: [PATCH 1207/1506] 4772 - Update credentials view styles --- .../scripts/credentials/partials/list.html | 90 ++++++++++--------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/server/www/scripts/credentials/partials/list.html b/server/www/scripts/credentials/partials/list.html index e73a1185aa4..abfd04b30a8 100644 --- a/server/www/scripts/credentials/partials/list.html +++ b/server/www/scripts/credentials/partials/list.html @@ -6,50 +6,56 @@
    -

    - - - -

    +
    +
    +
    + +
    +
    + +
    +
    + +
    +
    +
    +
    +
    + + + + + +
    +
    +
    +
    +
    +
    -
    -
    -
    - - - - -
    -
    -
    - +
    - - - + + - - - @@ -59,11 +65,11 @@

    selection-model-mode="multiple-additive" selection-model-selected-class="multi-selected" selection-model-on-change="selectedCredentials()"> -

    - - - - + + + + +
    - Target +
    + Target - Name + + Name - Username + + Username - Password + + Password
    {{credential.target}} {{credential.name}}{{credential.username}}{{credential.password}} {{credential.target}} {{credential.name}}{{credential.username}}{{credential.password}}
    From 1aff17d1c23b4517daad76487da4e7e4a684eb9c Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 30 May 2018 19:07:46 -0300 Subject: [PATCH 1208/1506] First version upload report from webUi --- persistence/server/models.py | 3 +- server/api/modules/upload_reports.py | 112 +++++++++++++++++++++++++++ server/app.py | 2 + server/config.py | 10 +++ server/web.py | 4 + 5 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 server/api/modules/upload_reports.py diff --git a/persistence/server/models.py b/persistence/server/models.py index f275dd6f413..9e2ab3c30eb 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -748,7 +748,8 @@ def getID(self): retries = 1 max_retries = 6 while retries <= max_retries and self.id is None: - print('Retrying getID timeout {0}'.format(timeout)) + if timeout >= 8: + logger.info('Retrying getID timeout {0}'.format(timeout)) self.id_available.wait(timeout=timeout) timeout = timeout << retries - 1 retries += 1 diff --git a/server/api/modules/upload_reports.py b/server/api/modules/upload_reports.py new file mode 100644 index 00000000000..32ba00a539e --- /dev/null +++ b/server/api/modules/upload_reports.py @@ -0,0 +1,112 @@ +# Faraday Penetration Test IDE +# Copyright (C) 2018 Infobyte LLC (http://www.infobytesec.com/) +# See the file 'doc/LICENSE' for the license information + +from Queue import Queue +from multiprocessing import Queue as MultiProcessingQueue +from threading import Thread + +import os +import string +import random +import logging +import model.api + +from flask import request, abort, jsonify, Blueprint +from werkzeug.utils import secure_filename + +from server.utils.logger import get_logger +from server.utils.web import gzipped + +from model.controller import ModelController +from model.cli_app import CliApp + +from plugins.controller import PluginController +from plugins.manager import PluginManager + +from managers.workspace_manager import WorkspaceManager +from managers.mapper_manager import MapperManager + +from config.configuration import getInstanceConfiguration +CONF = getInstanceConfiguration() + +upload_api = Blueprint('upload_reports', __name__) +logger = logging.getLogger(__name__) +UPLOAD_REPORTS_QUEUE = MultiProcessingQueue() + + +class RawReportProcessor(Thread): + def __init__(self): + + super(RawReportProcessor, self).__init__() + plugin_manager = PluginManager(os.path.join(CONF.getConfigPath(), "plugins")) + + mappers_manager = MapperManager() + self.pending_actions = Queue() + + model_controller = ModelController(mappers_manager, self.pending_actions) + + workspace_manager = WorkspaceManager(mappers_manager) + CONF.setMergeStrategy("new") + model.api.setUpAPIs(model_controller, workspace_manager, 0, 0) + model_controller.start() + + plugin_controller = PluginController( + 'PluginController', + plugin_manager, + mappers_manager, + self.pending_actions) + + self.cli_import = CliApp(workspace_manager, plugin_controller) + + def run(self): + + while True: + try: + + workspace, file_path = UPLOAD_REPORTS_QUEUE.get() + logger.info('Processing raw report {0}'.format(file_path)) + + class Arg(): + def __init__(self): + self.workspace = workspace + self.filename = file_path + + self.cli_import.run(Arg()) + except KeyboardInterrupt: + break + + +@gzipped +@upload_api.route('/v2/ws//upload_report', methods=['POST']) +def file_upload(workspace=None): + """ + Upload a report file to Server and process that report with Faraday client plugins. + """ + + get_logger(__name__).debug("Importing new plugin report in server...") + + if 'file' not in request.files: + abort(400) + + report_file = request.files['file'] + + if report_file.filename == '': + abort(400) + + if report_file: + + chars = string.ascii_uppercase + string.digits + random_prefix = ''.join(random.choice(chars) for x in range(12)) + raw_report_filename = '{0}{1}'.format(random_prefix, secure_filename(report_file.filename)) + + file_path = os.path.join( + CONF.getConfigPath(), + 'uploaded_reports/{0}'.format(raw_report_filename)) + + with open(file_path, 'w') as output: + output.write(report_file.read()) + + UPLOAD_REPORTS_QUEUE.put((workspace, file_path)) + + return jsonify({"status": "processing"}) diff --git a/server/app.py b/server/app.py index 4bd204cc075..c570277da64 100644 --- a/server/app.py +++ b/server/app.py @@ -64,6 +64,7 @@ def register_blueprints(app): from server.api.modules.workspaces import workspace_api from server.api.modules.handlers import handlers_api from server.api.modules.comments import comment_api + from server.api.modules.upload_reports import upload_api app.register_blueprint(commandsrun_api) app.register_blueprint(credentials_api) app.register_blueprint(host_api) @@ -76,6 +77,7 @@ def register_blueprints(app): app.register_blueprint(workspace_api) app.register_blueprint(handlers_api) app.register_blueprint(comment_api) + app.register_blueprint(upload_api) def check_testing_configuration(testing, app): diff --git a/server/config.py b/server/config.py index acee0692cbf..6233f09a52e 100644 --- a/server/config.py +++ b/server/config.py @@ -26,12 +26,15 @@ REPORTS_VIEWS_DIR = os.path.join(FARADAY_BASE, 'views/reports') LOCAL_CONFIG_FILE = os.path.expanduser( os.path.join(CONSTANTS.CONST_FARADAY_HOME_PATH, 'config/server.ini')) +LOCAL_REPORTS_FOLDER = os.path.expanduser( + os.path.join(CONSTANTS.CONST_FARADAY_HOME_PATH, 'uploaded_reports/')) CONFIG_FILES = [DEFAULT_CONFIG_FILE, LOCAL_CONFIG_FILE] WS_BLACKLIST = CONSTANTS.CONST_BLACKDBS def copy_default_config_to_local(): + if os.path.exists(LOCAL_CONFIG_FILE): return @@ -45,6 +48,13 @@ def copy_default_config_to_local(): # Copy default config file into faraday local config shutil.copyfile(DEFAULT_CONFIG_FILE, LOCAL_CONFIG_FILE) + if not os.path.exists(LOCAL_REPORTS_FOLDER): + try: + os.makedirs(LOCAL_REPORTS_FOLDER) + except OSError as e: + if e.errno != errno.EEXIST: + raise + from server.utils.logger import get_logger get_logger(__name__).info(u"Local faraday-server configuration created at {}".format(LOCAL_CONFIG_FILE)) diff --git a/server/web.py b/server/web.py index af78e5de6e3..bcb90341f2d 100644 --- a/server/web.py +++ b/server/web.py @@ -32,6 +32,7 @@ WorkspaceServerFactory, BroadcastServerProtocol ) +from server.api.modules.upload_reports import RawReportProcessor app = create_app() # creates a Flask(__name__) app logger = server.utils.logger.get_logger(__name__) @@ -141,6 +142,9 @@ def run(self): self.__listen_func = reactor.listenTCP try: + # start threads and processes + raw_report_processor = RawReportProcessor() + raw_report_processor.start() # web and static content self.__listen_func( self.__listen_port, site, From 260c6edac7a6dcccc8db0c4510fd94a9ca55be37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 31 May 2018 12:00:20 -0300 Subject: [PATCH 1209/1506] Add copyright and license information for new files in v3 And in other files that didn't have any license information --- data/fix_severities.py | 3 +++ gui/gtk/__init__.py | 4 ++++ gui/gtk/decorators.py | 3 +++ helpers/cfdbToCsv.py | 1 + helpers/vulndbToCsv.py | 1 + manage.py | 6 ++++++ persistence/server/__init__.py | 7 +++++++ plugins/fplugin_utils.py | 6 ++++++ plugins/repo/pasteanalyzer/plugin.py | 6 ++++++ plugins/repo/traceroute/plugin.py | 6 ++++++ plugins/repo/wpscan/__init__.py | 7 +++++++ scripts/cscan/plugin/msfrpc.py | 6 ++++++ scripts/reposify/reposify.py | 6 ++++++ scripts/reposify/reposify_api.py | 8 +++++++- scripts/wcscans/__init__.py | 7 +++++++ server/api/base.py | 6 ++++++ server/api/modules/__init__.py | 7 +++++++ server/api/modules/handlers.py | 8 +++++++- server/api/modules/session.py | 6 ++++++ server/commands/__init__.py | 7 +++++++ server/commands/app_urls.py | 8 +++++++- server/commands/faraday_schema_display.py | 6 ++++++ server/commands/initdb.py | 6 ++++++ server/commands/reports.py | 8 +++++++- server/commands/reset_db.py | 6 ++++++ server/events.py | 6 ++++++ server/fields.py | 6 ++++++ server/schemas.py | 6 ++++++ server/utils/invalid_chars.py | 6 ++++++ server/websocket_factories.py | 6 ++++++ test_cases/conftest.py | 6 ++++++ test_cases/factories.py | 6 ++++++ test_cases/models/test_cascades.py | 6 ++++++ test_cases/models/test_file.py | 6 ++++++ test_cases/models/test_host.py | 6 ++++++ test_cases/models/test_tag.py | 6 ++++++ test_cases/models/test_vulnerability.py | 6 ++++++ test_cases/models/test_workspace.py | 6 ++++++ test_cases/plugins/test_common.py | 6 ++++++ test_cases/test_ModelObjectFactory.py | 7 +++++++ test_cases/test_api_commands.py | 8 +++++++- test_cases/test_api_comment.py | 8 +++++++- test_cases/test_api_credentials.py | 6 ++++++ test_cases/test_api_hosts.py | 6 ++++++ test_cases/test_api_info.py | 8 +++++++- test_cases/test_api_license.py | 6 ++++++ test_cases/test_api_non_workspaced_base.py | 8 +++++++- test_cases/test_api_pagination.py | 6 ++++++ test_cases/test_api_services.py | 6 ++++++ test_cases/test_api_vulnerability.py | 6 ++++++ test_cases/test_api_vulnerability_template.py | 8 +++++++- test_cases/test_api_websocket_auth.py | 6 ++++++ test_cases/test_api_workspace.py | 6 ++++++ test_cases/test_api_workspaced_base.py | 6 ++++++ test_cases/test_config_configuration.py | 7 +++++++ test_cases/test_import_users_from_couch.py | 6 ++++++ test_cases/test_managers_mapper_manager.py | 6 ++++++ test_cases/test_marshmallow_fields.py | 6 ++++++ test_cases/test_model_controller.py | 6 ++++++ test_cases/test_model_events.py | 6 ++++++ test_cases/test_model_fields.py | 8 +++++++- test_cases/test_models.py | 6 ++++++ test_cases/test_persistence_server_models.py | 6 ++++++ test_cases/test_persistence_server_server.py | 6 ++++++ test_cases/test_plugins_controller.py | 6 ++++++ test_cases/test_server.py | 6 ++++++ test_cases/test_server_config.py | 6 ++++++ test_cases/test_server_io.py | 6 ++++++ test_cases/test_utils_database.py | 8 +++++++- 69 files changed, 414 insertions(+), 11 deletions(-) diff --git a/data/fix_severities.py b/data/fix_severities.py index dde72aac7b2..0e318709835 100644 --- a/data/fix_severities.py +++ b/data/fix_severities.py @@ -1,3 +1,6 @@ +# Faraday Penetration Test IDE +# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) +# See the file 'doc/LICENSE' for the license information import csv import click from collections import OrderedDict diff --git a/gui/gtk/__init__.py b/gui/gtk/__init__.py index e69de29bb2d..50a8239b3f9 100644 --- a/gui/gtk/__init__.py +++ b/gui/gtk/__init__.py @@ -0,0 +1,4 @@ +# Faraday Penetration Test IDE +# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) +# See the file 'doc/LICENSE' for the license information + diff --git a/gui/gtk/decorators.py b/gui/gtk/decorators.py index a5e82fdab22..adba7918b64 100644 --- a/gui/gtk/decorators.py +++ b/gui/gtk/decorators.py @@ -1,3 +1,6 @@ +# Faraday Penetration Test IDE +# Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) +# See the file 'doc/LICENSE' for the license information import requests from gi.repository import Gtk from utils.logs import getLogger diff --git a/helpers/cfdbToCsv.py b/helpers/cfdbToCsv.py index 312c082fec6..a2bc8308acd 100755 --- a/helpers/cfdbToCsv.py +++ b/helpers/cfdbToCsv.py @@ -3,6 +3,7 @@ ''' Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) Author: Ezequiel Tavella +See the file 'doc/LICENSE' for the license information This script generate a CSV file with information about the cfdb database. CSV Format: diff --git a/helpers/vulndbToCsv.py b/helpers/vulndbToCsv.py index 1196e8549f5..e8fa5e10392 100755 --- a/helpers/vulndbToCsv.py +++ b/helpers/vulndbToCsv.py @@ -3,6 +3,7 @@ ''' Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) Author: Ezequiel Tavella +See the file 'doc/LICENSE' for the license information This script generate a CSV file with information about the vulndb database. CSV Format: diff --git a/manage.py b/manage.py index 7bdce489079..5a1bcf01b52 100755 --- a/manage.py +++ b/manage.py @@ -1,4 +1,10 @@ #!/usr/bin/env python +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import re diff --git a/persistence/server/__init__.py b/persistence/server/__init__.py index e69de29bb2d..47c52b07c21 100644 --- a/persistence/server/__init__.py +++ b/persistence/server/__init__.py @@ -0,0 +1,7 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' + diff --git a/plugins/fplugin_utils.py b/plugins/fplugin_utils.py index eb04b1e7726..20f93edf174 100644 --- a/plugins/fplugin_utils.py +++ b/plugins/fplugin_utils.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import imp import os import sys diff --git a/plugins/repo/pasteanalyzer/plugin.py b/plugins/repo/pasteanalyzer/plugin.py index f9e97b6dee2..d1419a1c37e 100644 --- a/plugins/repo/pasteanalyzer/plugin.py +++ b/plugins/repo/pasteanalyzer/plugin.py @@ -1,5 +1,11 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' # Author: @EzequielTBH diff --git a/plugins/repo/traceroute/plugin.py b/plugins/repo/traceroute/plugin.py index 6638e47709b..9e678d4ee59 100644 --- a/plugins/repo/traceroute/plugin.py +++ b/plugins/repo/traceroute/plugin.py @@ -1,5 +1,11 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' from plugins import core import re diff --git a/plugins/repo/wpscan/__init__.py b/plugins/repo/wpscan/__init__.py index e69de29bb2d..47c52b07c21 100644 --- a/plugins/repo/wpscan/__init__.py +++ b/plugins/repo/wpscan/__init__.py @@ -0,0 +1,7 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' + diff --git a/scripts/cscan/plugin/msfrpc.py b/scripts/cscan/plugin/msfrpc.py index c18da648436..641138d0056 100755 --- a/scripts/cscan/plugin/msfrpc.py +++ b/scripts/cscan/plugin/msfrpc.py @@ -1,4 +1,10 @@ #!/usr/bin/env python2 +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import os import time diff --git a/scripts/reposify/reposify.py b/scripts/reposify/reposify.py index 58ecd2dd460..7939b914c42 100644 --- a/scripts/reposify/reposify.py +++ b/scripts/reposify/reposify.py @@ -1,4 +1,10 @@ #!/usr/bin/env python +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import sys import xmlrpclib diff --git a/scripts/reposify/reposify_api.py b/scripts/reposify/reposify_api.py index 8dedef259af..0e8b17a3f66 100644 --- a/scripts/reposify/reposify_api.py +++ b/scripts/reposify/reposify_api.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import requests import simplejson @@ -64,4 +70,4 @@ def reposify_search(key, banner, filters, page): if filters is not None: params['filters'] = filters res = api_request(key, '/v1/insights/search', params, None, 'https://api.reposify.com', 'get', 1) - return res \ No newline at end of file + return res diff --git a/scripts/wcscans/__init__.py b/scripts/wcscans/__init__.py index e69de29bb2d..47c52b07c21 100644 --- a/scripts/wcscans/__init__.py +++ b/scripts/wcscans/__init__.py @@ -0,0 +1,7 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' + diff --git a/server/api/base.py b/server/api/base.py index 40612fd1719..abf0d09ae8f 100644 --- a/server/api/base.py +++ b/server/api/base.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import json import flask diff --git a/server/api/modules/__init__.py b/server/api/modules/__init__.py index e69de29bb2d..47c52b07c21 100644 --- a/server/api/modules/__init__.py +++ b/server/api/modules/__init__.py @@ -0,0 +1,7 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' + diff --git a/server/api/modules/handlers.py b/server/api/modules/handlers.py index ee86d5a6d8a..7904447f0bd 100644 --- a/server/api/modules/handlers.py +++ b/server/api/modules/handlers.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' from flask import jsonify, Blueprint handlers_api = Blueprint('handlers_api', __name__) @@ -12,4 +18,4 @@ def error_response(e): -#.register(commandsrun_api) \ No newline at end of file +#.register(commandsrun_api) diff --git a/server/api/modules/session.py b/server/api/modules/session.py index 4348eff6ee5..a8473b03570 100644 --- a/server/api/modules/session.py +++ b/server/api/modules/session.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' from flask import jsonify, session, Blueprint, current_app session_api = Blueprint('session_api', __name__) diff --git a/server/commands/__init__.py b/server/commands/__init__.py index e69de29bb2d..47c52b07c21 100644 --- a/server/commands/__init__.py +++ b/server/commands/__init__.py @@ -0,0 +1,7 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' + diff --git a/server/commands/app_urls.py b/server/commands/app_urls.py index 4e1066ac05a..d90ac41c16a 100644 --- a/server/commands/app_urls.py +++ b/server/commands/app_urls.py @@ -1,5 +1,11 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' from server.web import app def show_all_urls(): - print(app.url_map) \ No newline at end of file + print(app.url_map) diff --git a/server/commands/faraday_schema_display.py b/server/commands/faraday_schema_display.py index de224b767d2..e62181e75d3 100644 --- a/server/commands/faraday_schema_display.py +++ b/server/commands/faraday_schema_display.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' from __future__ import print_function import sys from sqlalchemy import MetaData diff --git a/server/commands/initdb.py b/server/commands/initdb.py index aa88d9de9aa..f3949e5aebf 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import getpass import shutil import string diff --git a/server/commands/reports.py b/server/commands/reports.py index c1db8981c9c..89b5b569598 100644 --- a/server/commands/reports.py +++ b/server/commands/reports.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import os import logging from Queue import Queue @@ -61,4 +67,4 @@ def process_workspaces(mappers_manager, plugin_manager, query, disable_polling): # report_manager.join() #for controller in controllers: - # controller.join() \ No newline at end of file + # controller.join() diff --git a/server/commands/reset_db.py b/server/commands/reset_db.py index 773dc7ec927..5e95b73ffd0 100644 --- a/server/commands/reset_db.py +++ b/server/commands/reset_db.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' from server.models import db def reset_db_all(): diff --git a/server/events.py b/server/events.py index 9a2e5d1579c..6d9f35446a0 100644 --- a/server/events.py +++ b/server/events.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import sys import logging import inspect diff --git a/server/fields.py b/server/fields.py index 895fbb1268c..ec53b6b0217 100644 --- a/server/fields.py +++ b/server/fields.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import imghdr from tempfile import SpooledTemporaryFile diff --git a/server/schemas.py b/server/schemas.py index 30ed6c974f2..4824a9afa0d 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import time import datetime from marshmallow import fields, Schema diff --git a/server/utils/invalid_chars.py b/server/utils/invalid_chars.py index f3d8098f581..97238a5cfa5 100644 --- a/server/utils/invalid_chars.py +++ b/server/utils/invalid_chars.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import re import sys diff --git a/server/websocket_factories.py b/server/websocket_factories.py index b448656efcb..e6833841c6c 100644 --- a/server/websocket_factories.py +++ b/server/websocket_factories.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import json import logging import itsdangerous diff --git a/test_cases/conftest.py b/test_cases/conftest.py index 8f3c2d5c4b9..983f20ef428 100644 --- a/test_cases/conftest.py +++ b/test_cases/conftest.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' from tempfile import NamedTemporaryFile import os diff --git a/test_cases/factories.py b/test_cases/factories.py index a5ab5842c82..094eaa030e4 100644 --- a/test_cases/factories.py +++ b/test_cases/factories.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import random import factory import datetime diff --git a/test_cases/models/test_cascades.py b/test_cases/models/test_cascades.py index 4157cc8fcb3..5187ae529f7 100644 --- a/test_cases/models/test_cascades.py +++ b/test_cases/models/test_cascades.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import pytest from contextlib import contextmanager from server.models import ( diff --git a/test_cases/models/test_file.py b/test_cases/models/test_file.py index ff6190a2691..3fd91673400 100644 --- a/test_cases/models/test_file.py +++ b/test_cases/models/test_file.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import os import pytest from server.models import File diff --git a/test_cases/models/test_host.py b/test_cases/models/test_host.py index aba8e7d38c0..4979b7c925a 100644 --- a/test_cases/models/test_host.py +++ b/test_cases/models/test_host.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import random import pytest from functools import partial diff --git a/test_cases/models/test_tag.py b/test_cases/models/test_tag.py index 9c3dea4a8fd..2e02938644f 100644 --- a/test_cases/models/test_tag.py +++ b/test_cases/models/test_tag.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import pytest from server.models import TagObject diff --git a/test_cases/models/test_vulnerability.py b/test_cases/models/test_vulnerability.py index c0d2b99cbb2..62706f4f76a 100644 --- a/test_cases/models/test_vulnerability.py +++ b/test_cases/models/test_vulnerability.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import pytest from server.models import ( CommandObject, diff --git a/test_cases/models/test_workspace.py b/test_cases/models/test_workspace.py index 390bfa506f3..d4fcb6f434b 100644 --- a/test_cases/models/test_workspace.py +++ b/test_cases/models/test_workspace.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' from server.models import db, Workspace from test_cases.factories import ( HostFactory, diff --git a/test_cases/plugins/test_common.py b/test_cases/plugins/test_common.py index aee5bd38c1d..f879742ef25 100644 --- a/test_cases/plugins/test_common.py +++ b/test_cases/plugins/test_common.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' def skip(self, n): for x in range(n): action = self.plugin._pending_actions.get(block=True) diff --git a/test_cases/test_ModelObjectFactory.py b/test_cases/test_ModelObjectFactory.py index e69de29bb2d..47c52b07c21 100644 --- a/test_cases/test_ModelObjectFactory.py +++ b/test_cases/test_ModelObjectFactory.py @@ -0,0 +1,7 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' + diff --git a/test_cases/test_api_commands.py b/test_cases/test_api_commands.py index c9a3781901a..dd0fb71d863 100644 --- a/test_cases/test_api_commands.py +++ b/test_cases/test_api_commands.py @@ -1,4 +1,10 @@ #-*- coding: utf8 -*- +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' """Tests for many API endpoints that do not depend on workspace_name""" import datetime import pytest @@ -377,4 +383,4 @@ def test_delete_objects_preserve_history(self, session, test_client): assert len(command_history) command_history = command_history[0] assert command_history['hosts_count'] == 1 - assert command_history['tool'] == 'test' \ No newline at end of file + assert command_history['tool'] == 'test' diff --git a/test_cases/test_api_comment.py b/test_cases/test_api_comment.py index 05fda625f40..c7a13704e98 100644 --- a/test_cases/test_api_comment.py +++ b/test_cases/test_api_comment.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' from server.api.modules.comments import CommentView from server.models import Comment from test_cases.factories import ServiceFactory @@ -84,4 +90,4 @@ def test_create_unique_comment_for_plugins(self, session, test_client): res = test_client.post(url, data=raw_comment) assert res.status_code == 409 assert 'object' in res.json - assert type(res.json) == dict \ No newline at end of file + assert type(res.json) == dict diff --git a/test_cases/test_api_credentials.py b/test_cases/test_api_credentials.py index 3e4ee44a534..ae1578cb8b3 100644 --- a/test_cases/test_api_credentials.py +++ b/test_cases/test_api_credentials.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import pytest from test_cases import factories diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index a74e5b644e1..c4e2ee63977 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import time import operator diff --git a/test_cases/test_api_info.py b/test_cases/test_api_info.py index d3c7c5b25e4..6d64d04de60 100644 --- a/test_cases/test_api_info.py +++ b/test_cases/test_api_info.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import os import pytest @@ -18,4 +24,4 @@ def test_api_info(self, test_client): assert response.status_code == 200 assert response.json['Faraday Server'] == 'Running' # to avoid side effects - os.chdir(current_dir) \ No newline at end of file + os.chdir(current_dir) diff --git a/test_cases/test_api_license.py b/test_cases/test_api_license.py index e979d2f1eeb..7c8c6ac789c 100644 --- a/test_cases/test_api_license.py +++ b/test_cases/test_api_license.py @@ -1,4 +1,10 @@ #-*- coding: utf8 -*- +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' """Tests for many API endpoints that do not depend on workspace_name""" import pytest diff --git a/test_cases/test_api_non_workspaced_base.py b/test_cases/test_api_non_workspaced_base.py index 7e0f7b16757..bb9ca847920 100644 --- a/test_cases/test_api_non_workspaced_base.py +++ b/test_cases/test_api_non_workspaced_base.py @@ -1,4 +1,10 @@ #-*- coding: utf8 -*- +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' """Generic tests for APIs NOT prefixed with a workspace_name""" @@ -151,4 +157,4 @@ class ReadWriteAPITests(ReadWriteTestsMixin, class ReadOnlyAPITests(ListTestsMixin, RetrieveTestsMixin, GenericAPITest): - pass \ No newline at end of file + pass diff --git a/test_cases/test_api_pagination.py b/test_cases/test_api_pagination.py index cd60aa2e387..a5bb0d69099 100644 --- a/test_cases/test_api_pagination.py +++ b/test_cases/test_api_pagination.py @@ -1,4 +1,10 @@ #-*- coding: utf8 -*- +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' """Generic test mixins for APIs with pagination enabled when listing""" diff --git a/test_cases/test_api_services.py b/test_cases/test_api_services.py index c8186878ea3..557062d64d6 100644 --- a/test_cases/test_api_services.py +++ b/test_cases/test_api_services.py @@ -1,4 +1,10 @@ # -*- coding: utf8 -*- +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' """Tests for many API endpoints that do not depend on workspace_name""" try: from urllib import urlencode diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 4fdb9ec0b4b..21db9b7dbc9 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1,4 +1,10 @@ #-*- coding: utf8 -*- +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' from StringIO import StringIO from tempfile import NamedTemporaryFile diff --git a/test_cases/test_api_vulnerability_template.py b/test_cases/test_api_vulnerability_template.py index 1e37c3b784c..beb948b6f7a 100644 --- a/test_cases/test_api_vulnerability_template.py +++ b/test_cases/test_api_vulnerability_template.py @@ -1,4 +1,10 @@ #-*- coding: utf8 -*- +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import os import pytest @@ -167,4 +173,4 @@ def test_when_a_template_with_ref_is_deleted_ref_remains_at_database( res = test_client.delete(self.url(template)) assert res.status_code == 204 - assert session.query(ReferenceTemplate).count() == 1 \ No newline at end of file + assert session.query(ReferenceTemplate).count() == 1 diff --git a/test_cases/test_api_websocket_auth.py b/test_cases/test_api_websocket_auth.py index 3cc6d28ebcd..56766d215c4 100644 --- a/test_cases/test_api_websocket_auth.py +++ b/test_cases/test_api_websocket_auth.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import pytest diff --git a/test_cases/test_api_workspace.py b/test_cases/test_api_workspace.py index 6fdcd6f12af..0b8eb1b18ce 100644 --- a/test_cases/test_api_workspace.py +++ b/test_cases/test_api_workspace.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import time import pytest diff --git a/test_cases/test_api_workspaced_base.py b/test_cases/test_api_workspaced_base.py index 784ed4becf2..447709dc784 100644 --- a/test_cases/test_api_workspaced_base.py +++ b/test_cases/test_api_workspaced_base.py @@ -1,4 +1,10 @@ #-*- coding: utf8 -*- +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' """Generic tests for APIs prefixed with a workspace_name""" diff --git a/test_cases/test_config_configuration.py b/test_cases/test_config_configuration.py index e69de29bb2d..47c52b07c21 100644 --- a/test_cases/test_config_configuration.py +++ b/test_cases/test_config_configuration.py @@ -0,0 +1,7 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' + diff --git a/test_cases/test_import_users_from_couch.py b/test_cases/test_import_users_from_couch.py index cb7f9ca6761..fff212bd8df 100644 --- a/test_cases/test_import_users_from_couch.py +++ b/test_cases/test_import_users_from_couch.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import pytest from passlib.hash import pbkdf2_sha1 from server.importer import ImportCouchDBUsers diff --git a/test_cases/test_managers_mapper_manager.py b/test_cases/test_managers_mapper_manager.py index c018ac3e62e..f47ca70ba4e 100644 --- a/test_cases/test_managers_mapper_manager.py +++ b/test_cases/test_managers_mapper_manager.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' from functools import partial import pytest diff --git a/test_cases/test_marshmallow_fields.py b/test_cases/test_marshmallow_fields.py index a9cad9eaf0b..8ef9045a90d 100644 --- a/test_cases/test_marshmallow_fields.py +++ b/test_cases/test_marshmallow_fields.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import time import datetime import pytest diff --git a/test_cases/test_model_controller.py b/test_cases/test_model_controller.py index 1b2cac7e4f2..98d3824c147 100644 --- a/test_cases/test_model_controller.py +++ b/test_cases/test_model_controller.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' from Queue import Queue import time diff --git a/test_cases/test_model_events.py b/test_cases/test_model_events.py index ece1c9d78c9..476c2119dc8 100644 --- a/test_cases/test_model_events.py +++ b/test_cases/test_model_events.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import pytest from test_cases.factories import HostFactory, ServiceFactory diff --git a/test_cases/test_model_fields.py b/test_cases/test_model_fields.py index a6e91bb7884..3cdc0631629 100644 --- a/test_cases/test_model_fields.py +++ b/test_cases/test_model_fields.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import os from server.fields import FaradayUploadedFile @@ -26,4 +32,4 @@ def test_normal_attach_is_not_detected_as_image(): with open(os.path.join(CURRENT_PATH, 'data', 'report_w3af.xml'))as image_data: field = FaradayUploadedFile(image_data.read()) assert field['content_type'] == 'application/octet-stream' - assert len(field['files']) == 1 \ No newline at end of file + assert len(field['files']) == 1 diff --git a/test_cases/test_models.py b/test_cases/test_models.py index 4ca847a9bfe..ee387c6aa85 100644 --- a/test_cases/test_models.py +++ b/test_cases/test_models.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import unittest import json from persistence.server import models diff --git a/test_cases/test_persistence_server_models.py b/test_cases/test_persistence_server_models.py index 6f6ad78c6d6..81bc910d1a5 100644 --- a/test_cases/test_persistence_server_models.py +++ b/test_cases/test_persistence_server_models.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import persistence.server.models as models import pytest import responses diff --git a/test_cases/test_persistence_server_server.py b/test_cases/test_persistence_server_server.py index 743e298bc81..b8cf9d26413 100644 --- a/test_cases/test_persistence_server_server.py +++ b/test_cases/test_persistence_server_server.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import os import responses diff --git a/test_cases/test_plugins_controller.py b/test_cases/test_plugins_controller.py index 01bca562cb2..c11d7701c4d 100644 --- a/test_cases/test_plugins_controller.py +++ b/test_cases/test_plugins_controller.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import sys sys.path.append('.') import unittest diff --git a/test_cases/test_server.py b/test_cases/test_server.py index b55da585d6c..555d22f208e 100644 --- a/test_cases/test_server.py +++ b/test_cases/test_server.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import os import sys import unittest diff --git a/test_cases/test_server_config.py b/test_cases/test_server_config.py index c688bbf03a1..e6f9125d1c9 100644 --- a/test_cases/test_server_config.py +++ b/test_cases/test_server_config.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import random import string import mock diff --git a/test_cases/test_server_io.py b/test_cases/test_server_io.py index 4de4476fe41..ec5207856a1 100644 --- a/test_cases/test_server_io.py +++ b/test_cases/test_server_io.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import os import sys import unittest diff --git a/test_cases/test_utils_database.py b/test_cases/test_utils_database.py index 46ae7e6e685..4fa88dba492 100644 --- a/test_cases/test_utils_database.py +++ b/test_cases/test_utils_database.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import pytest from server.utils.database import get_unique_fields @@ -55,4 +61,4 @@ def test_unique_fields_workspace(obj_class, expected_unique_fields, session): object_ = obj_class() unique_constraints = get_unique_fields(session, object_) for unique_constraint in unique_constraints: - assert unique_constraint == expected_unique_fields \ No newline at end of file + assert unique_constraint == expected_unique_fields From 2bc297eb0e5058ed949c71eaaa9975385a35468c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 31 May 2018 12:17:41 -0300 Subject: [PATCH 1210/1506] Fix a couple non-deterministic test cases A module-level monkeypatch of one test suite broke another one depending of the server.ini file contents. Fixed --- test_cases/test_server_io.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test_cases/test_server_io.py b/test_cases/test_server_io.py index ec5207856a1..3e9f0346532 100644 --- a/test_cases/test_server_io.py +++ b/test_cases/test_server_io.py @@ -15,8 +15,6 @@ from persistence.server import server_io_exceptions from mock import MagicMock, patch -server.FARADAY_UP = False -server.SERVER_URL = "http://localhost:5984" example_url = "http://just_some_url" @@ -25,6 +23,14 @@ class ClientServerAPITests(unittest.TestCase): def setUp(self): self.ws_name = "a_ws" self.server_api_url = "http://localhost:5984/_api" + self.old_faraday_up = server.FARADAY_UP + self.old_server_url = server.SERVER_URL + server.FARADAY_UP = False + server.SERVER_URL = "http://localhost:5984" + + def tearDown(self): + server.FARADAY_UP = self.old_faraday_up + server.SERVER_URL = self.old_server_url def test_get_base_server_url(self): s = server._get_base_server_url() From 3146971b7a665c59fc78d39956100ec3831d17ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 31 May 2018 13:07:25 -0300 Subject: [PATCH 1211/1506] Revert "Fix a couple non-deterministic test cases" This reverts commit 2bc297eb0e5058ed949c71eaaa9975385a35468c. --- test_cases/test_server_io.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/test_cases/test_server_io.py b/test_cases/test_server_io.py index 3e9f0346532..ec5207856a1 100644 --- a/test_cases/test_server_io.py +++ b/test_cases/test_server_io.py @@ -15,6 +15,8 @@ from persistence.server import server_io_exceptions from mock import MagicMock, patch +server.FARADAY_UP = False +server.SERVER_URL = "http://localhost:5984" example_url = "http://just_some_url" @@ -23,14 +25,6 @@ class ClientServerAPITests(unittest.TestCase): def setUp(self): self.ws_name = "a_ws" self.server_api_url = "http://localhost:5984/_api" - self.old_faraday_up = server.FARADAY_UP - self.old_server_url = server.SERVER_URL - server.FARADAY_UP = False - server.SERVER_URL = "http://localhost:5984" - - def tearDown(self): - server.FARADAY_UP = self.old_faraday_up - server.SERVER_URL = self.old_server_url def test_get_base_server_url(self): s = server._get_base_server_url() From 44df67cb8bfe3f33c9580c40f9fa52fca18eaf55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 31 May 2018 13:09:47 -0300 Subject: [PATCH 1212/1506] Correct test case monkeypatched url --- test_cases/test_server_io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_cases/test_server_io.py b/test_cases/test_server_io.py index ec5207856a1..543a9b7d8b0 100644 --- a/test_cases/test_server_io.py +++ b/test_cases/test_server_io.py @@ -16,7 +16,7 @@ from mock import MagicMock, patch server.FARADAY_UP = False -server.SERVER_URL = "http://localhost:5984" +server.SERVER_URL = "http://localhost:5985" example_url = "http://just_some_url" @@ -24,7 +24,7 @@ class ClientServerAPITests(unittest.TestCase): def setUp(self): self.ws_name = "a_ws" - self.server_api_url = "http://localhost:5984/_api" + self.server_api_url = "http://localhost:5985/_api" def test_get_base_server_url(self): s = server._get_base_server_url() From ca5a6600f14b42c330850431253c62ead503654b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 31 May 2018 14:41:51 -0300 Subject: [PATCH 1213/1506] Make workspace name not blankable This was on pink but not in white. --- server/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index abe684513fb..c809e452277 100644 --- a/server/models.py +++ b/server/models.py @@ -1163,7 +1163,7 @@ class Workspace(Metadata): description = BlankColumn(Text) active = Column(Boolean(), nullable=False, default=True) # TBI end_date = Column(DateTime(), nullable=True) - name = Column(String(250), nullable=False, unique=True) + name = NonBlankColumn(String(250), unique=True, nullable=False) public = Column(Boolean(), nullable=False, default=False) # TBI start_date = Column(DateTime(), nullable=True) From 2a462a42361214d933b3065a36a29462a5a2550f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 31 May 2018 15:04:01 -0300 Subject: [PATCH 1214/1506] Update rollbar environment for white --- server/www/scripts/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/app.js b/server/www/scripts/app.js index 00117b132c4..88be6e820e2 100644 --- a/server/www/scripts/app.js +++ b/server/www/scripts/app.js @@ -291,7 +291,7 @@ faradayApp.config(['$routeProvider', 'ngClipProvider', '$uibTooltipProvider', 'R accessToken: "70f0c36ae96d4ffc90394565b42c5bf9", captureUncaught: true, payload: { - environment: "white" + environment: "white-newdesign" }}); }]); From b5267c0620d2c06386384a43082ea74f660797bf Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 31 May 2018 17:10:35 -0300 Subject: [PATCH 1215/1506] [ADD] Add version 3.0a0 to white --- VERSION | 2 +- plugins/plugin.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 860487ca19c..78480b2a640 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.7.1 +3.0a0+white diff --git a/plugins/plugin.py b/plugins/plugin.py index 473d36aa2a1..b45b4804933 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -33,7 +33,7 @@ from config.configuration import getInstanceConfiguration CONF = getInstanceConfiguration() -VERSION = server.config.__get_version() +VERSION = server.config.__get_version().split('+')[0] logger = logging.getLogger(__name__) From d4ba3b707891bade27a3ebc3e84c24d2431f99a9 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 31 May 2018 17:20:23 -0300 Subject: [PATCH 1216/1506] [MOD] update release.md --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index d1925242e10..53acf5e2f11 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -10,6 +10,7 @@ New features in the latest update TBA: --- +* Add reconng plugin * CouchDB was replaced by PostgreSQL :) * Host object changed, now the name property is called ip * Interface object was removed From 3ae9f663de29fe7ee466c7a1d22189d1cc99b1e0 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 31 May 2018 17:34:53 -0300 Subject: [PATCH 1217/1506] [MOD] add colors to database checks --- faraday-server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/faraday-server.py b/faraday-server.py index 8b8787947a2..ae70c0a5f68 100755 --- a/faraday-server.py +++ b/faraday-server.py @@ -8,6 +8,7 @@ import subprocess import sqlalchemy +from colorama import init, Fore import server.config import server.couchdb @@ -20,6 +21,7 @@ from faraday import FARADAY_BASE logger = server.utils.logger.get_logger(__name__) +init() def setup_environment(check_deps=False): @@ -90,7 +92,7 @@ def check_postgresql(): logger.warn('No workspaces found. Remeber to execute couchdb importer') except sqlalchemy.exc.OperationalError: logger.error( - 'Could not connect to postgresql, please check if database is running or configuration settings are correct. For first time installations execute python manage.py initdb') + '\n\n{RED}Could not connect to postgresql.\nPlease check{WHITE}: \n{YELLOW} * if database is running \n * configuration settings are correct. \n\n{RED}For first time installations execute{WHITE}: \n\n {GREEN} python manage.py initdb\n\n'.format(GREEN=Fore.GREEN, YELLOW=Fore.YELLOW, WHITE=Fore.WHITE, RED=Fore.RED)) sys.exit(1) From caf1e3457253fc844b7752538a11e5501413e29c Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 31 May 2018 18:13:51 -0300 Subject: [PATCH 1218/1506] Import from couchdb when using initdb --- manage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/manage.py b/manage.py index 0626d348690..1356b8b1111 100755 --- a/manage.py +++ b/manage.py @@ -75,6 +75,7 @@ def faraday_schema_display(): def initdb(): with app.app_context(): InitDB().run() + ImportCouchDB().run() @click.command() def import_from_couchdb(): From 3833610f7c30706c7d41fc19497c9fb7abdae018 Mon Sep 17 00:00:00 2001 From: Nicolas Ronchi Cesarini Date: Fri, 1 Jun 2018 08:33:55 -0300 Subject: [PATCH 1219/1506] Fixing issues and navigation enhacements --- server/www/estilos-v3.css | 4 ++++ server/www/scripts/commons/controllers/headerCtrl.js | 9 ++++----- server/www/scripts/commons/partials/header.html | 4 ++-- server/www/scripts/credentials/partials/list.html | 2 +- server/www/scripts/services/partials/list.html | 4 ++-- .../www/scripts/statusReport/controllers/statusReport.js | 2 +- server/www/scripts/statusReport/styles/status.css | 8 ++++---- server/www/scripts/vulndb/partials/vulndb.html | 2 +- server/www/scripts/vulns/providers/vulns.js | 9 +++++++-- 9 files changed, 26 insertions(+), 18 deletions(-) diff --git a/server/www/estilos-v3.css b/server/www/estilos-v3.css index 006459f3e7f..ddb5bef68be 100644 --- a/server/www/estilos-v3.css +++ b/server/www/estilos-v3.css @@ -144,3 +144,7 @@ .justify-flex-start { justify-content: flex-start !important; } + +.max-sized-table { + max-width: 1200px; +} diff --git a/server/www/scripts/commons/controllers/headerCtrl.js b/server/www/scripts/commons/controllers/headerCtrl.js index 427701c66bc..05cd681a7f0 100644 --- a/server/www/scripts/commons/controllers/headerCtrl.js +++ b/server/www/scripts/commons/controllers/headerCtrl.js @@ -13,6 +13,10 @@ angular.module('faradayApp') return noNav.indexOf($scope.component) < 0; }; + $scope.getVulnsNum = function() { + return vulnsManager.getVulnsNum(); + }; + init = function(name) { $scope.location = $location.path().split('/')[1]; $scope.workspace = $routeParams.wsId; @@ -21,11 +25,6 @@ angular.module('faradayApp') workspacesFact.list().then(function(wss) { $scope.workspaces = wss; }); - - vulnsManager.getVulns($scope.workspace, null, null, null, null, null) - .then(function(response) { - $scope.totalItems = response.count; - }); }; init(); diff --git a/server/www/scripts/commons/partials/header.html b/server/www/scripts/commons/partials/header.html index 51fe3347f84..56d5b188308 100644 --- a/server/www/scripts/commons/partials/header.html +++ b/server/www/scripts/commons/partials/header.html @@ -19,12 +19,12 @@
    - {{totalItems}} vulns total + {{getVulnsNum()}} vulns total
    diff --git a/server/www/scripts/credentials/partials/list.html b/server/www/scripts/credentials/partials/list.html index abfd04b30a8..173b20a8789 100644 --- a/server/www/scripts/credentials/partials/list.html +++ b/server/www/scripts/credentials/partials/list.html @@ -41,7 +41,7 @@
    - +
    diff --git a/server/www/scripts/services/partials/list.html b/server/www/scripts/services/partials/list.html index 808bb0070ca..75666d3f9ed 100644 --- a/server/www/scripts/services/partials/list.html +++ b/server/www/scripts/services/partials/list.html @@ -21,7 +21,7 @@

    - +

    Host services

     -
    +

    Host details diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 95009806ae3..3bdee68cf42 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -297,7 +297,7 @@ angular.module("faradayApp") $scope.gridOptions.columnDefs.push({ name : 'severity', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/severitycolumn.html', headerCellTemplate: header, - displayName : "sev", + displayName : "severity", type: 'string', visible: $scope.columns["severity"], sort: getColumnSort('severity'), diff --git a/server/www/scripts/statusReport/styles/status.css b/server/www/scripts/statusReport/styles/status.css index 01834b6ba50..eb3b6f78d02 100644 --- a/server/www/scripts/statusReport/styles/status.css +++ b/server/www/scripts/statusReport/styles/status.css @@ -87,7 +87,7 @@ } .status-report-grid .status.opened { - background-color: rgba(223, 57, 54, .6) !important; + background-color: rgba(223, 57, 54, .4) !important; text-align: center; } @@ -100,7 +100,7 @@ } .status-report-grid .status.risk-accepted { - background-color: rgba(46, 151, 189, 0.6) !important; + background-color: rgba(46, 151, 189, 0.4) !important; text-align: center; } @@ -109,7 +109,7 @@ } .status-report-grid .status.re-opened { - background-color: rgba(223, 191, 53, 0.6) !important; + background-color: rgba(223, 191, 53, 0.4) !important; text-align: center; } @@ -118,7 +118,7 @@ } .status-report-grid .status.closed { - background-color: rgba(161, 206, 49, 0.6) !important; + background-color: rgba(161, 206, 49, 0.4) !important; text-align: center; } diff --git a/server/www/scripts/vulndb/partials/vulndb.html b/server/www/scripts/vulndb/partials/vulndb.html index 0fa0f36b78c..06d94c8e6d1 100644 --- a/server/www/scripts/vulndb/partials/vulndb.html +++ b/server/www/scripts/vulndb/partials/vulndb.html @@ -6,7 +6,7 @@
    -
    +
    -
    -
    From 968fb8ce932383c21ae254b4eec8a81bc334e023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 1 Jun 2018 15:41:09 -0300 Subject: [PATCH 1223/1506] Add ugly hack to fill workspace vuln count in credentials view --- .../scripts/credentials/controllers/credentials.js | 7 +++++-- server/www/scripts/vulns/providers/vulns.js | 11 +++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/server/www/scripts/credentials/controllers/credentials.js b/server/www/scripts/credentials/controllers/credentials.js index b4bdb571176..c9c6a6cae77 100644 --- a/server/www/scripts/credentials/controllers/credentials.js +++ b/server/www/scripts/credentials/controllers/credentials.js @@ -6,8 +6,8 @@ angular.module('faradayApp') .controller('credentialsCtrl', - ['$scope', '$filter', '$q', '$uibModal', '$routeParams', '$window', 'commonsFact', 'credential', 'ServerAPI', 'workspacesFact', - function($scope, $filter, $q, $uibModal, $routeParams, $window, commonsFact, credential, ServerAPI, workspacesFact) { + ['$scope', '$filter', '$q', '$uibModal', '$routeParams', '$window', 'commonsFact', 'credential', 'ServerAPI', 'workspacesFact', 'vulnsManager', + function($scope, $filter, $q, $uibModal, $routeParams, $window, commonsFact, credential, ServerAPI, workspacesFact, vulnsManager) { $scope.workspace; $scope.workspaces; @@ -112,6 +112,9 @@ angular.module('faradayApp') getParent().then(function(){ getAndLoadCredentials(); }); + + // Make the workspace vuln counter work + vulnsManager.loadVulnsCounter($scope.workspace); }; var removeFromView = function(credential){ diff --git a/server/www/scripts/vulns/providers/vulns.js b/server/www/scripts/vulns/providers/vulns.js index a7f43824a3e..8b5aa56d0d5 100644 --- a/server/www/scripts/vulns/providers/vulns.js +++ b/server/www/scripts/vulns/providers/vulns.js @@ -4,8 +4,8 @@ angular.module('faradayApp') .factory('vulnsManager', - ['Vuln', 'WebVuln', '$q', 'ServerAPI', 'commonsFact', - function(Vuln, WebVuln, $q, ServerAPI, commonsFact) { + ['Vuln', 'WebVuln', '$q', 'ServerAPI', 'commonsFact', 'workspacesFact', + function(Vuln, WebVuln, $q, ServerAPI, commonsFact, workspacesFact) { var vulnsManager = {}; var vulnsCounter = 0; @@ -71,6 +71,13 @@ angular.module('faradayApp') return deferred.promise; }; + vulnsManager.loadVulnsCounter = function(ws){ + // Ugly hack to populate the vulnsCounter global variable + workspacesFact.get(ws).then(function(data){ + vulnsCounter = data.stats.total_vulns; + }) + }; + vulnsManager.getVulnsNum = function() { return vulnsCounter; }; From df0baadd5d3d4ed65b052ddbbe5603120ac78745 Mon Sep 17 00:00:00 2001 From: Nicolas Ronchi Cesarini Date: Fri, 1 Jun 2018 17:11:06 -0300 Subject: [PATCH 1224/1506] fixes to switcher for white --- server/www/header.css | 6 +++++- .../scripts/commons/controllers/headerCtrl.js | 9 ++++----- .../www/scripts/commons/partials/header.html | 18 ++++++++++++++++-- server/www/scripts/hosts/partials/list.html | 2 +- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/server/www/header.css b/server/www/header.css index 0642579e655..4bd7022d6d2 100644 --- a/server/www/header.css +++ b/server/www/header.css @@ -32,12 +32,16 @@ float: left; } -.workspace-selector-wrapper { +.workspace-selector-wrapper, .icon-wrapper { border-radius: 5px; margin-right: 20px; padding: 10px; } +.icon-wrapper { + padding-top: 15px; +} + .workspace-selector-wrapper, .workspace-switcher { display: flex; } diff --git a/server/www/scripts/commons/controllers/headerCtrl.js b/server/www/scripts/commons/controllers/headerCtrl.js index 05cd681a7f0..50294ee957e 100644 --- a/server/www/scripts/commons/controllers/headerCtrl.js +++ b/server/www/scripts/commons/controllers/headerCtrl.js @@ -7,12 +7,11 @@ angular.module('faradayApp') ['$scope', '$routeParams', '$location', 'dashboardSrv', 'workspacesFact', 'vulnsManager', function($scope, $routeParams, $location, dashboardSrv, workspacesFact, vulnsManager) { - - $scope.showHeader = function() { - var noNav = ["", "home", "login", "index"]; - return noNav.indexOf($scope.component) < 0; + $scope.showSwitcher = function() { + var noSwitcher = ["", "home", "login", "index", "vulndb", "credentials", "workspaces", "users", "licenses"]; + return noSwitcher.indexOf($scope.component) < 0; }; - + $scope.getVulnsNum = function() { return vulnsManager.getVulnsNum(); }; diff --git a/server/www/scripts/commons/partials/header.html b/server/www/scripts/commons/partials/header.html index 56d5b188308..f2a8db4be6d 100644 --- a/server/www/scripts/commons/partials/header.html +++ b/server/www/scripts/commons/partials/header.html @@ -1,5 +1,5 @@ -
    -
    +
    +
    @@ -29,6 +32,17 @@
    +
    +
    + Dashboard + Status Report + Hosts + Credentials + Executive Report + Tasks + Vulns +
    +

    - - - - + + + + +
    - GOS + HOSTNAMES From e2cc7160850f0dc9e61b7132f39d5d67bad241bb Mon Sep 17 00:00:00 2001 From: Nicolas Ronchi Cesarini Date: Fri, 1 Jun 2018 17:26:05 -0300 Subject: [PATCH 1225/1506] alignment on select all --- server/www/scripts/statusReport/styles/status.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/statusReport/styles/status.css b/server/www/scripts/statusReport/styles/status.css index eb3b6f78d02..9694a701299 100644 --- a/server/www/scripts/statusReport/styles/status.css +++ b/server/www/scripts/statusReport/styles/status.css @@ -79,7 +79,7 @@ .hosts-list .ui-grid-header .hosts-list-checkall{ position: relative; top: 3px; - left: 7px; + text-align: center; } .status-report-grid .ui-grid-pager-panel { From d47c8b9f3caef50f7d9e54c7d894f4f8ef23f75b Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 1 Jun 2018 19:20:48 -0300 Subject: [PATCH 1226/1506] Current user saved in Command object when you upload a report in webUi --- managers/reports_managers.py | 10 ++++----- plugins/controller.py | 23 +++++++++++++------ server/api/modules/upload_reports.py | 33 +++++++++++----------------- 3 files changed, 34 insertions(+), 32 deletions(-) diff --git a/managers/reports_managers.py b/managers/reports_managers.py index 28a320d22e3..fb7b082b46d 100755 --- a/managers/reports_managers.py +++ b/managers/reports_managers.py @@ -31,7 +31,7 @@ def __init__(self, plugin_controller, ws_name=None): self.plugin_controller = plugin_controller self.ws_name = ws_name - def processReport(self, filename): + def processReport(self, filename, user=""): """ Process one Report """ @@ -44,13 +44,13 @@ def processReport(self, filename): 'Plugin not found: automatic and manual try!') return False - return self.sendReport(parser.report_type, filename) + return self.sendReport(parser.report_type, filename, user=user) - def sendReport(self, plugin_id, filename): + def sendReport(self, plugin_id, filename, user=""): """Sends a report to the appropiate plugin specified by plugin_id""" getLogger(self).info( 'The file is %s, %s' % (filename, plugin_id)) - if not self.plugin_controller.processReport(plugin_id, filename, self.ws_name): + if not self.plugin_controller.processReport(plugin_id, filename, self.ws_name, user=user): getLogger(self).error( "Faraday doesn't have a plugin for this tool..." " Processing: ABORT") @@ -343,7 +343,7 @@ def rType(self, tag, output): elif "netsparker" == tag: return "Netsparker" elif "netsparker-cloud" == tag: - return "NetsparkerCloud" + return "NetsparkerCloud" elif "maltego" == tag: return "Maltego" elif "lynis" == tag: diff --git a/plugins/controller.py b/plugins/controller.py index 8efd9cd8408..33238aba61e 100644 --- a/plugins/controller.py +++ b/plugins/controller.py @@ -268,15 +268,24 @@ def onCommandFinished(self, pid, exit_code, term_output): del self._active_plugins[pid] return True - def processReport(self, plugin, filepath, ws_name=None): + def processReport(self, plugin, filepath, ws_name=None, user=""): if not ws_name: ws_name = model.api.getActiveWorkspace().name - cmd_info = CommandRunInformation( - **{'workspace': ws_name, - 'itime': time.time(), - 'import_source': 'report', - 'command': plugin, - 'params': filepath}) + if user: + cmd_info = CommandRunInformation( + **{'workspace': ws_name, + 'itime': time.time(), + 'import_source': 'report', + 'command': plugin, + 'params': filepath, + 'user': user}) + else: + cmd_info = CommandRunInformation( + **{'workspace': ws_name, + 'itime': time.time(), + 'import_source': 'report', + 'command': plugin, + 'params': filepath}) self._mapper_manager.createMappers(ws_name) cmd_info.setID(self._mapper_manager.save(cmd_info)) diff --git a/server/api/modules/upload_reports.py b/server/api/modules/upload_reports.py index 32ba00a539e..59e22481598 100644 --- a/server/api/modules/upload_reports.py +++ b/server/api/modules/upload_reports.py @@ -1,4 +1,4 @@ -# Faraday Penetration Test IDE + # Faraday Penetration Test IDE # Copyright (C) 2018 Infobyte LLC (http://www.infobytesec.com/) # See the file 'doc/LICENSE' for the license information @@ -12,20 +12,21 @@ import logging import model.api -from flask import request, abort, jsonify, Blueprint +from flask import request, abort, jsonify, Blueprint, session from werkzeug.utils import secure_filename from server.utils.logger import get_logger from server.utils.web import gzipped from model.controller import ModelController -from model.cli_app import CliApp from plugins.controller import PluginController from plugins.manager import PluginManager -from managers.workspace_manager import WorkspaceManager from managers.mapper_manager import MapperManager +from managers.reports_managers import ReportProcessor + +from server.models import User from config.configuration import getInstanceConfiguration CONF = getInstanceConfiguration() @@ -39,16 +40,12 @@ class RawReportProcessor(Thread): def __init__(self): super(RawReportProcessor, self).__init__() - plugin_manager = PluginManager(os.path.join(CONF.getConfigPath(), "plugins")) - mappers_manager = MapperManager() self.pending_actions = Queue() + plugin_manager = PluginManager(os.path.join(CONF.getConfigPath(), "plugins")) + mappers_manager = MapperManager() model_controller = ModelController(mappers_manager, self.pending_actions) - - workspace_manager = WorkspaceManager(mappers_manager) - CONF.setMergeStrategy("new") - model.api.setUpAPIs(model_controller, workspace_manager, 0, 0) model_controller.start() plugin_controller = PluginController( @@ -57,22 +54,18 @@ def __init__(self): mappers_manager, self.pending_actions) - self.cli_import = CliApp(workspace_manager, plugin_controller) + self.processor = ReportProcessor(plugin_controller, None) def run(self): while True: try: - workspace, file_path = UPLOAD_REPORTS_QUEUE.get() + workspace, file_path, username = UPLOAD_REPORTS_QUEUE.get() logger.info('Processing raw report {0}'.format(file_path)) - class Arg(): - def __init__(self): - self.workspace = workspace - self.filename = file_path - - self.cli_import.run(Arg()) + self.processor.ws_name = workspace + self.processor.processReport(file_path, user=username) except KeyboardInterrupt: break @@ -83,7 +76,6 @@ def file_upload(workspace=None): """ Upload a report file to Server and process that report with Faraday client plugins. """ - get_logger(__name__).debug("Importing new plugin report in server...") if 'file' not in request.files: @@ -107,6 +99,7 @@ def file_upload(workspace=None): with open(file_path, 'w') as output: output.write(report_file.read()) - UPLOAD_REPORTS_QUEUE.put((workspace, file_path)) + user = User.query.filter_by(id=session["user_id"]).first() + UPLOAD_REPORTS_QUEUE.put((workspace, file_path, user.username)) return jsonify({"status": "processing"}) From a15c4815c24e676526c8602073b85fcc801558c4 Mon Sep 17 00:00:00 2001 From: Nicolas Ronchi Cesarini Date: Mon, 4 Jun 2018 10:50:59 -0300 Subject: [PATCH 1227/1506] fixes to statusreport --- .../statusReport/controllers/statusReport.js | 3 ++ .../partials/ui-grid/columns/namecolumn.html | 3 +- .../ui-grid/columns/servicecolumn.html | 7 ++- .../ui-grid/columns/severitycolumn.html | 2 +- .../ui-grid/columns/statuscolumn.html | 50 +++---------------- 5 files changed, 18 insertions(+), 47 deletions(-) diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 3bdee68cf42..c4c52d577ef 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -313,6 +313,8 @@ angular.module("faradayApp") cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/servicecolumn.html', headerCellTemplate: header, visible: $scope.columns["service"], + field: "service.summary", + displayName : "service", sort: getColumnSort('service'), }); $scope.gridOptions.columnDefs.push({ name : 'hostnames', @@ -383,6 +385,7 @@ angular.module("faradayApp") $scope.gridOptions.columnDefs.push({ name : 'status', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/statuscolumn.html', headerCellTemplate: header, + field: "status", sort: getColumnSort('status'), visible: $scope.columns["status"] }); diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/namecolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/namecolumn.html index 18516fdc150..2213902cc5f 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/namecolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/namecolumn.html @@ -1,4 +1,5 @@
    {{COL_FIELD CUSTOM_FILTERS}}
    \ No newline at end of file diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/servicecolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/servicecolumn.html index 452abf19aa6..1ec3afa5484 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/servicecolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/servicecolumn.html @@ -1 +1,6 @@ -
    {{COL_FIELD ? COL_FIELD.summary : 'EMPTY'}}
    +
    +
    + {{COL_FIELD}} +
    +
    +
    {{COL_FIELD}}
    diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html index 7fc5bc2a9d1..9328ee760c4 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/severitycolumn.html @@ -6,6 +6,6 @@ -
    +
    {{COL_FIELD | uppercase}}
    \ No newline at end of file diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html index 06c3bcee67f..9b4ba5fee07 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html @@ -1,50 +1,12 @@
    - + + {{COL_FIELD | uppercase}} +
    - + + {{COL_FIELD | uppercase}} +
    \ No newline at end of file From 6381649b8833a21bd3734f8e6c863f1b01916185 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 4 Jun 2018 15:46:49 -0300 Subject: [PATCH 1228/1506] [FIX] add authentication to import csv --- bin/fplugin | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/bin/fplugin b/bin/fplugin index 7c37661b25d..cbdc9f92d78 100755 --- a/bin/fplugin +++ b/bin/fplugin @@ -26,6 +26,7 @@ from colorama import Fore from config.configuration import getInstanceConfiguration from managers.mapper_manager import MapperManager from model.controller import ModelController +from persistence.server.server import login_user CONF = getInstanceConfiguration() @@ -48,7 +49,14 @@ def signal_handler(signal, frame): os._exit(0) -def dispatch(args, unknown, help): +def dispatch(args, unknown, user_help, username, password): + session_cookie = login_user(args.url, username, password) + if not session_cookie: + raise UserWarning('Invalid credentials!') + else: + CONF.setDBUser(username) + CONF.setDBSessionCookies(session_cookie) + if '--' in unknown: unknown.remove('--') @@ -58,7 +66,7 @@ def dispatch(args, unknown, help): model_controller = ModelController(mappers_manager, pending_actions) if not args.command: - print help + print(user_help) if not args.interactive: sys.exit(1) @@ -79,6 +87,8 @@ def dispatch(args, unknown, help): module_fplugin = imp.load_source('module_fplugin', plugin_path) module_fplugin.models.server.FARADAY_UP = False module_fplugin.models.server.SERVER_URL = args.url + module_fplugin.models.server.AUTH_USER = username + module_fplugin.models.server.AUTH_PASS = password call_main = getattr(module_fplugin, 'main', None) @@ -167,11 +177,19 @@ if __name__ == '__main__': help='Faraday Server URL. Example: http://localhost:5985', default='http://localhost:5985') + parser.add_argument( + '--username', + required=True) + + parser.add_argument( + '--password', + required=True) + # Only parse known args. Unknown ones will be passed on the the called script args, unknown = parser.parse_known_args() if not args.interactive: - dispatch(args, unknown, parser.format_help()) + dispatch(args, unknown, parser.format_help(), args.username, args.password) else: # print "Loading command history..." @@ -215,7 +233,7 @@ if __name__ == '__main__': parsed_args, new_unknown = parser.parse_known_args(new_args) parsed_args.interactive = True - last_id = dispatch(parsed_args, new_unknown, parser.format_help()) or last_id + last_id = dispatch(parsed_args, new_unknown, parser.format_help(), args.username, args.password) or last_id # print '$last = %s' % last_id except (EOFError, KeyboardInterrupt): print 'Bye Bye!' From 07f959403b380bbc05c9e5fc9693a183217dfedf Mon Sep 17 00:00:00 2001 From: Nicolas Ronchi Cesarini Date: Mon, 4 Jun 2018 16:28:21 -0300 Subject: [PATCH 1229/1506] navigation active opacity when changing active url --- server/www/estilos.css | 1 + .../navigation/controllers/navigationCtrl.js | 2 +- .../scripts/navigation/partials/leftBar.html | 20 +++++++++---------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/server/www/estilos.css b/server/www/estilos.css index 5a006d6b869..ea9b2c3e230 100644 --- a/server/www/estilos.css +++ b/server/www/estilos.css @@ -126,6 +126,7 @@ header.head { aside nav a.active { background-color: #d9d9d9; + opacity: 1; } .left-nav ul{ diff --git a/server/www/scripts/navigation/controllers/navigationCtrl.js b/server/www/scripts/navigation/controllers/navigationCtrl.js index da945b23f79..3453885f859 100644 --- a/server/www/scripts/navigation/controllers/navigationCtrl.js +++ b/server/www/scripts/navigation/controllers/navigationCtrl.js @@ -6,7 +6,7 @@ angular.module('faradayApp') .controller('navigationCtrl', ['$scope', '$http', '$route', '$routeParams', '$cookies', '$location', '$interval', '$uibModal', 'configSrv', 'workspacesFact', 'Notification', function($scope, $http, $route, $routeParams, $cookies, $location, $interval, $uibModal, configSrv, workspacesFact, Notification) { - $scope.workspace = "asd"; + $scope.workspace = ""; $scope.component = ""; var componentsNeedsWS = ["dashboard","status","hosts"]; diff --git a/server/www/scripts/navigation/partials/leftBar.html b/server/www/scripts/navigation/partials/leftBar.html index 3e38ec2e35f..02f09f68e45 100644 --- a/server/www/scripts/navigation/partials/leftBar.html +++ b/server/www/scripts/navigation/partials/leftBar.html @@ -10,52 +10,52 @@
    @@ -41,6 +42,7 @@ Executive Report Tasks Vulns +

    Licenses

    diff --git a/server/www/scripts/licenses/partials/list.html b/server/www/scripts/licenses/partials/list.html index fd4e013cd74..43bf9d67b3f 100644 --- a/server/www/scripts/licenses/partials/list.html +++ b/server/www/scripts/licenses/partials/list.html @@ -4,10 +4,8 @@
    +
    -

    - Licenses ({{licenses.length}}) -

    From 44f65eb76f2b015dfd825dfae7a0e592b2e98d2a Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Wed, 6 Jun 2018 08:35:09 -0300 Subject: [PATCH 1239/1506] 4814 - Add missing 'New' button on vulndb --- server/www/scripts/vulndb/partials/vulndb.html | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/server/www/scripts/vulndb/partials/vulndb.html b/server/www/scripts/vulndb/partials/vulndb.html index 06d94c8e6d1..50fa79cb4d2 100644 --- a/server/www/scripts/vulndb/partials/vulndb.html +++ b/server/www/scripts/vulndb/partials/vulndb.html @@ -6,11 +6,11 @@
    -
    +
    -
    @@ -36,6 +36,12 @@
    +
    +
    + +
    From 858a63fefdc30e19bd49ea404fb382008ad71d28 Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Wed, 6 Jun 2018 09:12:22 -0300 Subject: [PATCH 1240/1506] 4803 - Fix host edit/details margin --- server/www/scripts/services/partials/list.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/services/partials/list.html b/server/www/scripts/services/partials/list.html index 75666d3f9ed..465c23e9b1b 100644 --- a/server/www/scripts/services/partials/list.html +++ b/server/www/scripts/services/partials/list.html @@ -21,7 +21,7 @@

    - +

    Host services

    @@ -43,6 +44,7 @@ Tasks Vulns

    Licenses

    +

    Workspaces

    diff --git a/server/www/scripts/statusReport/styles/status.css b/server/www/scripts/statusReport/styles/status.css index 9694a701299..7f7ac41dd1c 100644 --- a/server/www/scripts/statusReport/styles/status.css +++ b/server/www/scripts/statusReport/styles/status.css @@ -86,6 +86,7 @@ background-color: #f1f1f1; } + .status-report-grid .status.opened { background-color: rgba(223, 57, 54, .4) !important; text-align: center; @@ -270,3 +271,14 @@ button.btn-new { margin-right: 16px; } + +.status-report-grid .ui-grid-row-selected .ui-grid-cell .status { + color: white !important; + background-color:transparent !important; +} + + +.status-report-grid .ui-grid-row-selected .ui-grid-cell .status span { + color: white !important; + background-color:transparent !important; +} \ No newline at end of file diff --git a/server/www/scripts/workspaces/partials/list.html b/server/www/scripts/workspaces/partials/list.html index b1edcec9309..1a14b41b82f 100644 --- a/server/www/scripts/workspaces/partials/list.html +++ b/server/www/scripts/workspaces/partials/list.html @@ -4,42 +4,11 @@
    -
    -

    -
    -
    -
    - Workspaces -
    - -
    -
    -

    +
    + +
    +
    +
    From c08602f8a0d871ba08a2a91d08f8a04b700b44cc Mon Sep 17 00:00:00 2001 From: Nicolas Ronchi Cesarini Date: Wed, 6 Jun 2018 12:37:58 -0300 Subject: [PATCH 1243/1506] add field last modified --- server/www/scripts/workspaces/partials/list.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/www/scripts/workspaces/partials/list.html b/server/www/scripts/workspaces/partials/list.html index 1a14b41b82f..3b6e92eb011 100644 --- a/server/www/scripts/workspaces/partials/list.html +++ b/server/www/scripts/workspaces/partials/list.html @@ -48,6 +48,8 @@
    + + @@ -63,6 +65,7 @@ + From baa030ba590ba89427392601f6f03086a82be4d3 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 6 Jun 2018 12:43:02 -0300 Subject: [PATCH 1244/1506] [FIX] mock default SERVER_URL value --- persistence/server/server.py | 5 ++--- test_cases/test_persistence_server_models.py | 6 ++++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/persistence/server/server.py b/persistence/server/server.py index dbdfdae66e1..75bed0230c6 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -27,7 +27,7 @@ import os import json import logging -import ConfigParser +from ConfigParser import SafeConfigParser try: import urlparse @@ -77,7 +77,6 @@ def _conf(): - from config.configuration import getInstanceConfiguration CONF = getInstanceConfiguration() @@ -94,7 +93,7 @@ def _conf(): def _get_base_server_url(): if FARADAY_UPLOAD_REPORTS_WEB_COOKIE: - parser = ConfigParser.SafeConfigParser() + parser = SafeConfigParser() parser.read(LOCAL_CONFIG_FILE) server_url = 'http://{0}:{1}'.format( parser.get('faraday_server', 'bind_address'), diff --git a/test_cases/test_persistence_server_models.py b/test_cases/test_persistence_server_models.py index c1ecdf65a8c..0a625fa6a57 100644 --- a/test_cases/test_persistence_server_models.py +++ b/test_cases/test_persistence_server_models.py @@ -23,11 +23,12 @@ class TestVulnPersistanceModelsFuncions(GenericAPITest): @responses.activate @patch('config.configuration.getInstanceConfiguration') + @patch('persistence.server.server.SERVER_URL', 'http://localhost:5985') def test_persistence_server_update_vuln(self, getInstanceConfigurationMock): fo = self.first_object conf_mock = Mock() getInstanceConfigurationMock.return_value = conf_mock - port = 5984 + port = 5985 conf_mock.getDBSessionCookies.return_value = None conf_mock.getAPIUrl.return_value = 'http://localhost:{0}'.format(port) conf_mock.getServerURI.return_value = 'http://localhost:{0}'.format(port) @@ -97,12 +98,13 @@ class TestVulnWebPersistanceModelsFuncions(GenericAPITest): @responses.activate @patch('config.configuration.getInstanceConfiguration') + @patch('persistence.server.server.SERVER_URL', 'http://localhost:5985') def test_persistence_server_update_vuln_web(self, getInstanceConfigurationMock): fo = self.first_object conf_mock = Mock() getInstanceConfigurationMock.return_value = conf_mock - port = 5984 + port = 5985 conf_mock.getDBSessionCookies.return_value = None conf_mock.getAPIUrl.return_value = 'http://localhost:{0}'.format(port) conf_mock.getServerURI.return_value = 'http://localhost:{0}'.format(port) From 384a4385d660991032b22f7450e30b1ca9a7f7b9 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 6 Jun 2018 12:56:10 -0300 Subject: [PATCH 1245/1506] [ADD] add create_date and update_date --- server/api/modules/workspaces.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/api/modules/workspaces.py b/server/api/modules/workspaces.py index 884b33f5647..9149ff3a34b 100644 --- a/server/api/modules/workspaces.py +++ b/server/api/modules/workspaces.py @@ -63,10 +63,18 @@ class WorkspaceSchema(AutoSchema): fields.List(fields.String) ) + create_date = fields.DateTime(attribute='create_date', + dump_only=True) + + update_date = fields.DateTime(attribute='update_date', + dump_only=True) + + class Meta: model = Workspace fields = ('_id', 'id', 'customer', 'description', 'active', - 'duration', 'name', 'public', 'scope', 'stats') + 'duration', 'name', 'public', 'scope', 'stats', + 'create_date', 'update_date') @post_load def post_load_duration(self, data): From afdbfedab3943b11f5bc9b7c328989be2febf1fb Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Wed, 6 Jun 2018 15:04:08 -0300 Subject: [PATCH 1246/1506] 4810 - Replace new host modal for new host view --- server/www/dashboard-v3.css | 8 ++++++++ server/www/scripts/hosts/controllers/hosts.js | 15 ++------------- server/www/scripts/hosts/partials/new.html | 5 ++--- server/www/scripts/services/partials/list.html | 2 +- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/server/www/dashboard-v3.css b/server/www/dashboard-v3.css index 2d33cd9623e..5f5bfa511be 100644 --- a/server/www/dashboard-v3.css +++ b/server/www/dashboard-v3.css @@ -78,10 +78,18 @@ padding-left: 5px; } +.pl { + padding-left: 15px; +} + .pr-sm { padding-right: 5px; } +.pr { + padding-right: 15px !important; +} + .progress-bar-tall { height: 57px; } diff --git a/server/www/scripts/hosts/controllers/hosts.js b/server/www/scripts/hosts/controllers/hosts.js index f016354bc4d..1291458e69f 100644 --- a/server/www/scripts/hosts/controllers/hosts.js +++ b/server/www/scripts/hosts/controllers/hosts.js @@ -84,7 +84,7 @@ angular.module('faradayApp') }; var createCredential = function(credentialData, parent_id){ - + // Add parent id, create credential and save to server. try { var credentialObj = new credential(credentialData, parent_id); @@ -224,18 +224,7 @@ angular.module('faradayApp') } $scope.new = function() { - var modal = $uibModal.open({ - templateUrl: 'scripts/hosts/partials/modalNew.html', - controller: 'hostsModalNew', - size: 'lg', - resolve: {} - }); - - modal.result.then(function(data) { - var hostdata = data[0]; - var credentialData = data[1]; - $scope.insert(hostdata, credentialData); - }); + $location.path('/host/ws/' + $scope.workspace + '/new'); }; $scope.update = function(host, hostdata) { diff --git a/server/www/scripts/hosts/partials/new.html b/server/www/scripts/hosts/partials/new.html index 4ebd944360f..7e9263d94e0 100644 --- a/server/www/scripts/hosts/partials/new.html +++ b/server/www/scripts/hosts/partials/new.html @@ -19,7 +19,7 @@

    -
    +

    Host details @@ -85,7 +85,7 @@

    Description
    -
    +

    Hostnames @@ -109,4 +109,3 @@

    - diff --git a/server/www/scripts/services/partials/list.html b/server/www/scripts/services/partials/list.html index 465c23e9b1b..93de17c92bb 100644 --- a/server/www/scripts/services/partials/list.html +++ b/server/www/scripts/services/partials/list.html @@ -21,7 +21,7 @@

    - +

    Host services

    - -
    -

    Import from workspace

    - -
    -

    Select workspace from where to import

    - -
    - From 68942df16cf26f4b1eeefca759f62b1836453424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 6 Jun 2018 16:06:09 -0300 Subject: [PATCH 1249/1506] Add CSRF protection to upload reports API endpoint --- server/api/modules/session.py | 5 ++++- server/api/modules/upload_reports.py | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/server/api/modules/session.py b/server/api/modules/session.py index a8473b03570..cd41c86c8a6 100644 --- a/server/api/modules/session.py +++ b/server/api/modules/session.py @@ -5,10 +5,13 @@ ''' from flask import jsonify, session, Blueprint, current_app +from flask_wtf.csrf import generate_csrf session_api = Blueprint('session_api', __name__) @session_api.route('/session') def session_info(): user = current_app.user_datastore.get_user(session['user_id']) - return jsonify(user.get_security_payload()) + data = user.get_security_payload() + data['csrf_token'] = generate_csrf() + return jsonify(data) diff --git a/server/api/modules/upload_reports.py b/server/api/modules/upload_reports.py index ea1a5f3dbb2..e302dba45e5 100644 --- a/server/api/modules/upload_reports.py +++ b/server/api/modules/upload_reports.py @@ -14,7 +14,9 @@ import model.api from flask import request, abort, jsonify, Blueprint, session, make_response +from flask_wtf.csrf import validate_csrf from werkzeug.utils import secure_filename +from wtforms import ValidationError from server.utils.logger import get_logger from server.utils.web import gzipped @@ -97,6 +99,11 @@ def file_upload(workspace=None): if 'file' not in request.files: abort(400) + try: + validate_csrf(request.form.get('csrf_token')) + except ValidationError: + abort(403) + report_file = request.files['file'] if report_file.filename == '': From c6e7e78e923c3617fb7b578f73abd30b0575374a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 6 Jun 2018 16:06:38 -0300 Subject: [PATCH 1250/1506] Always create the uploaded reports directory Before it was created only on the first installation --- server/config.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/server/config.py b/server/config.py index 6233f09a52e..13ac2b0bb5c 100644 --- a/server/config.py +++ b/server/config.py @@ -32,6 +32,14 @@ CONFIG_FILES = [DEFAULT_CONFIG_FILE, LOCAL_CONFIG_FILE] WS_BLACKLIST = CONSTANTS.CONST_BLACKDBS +if not os.path.exists(LOCAL_REPORTS_FOLDER): + try: + os.makedirs(LOCAL_REPORTS_FOLDER) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + def copy_default_config_to_local(): @@ -48,13 +56,6 @@ def copy_default_config_to_local(): # Copy default config file into faraday local config shutil.copyfile(DEFAULT_CONFIG_FILE, LOCAL_CONFIG_FILE) - if not os.path.exists(LOCAL_REPORTS_FOLDER): - try: - os.makedirs(LOCAL_REPORTS_FOLDER) - except OSError as e: - if e.errno != errno.EEXIST: - raise - from server.utils.logger import get_logger get_logger(__name__).info(u"Local faraday-server configuration created at {}".format(LOCAL_CONFIG_FILE)) From caef7125c0eb2f36a7c8613b378fe62656d9cdfa Mon Sep 17 00:00:00 2001 From: Guillermo Garcia Date: Wed, 6 Jun 2018 16:29:44 -0300 Subject: [PATCH 1251/1506] 4805 - Add dashboard confirmed button --- server/www/dashboard-v3.css | 5 ++ .../scripts/commons/controllers/headerCtrl.js | 13 ++- .../www/scripts/commons/partials/header.html | 9 +- .../dashboard/controllers/dashboard.js | 6 +- .../scripts/dashboard/partials/dashboard.html | 83 ++++++++++--------- 5 files changed, 65 insertions(+), 51 deletions(-) diff --git a/server/www/dashboard-v3.css b/server/www/dashboard-v3.css index 5f5bfa511be..cbc3990824f 100644 --- a/server/www/dashboard-v3.css +++ b/server/www/dashboard-v3.css @@ -1,3 +1,8 @@ +.dashboard .filter-wrapper .confirm-button { + position: relative; + bottom: 4px; +} + .dashboard .faraday-page-header { margin-bottom: 0px; } diff --git a/server/www/scripts/commons/controllers/headerCtrl.js b/server/www/scripts/commons/controllers/headerCtrl.js index 50294ee957e..2167d9786b9 100644 --- a/server/www/scripts/commons/controllers/headerCtrl.js +++ b/server/www/scripts/commons/controllers/headerCtrl.js @@ -4,18 +4,25 @@ angular.module('faradayApp') .controller('headerCtrl', - ['$scope', '$routeParams', '$location', 'dashboardSrv', 'workspacesFact', 'vulnsManager', - function($scope, $routeParams, $location, dashboardSrv, workspacesFact, vulnsManager) { + ['$scope', '$routeParams', '$location', '$cookies', 'dashboardSrv', 'workspacesFact', 'vulnsManager', + function($scope, $routeParams, $location, $cookies, dashboardSrv, workspacesFact, vulnsManager) { + $scope.confirmed = ($cookies.get('confirmed') == undefined) ? false : JSON.parse($cookies.get('confirmed')); $scope.showSwitcher = function() { var noSwitcher = ["", "home", "login", "index", "vulndb", "credentials", "workspaces", "users", "licenses"]; return noSwitcher.indexOf($scope.component) < 0; }; - + $scope.getVulnsNum = function() { return vulnsManager.getVulnsNum(); }; + $scope.toggleConfirmed = function() { + $scope.confirmed = !$scope.confirmed; + dashboardSrv.setConfirmed($scope.confirmed); + dashboardSrv.updateData(); + }; + init = function(name) { $scope.location = $location.path().split('/')[1]; $scope.workspace = $routeParams.wsId; diff --git a/server/www/scripts/commons/partials/header.html b/server/www/scripts/commons/partials/header.html index 90f71c667db..30de3d0dde9 100644 --- a/server/www/scripts/commons/partials/header.html +++ b/server/www/scripts/commons/partials/header.html @@ -1,7 +1,7 @@
    -
    +
    + + +
    @@ -78,4 +83,4 @@
    -
    \ No newline at end of file +
    diff --git a/server/www/scripts/dashboard/controllers/dashboard.js b/server/www/scripts/dashboard/controllers/dashboard.js index d3498a45746..5be2303c770 100644 --- a/server/www/scripts/dashboard/controllers/dashboard.js +++ b/server/www/scripts/dashboard/controllers/dashboard.js @@ -16,7 +16,7 @@ angular.module('faradayApp') workspacesFact.list().then(function(wss) { $scope.workspaces = wss; }); - + dashboardSrv.setConfirmedFromCookie(); dashboardSrv.startTimer(); }; @@ -25,10 +25,6 @@ angular.module('faradayApp') $location.path(route); }; - $scope.toggleConfirmed = function() { - dashboardSrv.setConfirmed(); - }; - $scope.$on('$destroy', function(){ dashboardSrv.stopTimer(); }) diff --git a/server/www/scripts/dashboard/partials/dashboard.html b/server/www/scripts/dashboard/partials/dashboard.html index c6225102343..9568eddda52 100644 --- a/server/www/scripts/dashboard/partials/dashboard.html +++ b/server/www/scripts/dashboard/partials/dashboard.html @@ -5,53 +5,54 @@
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +
    + +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    -
    -
    -
    -
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +
    +
    +
    -
    +
    From 9262ff04eac5b65b0643e8d12574505ab40f0015 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 6 Jun 2018 16:31:30 -0300 Subject: [PATCH 1252/1506] Redirect to dashboard on successful report imports --- server/api/modules/upload_reports.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/server/api/modules/upload_reports.py b/server/api/modules/upload_reports.py index e302dba45e5..1f87cfdf861 100644 --- a/server/api/modules/upload_reports.py +++ b/server/api/modules/upload_reports.py @@ -13,7 +13,15 @@ import logging import model.api -from flask import request, abort, jsonify, Blueprint, session, make_response +from flask import ( + redirect, + request, + abort, + jsonify, + Blueprint, + session, + make_response +) from flask_wtf.csrf import validate_csrf from werkzeug.utils import secure_filename from wtforms import ValidationError @@ -125,5 +133,6 @@ def file_upload(workspace=None): UPLOAD_REPORTS_QUEUE.put((workspace, file_path, request.cookies)) command_id = UPLOAD_REPORTS_CMD_QUEUE.get() if command_id: - return jsonify({"status": "processing", "command_id": command_id}) + # return jsonify({"status": "processing", "command_id": command_id}) + return redirect('/#/dashboard/ws/' + workspace) abort(make_response(jsonify(message="Invalid file."), 400)) From 7b7a8973366bbaa5a4515625c4f48b89fcc62a7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 6 Jun 2018 16:33:22 -0300 Subject: [PATCH 1253/1506] Add release information of #4751 --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index d1925242e10..4daaffdc456 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -32,6 +32,7 @@ TBA: * Add new screenshot fplugin which takes a screenshot of the ip:ports of a given protocol * Add fix for net sparker regular and cloud fix on severity * Removed Chat feature (data is kept inside notes) +* Plugin reports now can be imported in the server, from the Web UI November 17, 2017: --- From 0783cfa1afaa3c2dc85f13a1de2a5b7b132c590e Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 6 Jun 2018 19:39:20 -0300 Subject: [PATCH 1254/1506] Remove duplicated item, broke API --- data/cwe.csv | 1860 +++++++++++++++++++++++----------------------- data/cwe_all.csv | 1860 +++++++++++++++++++++++----------------------- 2 files changed, 1858 insertions(+), 1862 deletions(-) diff --git a/data/cwe.csv b/data/cwe.csv index 7941463b896..672a83382d8 100644 --- a/data/cwe.csv +++ b/data/cwe.csv @@ -1,4 +1,4 @@ -cwe,name,description,resolution,exploitation,references +cwe,name,description,resolution,exploitation,references CWE-119,EN-Improper Restriction of Operations within the Bounds of a Memory Buffer (Type: Class),"The software performs operations on a memory buffer, but it can read from or write to a memory location that is outside of the intended boundary of the buffer. Certain languages allow direct addressing of memory locations and do not automatically ensure that these locations are valid for the memory buffer that is being referenced. This can cause read or write operations to be performed on memory locations that may be associated with other variables, data structures, or internal program data. As a result, an attacker may be able to execute arbitrary code, alter the intended control flow, read sensitive information, or cause the system to crash.",,high,"Writing Secure Code: Chapter 5, ""Public Enemy #1: The Buffer Overrun"" Page 127; Chapter 14, ""Prevent I18N Buffer Overruns"" Page 441 @@ -9,9 +9,9 @@ Limiting buffer overflows with ExecShield: http://www.redhat.com/magazine/009jul PaX: http://en.wikipedia.org/wiki/PaX Understanding DEP as a mitigation technology part 1: http://blogs.technet.com/b/srd/archive/2009/06/12/understanding-dep-as-a-mitigation-technology-part-1.aspx The Art of Software Security Assessment: Chapter 5, ""Memory Corruption"", Page 167. -The Art of Software Security Assessment: Chapter 5, ""Protection Mechanisms"", Page 189." +The Art of Software Security Assessment: Chapter 5, ""Protection Mechanisms"", Page 189." CWE-123,EN-Write-what-where Condition (Type: Base),"Any condition where the attacker has the ability to write an arbitrary value to an arbitrary location, often as the result of a buffer overflow. -A buffer overflow condition exists when a program attempts to put more data in a buffer than it can hold, or when a program attempts to put data in a memory area outside of the boundaries of a buffer. The simplest type of error, and the most common cause of buffer overflows, is the ""classic"" case in which the program copies the buffer without restricting how much is copied. Other variants exist, but the existence of a classic overflow strongly suggests that the programmer is not considering even the most basic of security protections.",,high,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89" +A buffer overflow condition exists when a program attempts to put more data in a buffer than it can hold, or when a program attempts to put data in a memory area outside of the boundaries of a buffer. The simplest type of error, and the most common cause of buffer overflows, is the ""classic"" case in which the program copies the buffer without restricting how much is copied. Other variants exist, but the existence of a classic overflow strongly suggests that the programmer is not considering even the most basic of security protections.",,high,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89" CWE-129,EN-Improper Validation of Array Index (Type: Base),"The product uses untrusted input when calculating or using an array index, but the product does not validate or incorrectly validates the index to ensure the index references a valid position within the array. This typically occurs when the pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. This may result in exposure of sensitive information or possibly a crash.",,high,"Writing Secure Code: Chapter 5, ""Array Indexing Errors"" Page 144 Top 25 Series - Rank 14 - Improper Validation of Array Index: http://blogs.sans.org/appsecstreetfighter/2010/03/12/top-25-series-rank-14-improper-validation-of-array-index/ @@ -19,22 +19,22 @@ Address Space Layout Randomization in Windows Vista: http://blogs.msdn.com/micha PaX: http://en.wikipedia.org/wiki/PaX Understanding DEP as a mitigation technology part 1: http://blogs.technet.com/b/srd/archive/2009/06/12/understanding-dep-as-a-mitigation-technology-part-1.aspx Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html -24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89" +24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89" CWE-194,EN-Unexpected Sign Extension (Type: Base),"The software performs an operation on a number that causes it to be sign extended when it is transformed into a larger data type. When the original number is negative, this can produce unexpected values that lead to resultant weaknesses. This can happen in signed and unsigned cases.",,high,"C Language Issues for Application Security: http://www.informit.com/articles/article.aspx?p=686170&seqNum=6 -Integral Security: http://www.ddj.com/security/193501774" +Integral Security: http://www.ddj.com/security/193501774" CWE-20,EN-Improper Input Validation (Type: Class),"The product does not validate or incorrectly validates input that can affect the control flow or data flow of a program. When software does not validate input properly, an attacker is able to craft the input in a form that is not expected by the rest of the application. This will lead to parts of the system receiving unintended input, which may result in altered control flow, arbitrary control of a resource, or arbitrary code execution.",,high,"Input Validation with ESAPI - Very Important: http://manicode.blogspot.com/2008/08/input-validation-with-esapi.html OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI Hacking Exposed Web Applications, Second Edition: Input Validation Attacks Input validation or output filtering, which is better?: http://jeremiahgrossman.blogspot.com/2007/01/input-validation-or-output-filtering.html The importance of input validation: http://searchsoftwarequality.techtarget.com/tip/0,289483,sid92_gci1214373,00.html -Writing Secure Code: Chapter 10, ""All Input Is Evil!"" Page 341" +Writing Secure Code: Chapter 10, ""All Input Is Evil!"" Page 341" CWE-200,EN-Information Exposure (Type: Class),"An information exposure is the intentional or unintentional disclosure of information to an actor that is not explicitly authorized to have access to that information. The information either is regarded as sensitive within the product's own functionality, such as a private message; or provides information about the product or its environment that could be useful in an attack but is normally not available to the attacker, such as the installation path of a product that is remotely accessible. -Many information exposures are resultant (e.g. PHP script error revealing the full path of the program), but they can also be primary (e.g. timing discrepancies in cryptography). There are many different types of problems that involve information exposures. Their severity can range widely depending on the type of information that is revealed.",,high,Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/ +Many information exposures are resultant (e.g. PHP script error revealing the full path of the program), but they can also be primary (e.g. timing discrepancies in cryptography). There are many different types of problems that involve information exposures. Their severity can range widely depending on the type of information that is revealed.",,high,Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/ CWE-209,EN-Information Exposure Through an Error Message (Type: Base),"The software generates an error message that includes sensitive information about its environment, users, or associated data. The sensitive information may be valuable information on its own (such as a password), or it may be useful for launching other, more deadly attacks. If an attack fails, an attacker may use error information provided by the server to launch another more focused attack. For example, an attempt to exploit a path traversal weakness (CWE-22) might yield the full pathname of the installed application. In turn, this could be used to select the proper number of "".."" sequences to navigate to the targeted file. An attack using SQL injection (CWE-89) might not initially succeed, but an error message could reveal the malformed query, which would expose query logic and possibly even passwords or other sensitive information used within the query.",,high,"Information Leakage: http://www.webappsec.org/projects/threat/classes/information_leakage.shtml Secure Programming with Static Analysis: Section 9.2, page 326. @@ -42,19 +42,19 @@ Writing Secure Code: Chapter 16, ""General Good Practices."" Page 415 24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183 24 Deadly Sins of Software Security: ""Sin 12: Information Leakage."" Page 191 Top 25 Series - Rank 16 - Information Exposure Through an Error Message: http://software-security.sans.org/blog/2010/03/17/top-25-series-rank-16-information-exposure-through-an-error-message -The Art of Software Security Assessment: Chapter 3, ""Overly Verbose Error Messages"", Page 75." +The Art of Software Security Assessment: Chapter 3, ""Overly Verbose Error Messages"", Page 75." CWE-234,EN-Failure to Handle Missing Parameter (Type: Variant),"If too few arguments are sent to a function, the function will still pop the expected number of arguments from the stack. Potentially, a variable number of arguments could be exhausted in a function as well. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,high, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,high, CWE-242,EN-Use of Inherently Dangerous Function (Type: Base),"The program calls a function that can never be guaranteed to work safely. Certain functions behave in dangerous ways regardless of how they are used. Functions in this category were often implemented without taking security concerns into account. The gets() function is unsafe because it does not perform bounds checking on the size of its input. An attacker can easily send arbitrarily-sized input to gets() and overflow the destination buffer. Similarly, the >> operator is unsafe to use when reading into a statically-allocated character array because it does not perform bounds checking on the size of its input. An attacker can easily send arbitrarily-sized input to the >> operator and overflow the destination buffer.",,high,"Herb Schildt's C++ Programming Cookbook: Chapter 5. Working with I/O -Writing Secure Code: Chapter 5, ""gets and fgets"" Page 163" +Writing Secure Code: Chapter 5, ""gets and fgets"" Page 163" CWE-243,EN-Creation of chroot Jail Without Changing Working Directory (Type: Variant),"The program uses the chroot() system call to create a jail, but does not change the working directory afterward. This does not prevent access to files outside of the jail. -Improper use of chroot() may allow attackers to escape from the chroot jail. The chroot() function call does not change the process's current working directory, so relative paths may still refer to file system resources outside of the chroot jail after chroot() has been called.",,high, +Improper use of chroot() may allow attackers to escape from the chroot jail. The chroot() function call does not change the process's current working directory, so relative paths may still refer to file system resources outside of the chroot jail after chroot() has been called.",,high, CWE-268,EN-Privilege Chaining (Type: Base),"Two distinct privileges, roles, capabilities, or rights can be combined in a way that allows an entity to perform unsafe actions that would not be allowed without that combination. -Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,high,Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html +Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,high,Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html CWE-271,EN-Privilege Dropping / Lowering Errors (Type: Class),"The software does not drop privileges before passing control of a resource to an actor that does not have those privileges. In some contexts, a system executing with elevated permissions will hand off a process/file/etc. to another process or user. If the privileges of an entity are not reduced, then elevated privileges are spread throughout a system and possibly to an attacker.",,high,"24 Deadly Sins of Software Security: ""Sin 16: Executing Code With Too Much Privilege."" Page 243 -The Art of Software Security Assessment: Chapter 9, ""Dropping Privileges Permanently"", Page 479." +The Art of Software Security Assessment: Chapter 9, ""Dropping Privileges Permanently"", Page 479." CWE-285,EN-Improper Authorization (Type: Class),"The software does not perform or incorrectly performs an authorization check when an actor attempts to access a resource or perform an action. Assuming a user with a given identity, authorization is the process of determining whether that user can access a given resource, based on the user's privileges and any permissions or other access-control specifications that apply to the resource. When access control checks are not applied consistently - or not at all - users are able to access data or perform actions that they should not be allowed to perform. This can lead to a wide range of problems, including information exposures, denial of service, and arbitrary code execution.",,high,"Role Based Access Control and Role Based Security: http://csrc.nist.gov/groups/SNS/rbac/ @@ -63,79 +63,79 @@ Top 25 Series - Rank 5 - Improper Access Control (Authorization): http://blogs.s OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI Authentication using JAAS: http://www.javaranch.com/journal/2008/04/authentication-using-JAAS.html The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Authorization"", Page 39. -The Art of Software Security Assessment: Chapter 11, ""ACL Inheritance"", Page 649." +The Art of Software Security Assessment: Chapter 11, ""ACL Inheritance"", Page 649." CWE-291,EN-Reliance on IP Address for Authentication (Type: Variant),"The software uses an IP address for authentication. -IP addresses can be easily spoofed. Attackers can forge the source IP address of the packets they send, but response packets will return to the forged IP address. To see the response packets, the attacker has to sniff the traffic between the victim machine and the forged IP address. In order to accomplish the required sniffing, attackers typically attempt to locate themselves on the same subnet as the victim machine. Attackers may be able to circumvent this requirement by using source routing, but source routing is disabled across much of the Internet today. In summary, IP address verification can be a useful part of an authentication scheme, but it should not be the single factor required for authentication.",,high, +IP addresses can be easily spoofed. Attackers can forge the source IP address of the packets they send, but response packets will return to the forged IP address. To see the response packets, the attacker has to sniff the traffic between the victim machine and the forged IP address. In order to accomplish the required sniffing, attackers typically attempt to locate themselves on the same subnet as the victim machine. Attackers may be able to circumvent this requirement by using source routing, but source routing is disabled across much of the Internet today. In summary, IP address verification can be a useful part of an authentication scheme, but it should not be the single factor required for authentication.",,high, CWE-292,EN-DEPRECATED (Duplicate): Trusting Self-reported DNS Name (Type: Variant),"This entry has been deprecated because it was a duplicate of CWE-350. All content has been transferred to CWE-350. -IP addresses can be easily spoofed. Attackers can forge the source IP address of the packets they send, but response packets will return to the forged IP address. To see the response packets, the attacker has to sniff the traffic between the victim machine and the forged IP address. In order to accomplish the required sniffing, attackers typically attempt to locate themselves on the same subnet as the victim machine. Attackers may be able to circumvent this requirement by using source routing, but source routing is disabled across much of the Internet today. In summary, IP address verification can be a useful part of an authentication scheme, but it should not be the single factor required for authentication.",,high, +IP addresses can be easily spoofed. Attackers can forge the source IP address of the packets they send, but response packets will return to the forged IP address. To see the response packets, the attacker has to sniff the traffic between the victim machine and the forged IP address. In order to accomplish the required sniffing, attackers typically attempt to locate themselves on the same subnet as the victim machine. Attackers may be able to circumvent this requirement by using source routing, but source routing is disabled across much of the Internet today. In summary, IP address verification can be a useful part of an authentication scheme, but it should not be the single factor required for authentication.",,high, CWE-293,EN-Using Referer Field for Authentication (Type: Variant),"The referer field in HTTP requests can be easily modified and, as such, is not a valid means of message integrity checking. -IP addresses can be easily spoofed. Attackers can forge the source IP address of the packets they send, but response packets will return to the forged IP address. To see the response packets, the attacker has to sniff the traffic between the victim machine and the forged IP address. In order to accomplish the required sniffing, attackers typically attempt to locate themselves on the same subnet as the victim machine. Attackers may be able to circumvent this requirement by using source routing, but source routing is disabled across much of the Internet today. In summary, IP address verification can be a useful part of an authentication scheme, but it should not be the single factor required for authentication.",,high,"The Art of Software Security Assessment: Chapter 17, ""Referer Request Header"", Page 1030." +IP addresses can be easily spoofed. Attackers can forge the source IP address of the packets they send, but response packets will return to the forged IP address. To see the response packets, the attacker has to sniff the traffic between the victim machine and the forged IP address. In order to accomplish the required sniffing, attackers typically attempt to locate themselves on the same subnet as the victim machine. Attackers may be able to circumvent this requirement by using source routing, but source routing is disabled across much of the Internet today. In summary, IP address verification can be a useful part of an authentication scheme, but it should not be the single factor required for authentication.",,high,"The Art of Software Security Assessment: Chapter 17, ""Referer Request Header"", Page 1030." CWE-294,EN-Authentication Bypass by Capture-replay (Type: Base),"A capture-replay flaw exists when the design of the software makes it possible for a malicious user to sniff network traffic and bypass authentication by replaying it to the server in question to the same effect as the original message (or with minor changes). -Capture-replay attacks are common and can be difficult to defeat without cryptography. They are a subset of network injection attacks that rely on observing previously-sent valid commands, then changing them slightly if necessary and resending the same commands to the server.",,high, +Capture-replay attacks are common and can be difficult to defeat without cryptography. They are a subset of network injection attacks that rely on observing previously-sent valid commands, then changing them slightly if necessary and resending the same commands to the server.",,high, CWE-297,EN-Improper Validation of Certificate with Host Mismatch (Type: Variant),"The software communicates with a host that provides a certificate, but the software does not properly ensure that the certificate is actually associated with that host. Even if a certificate is well-formed, signed, and follows the chain of trust, it may simply be a valid certificate for a different site than the site that the software is interacting with. If the certificate's host-specific data is not properly checked - such as the Common Name (CN) in the Subject or the Subject Alternative Name (SAN) extension of an X.509 certificate - it may be possible for a redirection or spoofing attack to allow a malicious host with a valid certificate to provide data, impersonating a trusted host. In order to ensure data integrity, the certificate must be valid and it must pertain to the site that is being accessed. Even if the software attempts to check the hostname, it is still possible to incorrectly check the hostname. For example, attackers could create a certificate with a name that begins with a trusted name followed by a NUL byte, which could cause some string-based comparisons to only examine the portion that contains the trusted name.",,high,"The Most Dangerous Code in the World: Validating SSL Certificates in Non-Browser Software: http://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf Why Eve and Mallory Love Android: An Analysis of Android SSL (In)Security: http://www2.dcsec.uni-hannover.de/files/android/p50-fahl.pdf Secure programming with the OpenSSL API, Part 2: Secure handshake: http://www.ibm.com/developerworks/library/l-openssl2/index.html An Introduction to OpenSSL Programming (Part I): http://www.rtfm.com/openssl-examples/part1.pdf -24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347" +24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347" CWE-308,EN-Use of Single-factor Authentication (Type: Base),"The use of single-factor authentication can lead to unnecessary risk of compromise when compared with the benefits of a dual-factor authentication scheme. -While the use of multiple authentication schemes is simply piling on more complexity on top of authentication, it is inestimably valuable to have such measures of redundancy. The use of weak, reused, and common passwords is rampant on the internet. Without the added protection of multiple authentication schemes, a single mistake can result in the compromise of an account. For this reason, if multiple schemes are possible and also easy to use, they should be implemented and required.",,high, +While the use of multiple authentication schemes is simply piling on more complexity on top of authentication, it is inestimably valuable to have such measures of redundancy. The use of weak, reused, and common passwords is rampant on the internet. Without the added protection of multiple authentication schemes, a single mistake can result in the compromise of an account. For this reason, if multiple schemes are possible and also easy to use, they should be implemented and required.",,high, CWE-321,EN-Use of Hard-coded Cryptographic Key (Type: Base),"The use of a hard-coded cryptographic key significantly increases the possibility that encrypted data may be recovered. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '...' manipulation is useful for bypassing some path traversal protection schemes. On some Windows systems, it is equivalent to ""..\.."" and might bypass checks that assume only two dots are valid. Incomplete filtering, such as removal of ""./"" sequences, can ultimately produce valid "".."" sequences due to a collapse into unsafe value (CWE-182).",,high, +The '...' manipulation is useful for bypassing some path traversal protection schemes. On some Windows systems, it is equivalent to ""..\.."" and might bypass checks that assume only two dots are valid. Incomplete filtering, such as removal of ""./"" sequences, can ultimately produce valid "".."" sequences due to a collapse into unsafe value (CWE-182).",,high, CWE-322,EN-Key Exchange without Entity Authentication (Type: Base),"The software performs a key exchange with an actor without verifying the identity of that actor. Performing a key exchange will preserve the integrity of the information sent between two entities, but this will not guarantee that the entities are who they claim they are. This may enable a set of ""man-in-the-middle"" attacks. Typically, this involves a victim client that contacts a malicious server that is impersonating a trusted server. If the client skips authentication or ignores an authentication failure, the malicious server may request authentication information from the user. The malicious server can then use this authentication information to log in to the trusted server using the victim's credentials, sniff traffic between the victim and trusted server, etc.",,high,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347 -The Art of Software Security Assessment: Chapter 2, ""Untrustworthy Credentials"", Page 37." +The Art of Software Security Assessment: Chapter 2, ""Untrustworthy Credentials"", Page 37." CWE-323,"EN-Reusing a Nonce, Key Pair in Encryption (Type: Base)","Nonces should be used for the present occasion and only once. -Performing a key exchange will preserve the integrity of the information sent between two entities, but this will not guarantee that the entities are who they claim they are. This may enable a set of ""man-in-the-middle"" attacks. Typically, this involves a victim client that contacts a malicious server that is impersonating a trusted server. If the client skips authentication or ignores an authentication failure, the malicious server may request authentication information from the user. The malicious server can then use this authentication information to log in to the trusted server using the victim's credentials, sniff traffic between the victim and trusted server, etc.",,high, +Performing a key exchange will preserve the integrity of the information sent between two entities, but this will not guarantee that the entities are who they claim they are. This may enable a set of ""man-in-the-middle"" attacks. Typically, this involves a victim client that contacts a malicious server that is impersonating a trusted server. If the client skips authentication or ignores an authentication failure, the malicious server may request authentication information from the user. The malicious server can then use this authentication information to log in to the trusted server using the victim's credentials, sniff traffic between the victim and trusted server, etc.",,high, CWE-360,EN-Trust of System Event Data (Type: Base),"Security based on event locations are insecure and can be spoofed. -Events are a messaging system which may provide control data to programs listening for events. Events often do not have any type of authentication framework to allow them to be verified from a trusted source. Any application, in Windows, on a given desktop can send a message to any window on the same desktop. There is no authentication framework for these messages. Therefore, any message can be used to manipulate any process on the desktop if the process does not check the validity and safeness of those messages.",,high, +Events are a messaging system which may provide control data to programs listening for events. Events often do not have any type of authentication framework to allow them to be verified from a trusted source. Any application, in Windows, on a given desktop can send a message to any window on the same desktop. There is no authentication framework for these messages. Therefore, any message can be used to manipulate any process on the desktop if the process does not check the validity and safeness of those messages.",,high, CWE-378,EN-Creation of Temporary File With Insecure Permissions (Type: Base),"Opening temporary files without appropriate measures or controls can leave the file, its contents and any function that it impacts vulnerable to attack. -If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,high, +If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,high, CWE-416,EN-Use After Free (Type: Base),"Referencing memory after it has been freed can cause a program to crash, use unexpected values, or execute code. The use of previously-freed memory can have any number of adverse consequences, ranging from the corruption of valid data to the execution of arbitrary code, depending on the instantiation and timing of the flaw. The simplest way data corruption may occur involves the system's reuse of the freed memory. Use-after-free errors have two common and sometimes overlapping causes: Error conditions and other exceptional circumstances. Confusion over which part of the program is responsible for freeing the memory. In this scenario, the memory in question is allocated to another pointer validly at some point after it has been freed. The original pointer to the freed memory is used again and points to somewhere within the new allocation. As the data is changed, it corrupts the validly used memory; this induces undefined behavior in the process. -If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,high,"24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143" +If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,high,"24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143" CWE-457,EN-Use of Uninitialized Variable (Type: Variant),"The code uses a variable that has not been initialized, leading to unpredictable or unintended results. In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,high,"Exploiting Uninitialized Data: http://www.felinemenace.org/~mercy/papers/UBehavior/UBehavior.zip MS08-014 : The Case of the Uninitialized Stack Variable Vulnerability: http://blogs.technet.com/swi/archive/2008/03/11/the-case-of-the-uninitialized-stack-variable-vulnerability.aspx 24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143 -The Art of Software Security Assessment: Chapter 7, ""Variable Initialization"", Page 312." +The Art of Software Security Assessment: Chapter 7, ""Variable Initialization"", Page 312." CWE-467,EN-Use of sizeof() on a Pointer Type (Type: Variant),"The code calls sizeof() on a malloced pointer type, which always returns the wordsize/8. This can produce an unexpected result if the programmer intended to determine how much memory has been allocated. -Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,high,EXP01-A. Do not take the sizeof a pointer to determine the size of a type: https://www.securecoding.cert.org/confluence/display/seccode/EXP01-A.+Do+not+take+the+sizeof+a+pointer+to+determine+the+size+of+a+type +Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,high,EXP01-A. Do not take the sizeof a pointer to determine the size of a type: https://www.securecoding.cert.org/confluence/display/seccode/EXP01-A.+Do+not+take+the+sizeof+a+pointer+to+determine+the+size+of+a+type CWE-486,EN-Comparison of Classes by Name (Type: Variant),"The program compares classes by name, which can cause it to use the wrong class when multiple classes can have the same name. -If the decision to trust the methods and data of an object is based on the name of a class, it is possible for malicious users to send objects of the same name as trusted classes and thereby gain the trust afforded to known classes and types.",,high, +If the decision to trust the methods and data of an object is based on the name of a class, it is possible for malicious users to send objects of the same name as trusted classes and thereby gain the trust afforded to known classes and types.",,high, CWE-493,EN-Critical Public Variable Without Final Modifier (Type: Variant),"The product has a critical public variable that is not final, which allows the variable to be modified to contain unexpected values. -If a field is non-final and public, it can be changed once the value is set by any function that has access to the class which contains the field. This could lead to a vulnerability if other parts of the program make assumptions about the contents of that field.",,high, +If a field is non-final and public, it can be changed once the value is set by any function that has access to the class which contains the field. This could lead to a vulnerability if other parts of the program make assumptions about the contents of that field.",,high, CWE-499,EN-Serializable Class Containing Sensitive Data (Type: Variant),"The code contains a class with sensitive data, but the class does not explicitly deny serialization. The data can be accessed by serializing the class through another class. -Serializable classes are effectively open classes since data cannot be hidden in them. Classes that do not explicitly deny serialization can be serialized by any other class, which can then in turn use the data stored inside it.",,high, +Serializable classes are effectively open classes since data cannot be hidden in them. Classes that do not explicitly deny serialization can be serialized by any other class, which can then in turn use the data stored inside it.",,high, CWE-500,EN-Public Static Field Not Marked Final (Type: Variant),"An object contains a public static field that is not marked final, which might allow it to be modified in unexpected ways. -Public static variables can be read without an accessor and changed without a mutator by any classes in the application.",,high, +Public static variables can be read without an accessor and changed without a mutator by any classes in the application.",,high, CWE-515,EN-Covert Storage Channel (Type: Base),"A covert storage channel transfers information through the setting of bits by one program and the reading of those bits by another. What distinguishes this case from that of ordinary operation is that the bits are used to convey encoded information. -Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,high, +Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,high, CWE-639,EN-Authorization Bypass Through User-Controlled Key (Type: Base),"The system's authorization functionality does not prevent one user from gaining access to another user's data or record by modifying the key value identifying the data. -Retrieval of a user record occurs in the system based on some key value that is under user control. The key would typically identify a user related record stored in the system and would be used to lookup that record for presentation to the user. It is likely that an attacker would have to be an authenticated user in the system. However, the authorization process would not properly check the data access operation to ensure that the authenticated user performing the operation has sufficient entitlements to perform the requested data access, hence bypassing any other authorization checks present in the system. One manifestation of this weakness would be if a system used sequential or otherwise easily guessable session ids that would allow one user to easily switch to another user's session and read/modify their data.",,high, +Retrieval of a user record occurs in the system based on some key value that is under user control. The key would typically identify a user related record stored in the system and would be used to lookup that record for presentation to the user. It is likely that an attacker would have to be an authenticated user in the system. However, the authorization process would not properly check the data access operation to ensure that the authenticated user performing the operation has sufficient entitlements to perform the requested data access, hence bypassing any other authorization checks present in the system. One manifestation of this weakness would be if a system used sequential or otherwise easily guessable session ids that would allow one user to easily switch to another user's session and read/modify their data.",,high, CWE-640,EN-Weak Password Recovery Mechanism for Forgotten Password (Type: Base),"The software contains a mechanism for users to recover or change their passwords without knowing the original password, but the mechanism is weak. It is common for an application to have a mechanism that provides a means for a user to gain access to their account in the event they forget their password. Very often the password recovery mechanism is weak, which has the effect of making it more likely that it would be possible for a person other than the legitimate system user to gain access to that user's account. -This weakness may be that the security question is too easy to guess or find an answer to (e.g. because it is too common). Or there might be an implementation weakness in the password recovery mechanism code that may for instance trick the system into e-mailing the new password to an e-mail account other than that of the user. There might be no throttling done on the rate of password resets so that a legitimate user can be denied service by an attacker if an attacker tries to recover their password in a rapid succession. The system may send the original password to the user rather than generating a new temporary password. In summary, password recovery functionality, if not carefully designed and implemented can often become the system's weakest link that can be misused in a way that would allow an attacker to gain unauthorized access to the system. Weak password recovery schemes completely undermine a strong password authentication scheme.",,high,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" +This weakness may be that the security question is too easy to guess or find an answer to (e.g. because it is too common). Or there might be an implementation weakness in the password recovery mechanism code that may for instance trick the system into e-mailing the new password to an e-mail account other than that of the user. There might be no throttling done on the rate of password resets so that a legitimate user can be denied service by an attacker if an attacker tries to recover their password in a rapid succession. The system may send the original password to the user rather than generating a new temporary password. In summary, password recovery functionality, if not carefully designed and implemented can often become the system's weakest link that can be misused in a way that would allow an attacker to gain unauthorized access to the system. Weak password recovery schemes completely undermine a strong password authentication scheme.",,high,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" CWE-642,EN-External Control of Critical State Data (Type: Class),"The software stores security-critical state information about its users, or the software itself, in a location that is accessible to unauthorized actors. If an attacker can modify the state information without detection, then it could be used to perform unauthorized actions or access unexpected resources, since the application programmer does not expect that the state can be changed. State information can be stored in various locations such as a cookie, in a hidden web form field, input parameter or argument, an environment variable, a database record, within a settings file, etc. All of these locations have the potential to be modified by an attacker. When this state information is used to control security or determine resource usage, then it may create a vulnerability. For example, an application may perform authentication, then save the state in an ""authenticated=true"" cookie. An attacker may simply create this cookie in order to bypass the authentication.",,high,"Top 10 2007-Insecure Direct Object Reference: http://www.owasp.org/index.php/Top_10_2007-A4 HMAC: http://en.wikipedia.org/wiki/Hmac -24 Deadly Sins of Software Security: ""Sin 4: Use of Magic URLs, Predictable Cookies, and Hidden Form Fields."" Page 75" +24 Deadly Sins of Software Security: ""Sin 4: Use of Magic URLs, Predictable Cookies, and Hidden Form Fields."" Page 75" CWE-643,EN-Improper Neutralization of Data within XPath Expressions (XPath Injection) (Type: Base),"The software uses external input to dynamically construct an XPath expression used to retrieve data from an XML database, but it does not neutralize or incorrectly neutralizes that input. This allows an attacker to control the structure of the query. The net effect is that the attacker will have control over the information selected from the XML database and may use that ability to control application flow, modify logic, retrieve unauthorized data, or bypass important checks (e.g. authentication).",,high,"XPath Injection: http://www.webappsec.org/projects/threat/classes/xpath_injection.shtml -The Art of Software Security Assessment: Chapter 17, ""XPath Injection"", Page 1070." +The Art of Software Security Assessment: Chapter 17, ""XPath Injection"", Page 1070." CWE-644,EN-Improper Neutralization of HTTP Headers for Scripting Syntax (Type: Variant),"The application does not neutralize or incorrectly neutralizes web scripting syntax in HTTP headers that can be used by web browser components that can process raw headers, such as Flash. An attacker may be able to conduct cross-site scripting and other attacks against users who have these components enabled. -If an application does not neutralize user controlled data being placed in the header of an HTTP response coming from the server, the header may contain a script that will get executed in the client's browser context, potentially resulting in a cross site scripting vulnerability or possibly an HTTP response splitting attack. It is important to carefully control data that is being placed both in HTTP response header and in the HTTP response body to ensure that no scripting syntax is present, taking various encodings into account.",,high, +If an application does not neutralize user controlled data being placed in the header of an HTTP response coming from the server, the header may contain a script that will get executed in the client's browser context, potentially resulting in a cross site scripting vulnerability or possibly an HTTP response splitting attack. It is important to carefully control data that is being placed both in HTTP response header and in the HTTP response body to ensure that no scripting syntax is present, taking various encodings into account.",,high, CWE-645,EN-Overly Restrictive Account Lockout Mechanism (Type: Base),"The software contains an account lockout protection mechanism, but the mechanism is too restrictive and can be triggered too easily. This allows attackers to deny service to legitimate users by causing their accounts to be locked out. -Account lockout is a security feature often present in applications as a countermeasure to the brute force attack on the password based authentication mechanism of the system. After a certain number of failed login attempts, the users' account may be disabled for a certain period of time or until it is unlocked by an administrator. Other security events may also possibly trigger account lockout. However, an attacker may use this very security feature to deny service to legitimate system users. It is therefore important to ensure that the account lockout security mechanism is not overly restrictive.",,high, +Account lockout is a security feature often present in applications as a countermeasure to the brute force attack on the password based authentication mechanism of the system. After a certain number of failed login attempts, the users' account may be disabled for a certain period of time or until it is unlocked by an administrator. Other security events may also possibly trigger account lockout. However, an attacker may use this very security feature to deny service to legitimate system users. It is therefore important to ensure that the account lockout security mechanism is not overly restrictive.",,high, CWE-646,EN-Reliance on File Name or Extension of Externally-Supplied File (Type: Variant),"The software allows a file to be uploaded, but it relies on the file name or extension of the file to determine the appropriate behaviors. This could be used by attackers to cause the file to be misclassified and processed in a dangerous fashion. -An application might use the file name or extension of of a user-supplied file to determine the proper course of action, such as selecting the correct process to which control should be passed, deciding what data should be made available, or what resources should be allocated. If the attacker can cause the code to misclassify the supplied file, then the wrong action could occur. For example, an attacker could supply a file that ends in a "".php.gif"" extension that appears to be a GIF image, but would be processed as PHP code. In extreme cases, code execution is possible, but the attacker could also cause exhaustion of resources, denial of service, exposure of debug or system data (including application source code), or being bound to a particular server side process. This weakness may be due to a vulnerability in any of the technologies used by the web and application servers, due to misconfiguration, or resultant from another flaw in the application itself.",,high, +An application might use the file name or extension of of a user-supplied file to determine the proper course of action, such as selecting the correct process to which control should be passed, deciding what data should be made available, or what resources should be allocated. If the attacker can cause the code to misclassify the supplied file, then the wrong action could occur. For example, an attacker could supply a file that ends in a "".php.gif"" extension that appears to be a GIF image, but would be processed as PHP code. In extreme cases, code execution is possible, but the attacker could also cause exhaustion of resources, denial of service, exposure of debug or system data (including application source code), or being bound to a particular server side process. This weakness may be due to a vulnerability in any of the technologies used by the web and application servers, due to misconfiguration, or resultant from another flaw in the application itself.",,high, CWE-647,EN-Use of Non-Canonical URL Paths for Authorization Decisions (Type: Variant),"The software defines policy namespaces and makes authorization decisions based on the assumption that a URL is canonical. This can allow a non-canonical URL to bypass the authorization. If an application defines policy namespaces and makes authorization decisions based on the URL, but it does not require or convert to a canonical URL before making the authorization decision, then it opens the application to attack. For example, if the application only wants to allow access to http://www.example.com/mypage, then the attacker might be able to bypass this restriction using equivalent URLs such as: http://WWW.EXAMPLE.COM/mypage @@ -143,21 +143,21 @@ http://www.example.com/%6Dypage (alternate encoding) http://192.168.1.1/mypage (IP address) http://www.example.com/mypage/ (trailing /) http://www.example.com:80/mypage -Therefore it is important to specify access control policy that is based on the path information in some canonical form with all alternate encodings rejected (which can be accomplished by a default deny rule).",,high, +Therefore it is important to specify access control policy that is based on the path information in some canonical form with all alternate encodings rejected (which can be accomplished by a default deny rule).",,high, CWE-649,EN-Reliance on Obfuscation or Encryption of Security-Relevant Inputs without Integrity Checking (Type: Base),"The software uses obfuscation or encryption of inputs that should not be mutable by an external actor, but the software does not use integrity checks to detect if those inputs have been modified. -When an application relies on obfuscation or incorrectly applied / weak encryption to protect client-controllable tokens or parameters, that may have an effect on the user state, system state, or some decision made on the server. Without protecting the tokens/parameters for integrity, the application is vulnerable to an attack where an adversary blindly traverses the space of possible values of the said token/parameter in order to attempt to gain an advantage. The goal of the attacker is to find another admissible value that will somehow elevate his or her privileges in the system, disclose information or change the behavior of the system in some way beneficial to the attacker. If the application does not protect these critical tokens/parameters for integrity, it will not be able to determine that these values have been tampered with. Measures that are used to protect data for confidentiality should not be relied upon to provide the integrity service.",,high, +When an application relies on obfuscation or incorrectly applied / weak encryption to protect client-controllable tokens or parameters, that may have an effect on the user state, system state, or some decision made on the server. Without protecting the tokens/parameters for integrity, the application is vulnerable to an attack where an adversary blindly traverses the space of possible values of the said token/parameter in order to attempt to gain an advantage. The goal of the attacker is to find another admissible value that will somehow elevate his or her privileges in the system, disclose information or change the behavior of the system in some way beneficial to the attacker. If the application does not protect these critical tokens/parameters for integrity, it will not be able to determine that these values have been tampered with. Measures that are used to protect data for confidentiality should not be relied upon to provide the integrity service.",,high, CWE-650,EN-Trusting HTTP Permission Methods on the Server Side (Type: Variant),"The server contains a protection mechanism that assumes that any URI that is accessed using HTTP GET will not cause a state change to the associated resource. This might allow attackers to bypass intended access restrictions and conduct resource modification and deletion attacks, since some applications allow GET to modify state. -An application may disallow the HTTP requests to perform DELETE, PUT and POST operations on the resource representation, believing that it will be enough to prevent unintended resource alterations. Even though the HTTP GET specification requires that GET requests should not have side effects, there is nothing in the HTTP protocol itself that prevents the HTTP GET method from performing more than just query of the data. For instance, it is a common practice with REST based Web Services to have HTTP GET requests modifying resources on the server side. Whenever that happens however, the access control needs to be properly enforced in the application. No assumptions should be made that only HTTP DELETE, PUT, and POST methods have the power to alter the representation of the resource being accessed in the request.",,high, +An application may disallow the HTTP requests to perform DELETE, PUT and POST operations on the resource representation, believing that it will be enough to prevent unintended resource alterations. Even though the HTTP GET specification requires that GET requests should not have side effects, there is nothing in the HTTP protocol itself that prevents the HTTP GET method from performing more than just query of the data. For instance, it is a common practice with REST based Web Services to have HTTP GET requests modifying resources on the server side. Whenever that happens however, the access control needs to be properly enforced in the application. No assumptions should be made that only HTTP DELETE, PUT, and POST methods have the power to alter the representation of the resource being accessed in the request.",,high, CWE-652,EN-Improper Neutralization of Data within XQuery Expressions (XQuery Injection) (Type: Base),"The software uses external input to dynamically construct an XQuery expression used to retrieve data from an XML database, but it does not neutralize or incorrectly neutralizes that input. This allows an attacker to control the structure of the query. -The net effect is that the attacker will have control over the information selected from the XML database and may use that ability to control application flow, modify logic, retrieve unauthorized data, or bypass important checks (e.g. authentication).",,high, +The net effect is that the attacker will have control over the information selected from the XML database and may use that ability to control application flow, modify logic, retrieve unauthorized data, or bypass important checks (e.g. authentication).",,high, CWE-676,EN-Use of Potentially Dangerous Function (Type: Base),"The program invokes a potentially dangerous function that could introduce a vulnerability if it is used incorrectly, but the function can also be used safely. Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,high,"Security Development Lifecycle (SDL) Banned Function Calls: http://msdn.microsoft.com/en-us/library/bb288454.aspx Writing Secure Code: Chapter 5, ""Safe String Handling"" Page 156, 160 -The Art of Software Security Assessment: Chapter 8, ""C String Handling"", Page 388." +The Art of Software Security Assessment: Chapter 8, ""C String Handling"", Page 388." CWE-682,EN-Incorrect Calculation (Type: Class),"The software performs a calculation that generates incorrect or unintended results that are later used in security-critical decisions or resource management.. When software performs a security-critical calculation incorrectly, it might lead to incorrect resource allocations, incorrect privilege assignments, or failed comparisons among other things. Many of the direct results of an incorrect calculation can lead to even larger problems such as failed protection mechanisms or even arbitrary code execution.",,high,"SafeInt: http://safeint.codeplex.com/ 24 Deadly Sins of Software Security: ""Sin 7: Integer Overflows."" Page 119 -The Art of Software Security Assessment: Chapter 6, ""Signed Integer Boundaries"", Page 220." +The Art of Software Security Assessment: Chapter 6, ""Signed Integer Boundaries"", Page 220." CWE-78,EN-Improper Neutralization of Special Elements used in an OS Command (OS Command Injection) (Type: Base),"The software constructs all or part of an OS command using externally-influenced input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could modify the intended OS command when it is sent to a downstream component.. This could allow attackers to execute unexpected, dangerous commands directly on the operating system. This weakness can lead to a vulnerability in environments in which the attacker does not have direct access to the operating system, such as in web applications. Alternately, if the weakness occurs in a privileged program, it could allow the attacker to specify commands that normally would not be accessible, or to call alternate commands with privileges that the attacker does not have. The problem is exacerbated if the compromised process does not follow the principle of least privilege, because the attacker-controlled commands may run with special system privileges that increases the amount of damage. There are at least two subtypes of OS command injection: @@ -172,10 +172,10 @@ Security Issues in Perl Scripts: http://www.cgisecurity.com/lib/sips.html Top 25 Series - Rank 9 - OS Command Injection: http://blogs.sans.org/appsecstreetfighter/2010/02/24/top-25-series-rank-9-os-command-injection/ OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html -The Art of Software Security Assessment: Chapter 8, ""Shell Metacharacters"", Page 425." +The Art of Software Security Assessment: Chapter 8, ""Shell Metacharacters"", Page 425." CWE-784,EN-Reliance on Cookies without Validation and Integrity Checking in a Security Decision (Type: Variant),"The application uses a protection mechanism that relies on the existence or values of a cookie, but it does not properly ensure that the cookie is valid for the associated user. Attackers can easily modify cookies, within the browser or by implementing the client-side code outside of the browser. Attackers can bypass protection mechanisms such as authorization and authentication by modifying the cookie to contain an expected value.",,high,"Unforgivable Vulnerabilities: http://cve.mitre.org/docs/docs-2007/unforgivable.pdf -Writing Secure Code: Chapter 13, ""Sensitive Data in Cookies and Fields"" Page 435" +Writing Secure Code: Chapter 13, ""Sensitive Data in Cookies and Fields"" Page 435" CWE-862,EN-Missing Authorization (Type: Class),"The software does not perform an authorization check when an actor attempts to access a resource or perform an action. Assuming a user with a given identity, authorization is the process of determining whether that user can access a given resource, based on the user's privileges and any permissions or other access-control specifications that apply to the resource. When access control checks are not applied, users are able to access data or perform actions that they should not be allowed to perform. This can lead to a wide range of problems, including information exposures, denial of service, and arbitrary code execution.",,high,"Role Based Access Control and Role Based Security: http://csrc.nist.gov/groups/SNS/rbac/ @@ -183,7 +183,7 @@ Writing Secure Code: Chapter 4, ""Authorization"" Page 114; Chapter 6, ""Determi Top 25 Series - Rank 5 - Improper Access Control (Authorization): http://blogs.sans.org/appsecstreetfighter/2010/03/04/top-25-series-rank-5-improper-access-control-authorization/ OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI Authentication using JAAS: http://www.javaranch.com/journal/2008/04/authentication-using-JAAS.html -The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Authorization"", Page 39." +The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Authorization"", Page 39." CWE-863,EN-Incorrect Authorization (Type: Class),"The software performs an authorization check when an actor attempts to access a resource or perform an action, but it does not correctly perform the check. This allows attackers to bypass intended access restrictions. Assuming a user with a given identity, authorization is the process of determining whether that user can access a given resource, based on the user's privileges and any permissions or other access-control specifications that apply to the resource. When access control checks are incorrectly applied, users are able to access data or perform actions that they should not be allowed to perform. This can lead to a wide range of problems, including information exposures, denial of service, and arbitrary code execution.",,high,"Role Based Access Control and Role Based Security: http://csrc.nist.gov/groups/SNS/rbac/ @@ -191,9 +191,9 @@ Writing Secure Code: Chapter 4, ""Authorization"" Page 114; Chapter 6, ""Determi Top 25 Series - Rank 5 - Improper Access Control (Authorization): http://blogs.sans.org/appsecstreetfighter/2010/03/04/top-25-series-rank-5-improper-access-control-authorization/ Authentication using JAAS: http://www.javaranch.com/journal/2008/04/authentication-using-JAAS.html OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI -The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Authorization"", Page 39." +The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Authorization"", Page 39." CWE-99,EN-Improper Control of Resource Identifiers (Resource Injection) (Type: Base),"The software receives input from an upstream component, but it does not restrict or incorrectly restricts the input before it is used as an identifier for a resource that may be outside the intended sphere of control. -This may enable an attacker to access or modify otherwise protected system resources.",,high, +This may enable an attacker to access or modify otherwise protected system resources.",,high, CWE-120,EN-Buffer Copy without Checking Size of Input (Classic Buffer Overflow) (Type: Base),"The program copies an input buffer to an output buffer without verifying that the size of the input buffer is less than the size of the output buffer, leading to a buffer overflow. A buffer overflow condition exists when a program attempts to put more data in a buffer than it can hold, or when a program attempts to put data in a memory area outside of the boundaries of a buffer. The simplest type of error, and the most common cause of buffer overflows, is the ""classic"" case in which the program copies the buffer without restricting how much is copied. Other variants exist, but the existence of a classic overflow strongly suggests that the programmer is not considering even the most basic of security protections.",,high,"Writing Secure Code: Chapter 5, ""Public Enemy #1: The Buffer Overrun"" Page 127 24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89 @@ -207,12 +207,12 @@ Understanding DEP as a mitigation technology part 1: http://blogs.technet.com/b/ Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html The Art of Software Security Assessment: Chapter 3, ""Nonexecutable Stack"", Page 76. The Art of Software Security Assessment: Chapter 5, ""Protection Mechanisms"", Page 189. -The Art of Software Security Assessment: Chapter 8, ""C String Handling"", Page 388." +The Art of Software Security Assessment: Chapter 8, ""C String Handling"", Page 388." CWE-122,EN-Heap-based Buffer Overflow (Type: Variant),"A heap overflow condition is a buffer overflow, where the buffer that can be overwritten is allocated in the heap portion of memory, generally meaning that the buffer was allocated using a routine such as malloc(). A buffer overflow condition exists when a program attempts to put more data in a buffer than it can hold, or when a program attempts to put data in a memory area outside of the boundaries of a buffer. The simplest type of error, and the most common cause of buffer overflows, is the ""classic"" case in which the program copies the buffer without restricting how much is copied. Other variants exist, but the existence of a classic overflow strongly suggests that the programmer is not considering even the most basic of security protections.",,high,"Writing Secure Code: Chapter 5, ""Heap Overruns"" Page 138 24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89 The Art of Software Security Assessment: Chapter 3, ""Nonexecutable Stack"", Page 76. -The Art of Software Security Assessment: Chapter 5, ""Protection Mechanisms"", Page 189." +The Art of Software Security Assessment: Chapter 5, ""Protection Mechanisms"", Page 189." CWE-131,EN-Incorrect Calculation of Buffer Size (Type: Base),"The software does not correctly calculate the size to be used when allocating a buffer, which could lead to a buffer overflow. If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,high,"SafeInt: http://safeint.codeplex.com/ Top 25 Series - Rank 18 - Incorrect Calculation of Buffer Size: http://software-security.sans.org/blog/2010/03/19/top-25-series-rank-18-incorrect-calculation-of-buffer-size @@ -222,7 +222,7 @@ PaX: http://en.wikipedia.org/wiki/PaX Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html Writing Secure Code: Chapter 20, ""Integer Overflows"" Page 620 24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89 -The Art of Software Security Assessment: Chapter 8, ""Incrementing Pointers Incorrectly"", Page 401." +The Art of Software Security Assessment: Chapter 8, ""Incrementing Pointers Incorrectly"", Page 401." CWE-22,EN-Improper Limitation of a Pathname to a Restricted Directory (Path Traversal) (Type: Class),"The software uses external input to construct a pathname that is intended to identify a file or directory that is located underneath a restricted parent directory, but the software does not properly neutralize special elements within the pathname that can cause the pathname to resolve to a location that is outside of the restricted directory. Many file operations are intended to take place within a restricted directory. By using special elements such as "".."" and ""/"" separators, attackers can escape outside of the restricted location to access files or directories that are elsewhere on the system. One of the most common special elements is the ""../"" sequence, which in most modern operating systems is interpreted as the parent directory of the current location. This is referred to as relative path traversal. Path traversal also covers the use of absolute pathnames such as ""/usr/local/bin"", which may also be useful in accessing unexpected files. This is referred to as absolute path traversal. In many programming languages, the injection of a null byte (the 0 or NUL) may allow an attacker to truncate a generated filename to widen the scope of attack. For example, the software may add "".txt"" to any pathname, thus limiting the attacker to text files, but a null injection may effectively remove this restriction.",,high,"Writing Secure Code: Chapter 11, ""Directory Traversal and Using Parent Paths (..)"" Page 370 @@ -230,26 +230,26 @@ OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ES Testing for Path Traversal (OWASP-AZ-001): http://www.owasp.org/index.php/Testing_for_Path_Traversal_(OWASP-AZ-001) Top 25 Series - Rank 7 - Path Traversal: http://blogs.sans.org/appsecstreetfighter/2010/03/09/top-25-series-rank-7-path-traversal/ Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html -The Art of Software Security Assessment: Chapter 9, ""Filenames and Paths"", Page 503." +The Art of Software Security Assessment: Chapter 9, ""Filenames and Paths"", Page 503." CWE-311,EN-Missing Encryption of Sensitive Data (Type: Base),"The software does not encrypt sensitive or critical information before storage or transmission. The lack of proper data encryption passes up the guarantees of confidentiality, integrity, and accountability that properly implemented encryption conveys.",,high,"Writing Secure Code: Chapter 9, ""Protecting Secret Data"" Page 299 24 Deadly Sins of Software Security: ""Sin 17: Failure to Protect Stored Data."" Page 253 Top 25 Series - Rank 10 - Missing Encryption of Sensitive Data: http://blogs.sans.org/appsecstreetfighter/2010/02/26/top-25-series-rank-10-missing-encryption-of-sensitive-data/ The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Encryption"", Page 43. -SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf" +SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf" CWE-464,EN-Addition of Data Structure Sentinel (Type: Base),"The accidental addition of a data-structure sentinel can cause serious programming logic problems. -Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,high, +Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,high, CWE-67,EN-Improper Handling of Windows Device Names (Type: Variant),"The software constructs pathnames from user input, but it does not handle or incorrectly handles a pathname containing a Windows device name such as AUX or CON. This typically leads to denial of service or an information exposure when the application attempts to process the pathname as a regular file. Not properly handling virtual filenames (e.g. AUX, CON, PRN, COM1, LPT1) can result in different types of vulnerabilities. In some cases an attacker can request a device via injection of a virtual filename in a URL, which may cause an error that leads to a denial of service or an error page that reveals sensitive information. A software system that allows device names to bypass filtering runs the risk of an attacker injecting malicious code in a file with the name of a device.",,high,"Writing Secure Code -The Art of Software Security Assessment: Chapter 11, ""Device Files"", Page 666." +The Art of Software Security Assessment: Chapter 11, ""Device Files"", Page 666." CWE-73,EN-External Control of File Name or Path (Type: Class),"The software allows user input to control or influence paths or file names that are used in filesystem operations. This could allow an attacker to access or modify system files or other files that are critical to the application. Path manipulation errors occur when the following two conditions are met: 1. An attacker can specify a path used in an operation on the filesystem. 2. By specifying the resource, the attacker gains a capability that would not otherwise be permitted. -For example, the program may give the attacker the ability to overwrite the specified file or run with a configuration controlled by the attacker.",,high,OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI +For example, the program may give the attacker the ability to overwrite the specified file or run with a configuration controlled by the attacker.",,high,OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI CWE-76,EN-Improper Neutralization of Equivalent Special Elements (Type: Base),"The software properly neutralizes certain special elements, but it improperly neutralizes equivalent special elements. -The software may have a fixed list of special characters it believes is complete. However, there may be alternate encodings, or representations that also have the same meaning. For example, the software may filter out a leading slash (/) to prevent absolute path names, but does not account for a tilde (~) followed by a user name, which on some *nix systems could be expanded to an absolute pathname. Alternately, the software might filter a dangerous ""-e"" command-line switch when calling an external program, but it might not account for ""--exec"" or other switches that have the same semantics.",,high, +The software may have a fixed list of special characters it believes is complete. However, there may be alternate encodings, or representations that also have the same meaning. For example, the software may filter out a leading slash (/) to prevent absolute path names, but does not account for a tilde (~) followed by a user name, which on some *nix systems could be expanded to an absolute pathname. Alternately, the software might filter a dangerous ""-e"" command-line switch when calling an external program, but it might not account for ""--exec"" or other switches that have the same semantics.",,high, CWE-79,EN-Improper Neutralization of Input During Web Page Generation (Cross-site Scripting) (Type: Base),"The software does not neutralize or incorrectly neutralizes user-controllable input before it is placed in output that is used as a web page that is served to other users. Cross-site scripting (XSS) vulnerabilities occur when: 1. Untrusted data enters a web application, typically from a web request. @@ -281,26 +281,26 @@ Apache Wicket: http://wicket.apache.org/ XSS (Cross Site Scripting) Prevention Cheat Sheet: http://www.owasp.org/index.php/XSS_(Cross_Site_Scripting)_Prevention_Cheat_Sheet DOM based XSS Prevention Cheat Sheet: http://www.owasp.org/index.php/DOM_based_XSS_Prevention_Cheat_Sheet Top 25 series - Rank 1 - Cross Site Scripting: http://blogs.sans.org/appsecstreetfighter/2010/02/22/top-25-series-rank-1-cross-site-scripting/ -The Art of Software Security Assessment: Chapter 17, ""Cross Site Scripting"", Page 1071." +The Art of Software Security Assessment: Chapter 17, ""Cross Site Scripting"", Page 1071." CWE-80,EN-Improper Neutralization of Script-Related HTML Tags in a Web Page (Basic XSS) (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special characters such as ""<"", "">"", and ""&"" that could be interpreted as web-scripting elements when they are sent to a downstream component that processes web pages. -This may allow such characters to be treated as control characters, which are executed client-side in the context of the user's session. Although this can be classified as an injection problem, the more pertinent issue is the improper conversion of such special characters to respective context-appropriate entities before displaying them to the user.",,high, +This may allow such characters to be treated as control characters, which are executed client-side in the context of the user's session. Although this can be classified as an injection problem, the more pertinent issue is the improper conversion of such special characters to respective context-appropriate entities before displaying them to the user.",,high, CWE-98,EN-Improper Control of Filename for Include/Require Statement in PHP Program (PHP Remote File Inclusion) (Type: Base),"The PHP application receives input from an upstream component, but it does not restrict or incorrectly restricts the input before its usage in ""require,"" ""include,"" or similar functions. In certain versions and configurations of PHP, this can allow an attacker to specify a URL to a remote location from which the software will obtain the code to execute. In other cases in association with path traversal, the attacker can specify a local file that may contain executable statements that can be parsed by PHP.",,high,"Testing for Path Traversal (OWASP-AZ-001): http://www.owasp.org/index.php/Testing_for_Path_Traversal_(OWASP-AZ-001) Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html A Study in Scarlet: http://www.cgisecurity.com/lib/studyinscarlet.txt Suhosin: http://www.hardened-php.net/suhosin/ -Top 25 Series - Rank 13 - PHP File Inclusion: http://blogs.sans.org/appsecstreetfighter/2010/03/11/top-25-series-rank-13-php-file-inclusion/" +Top 25 Series - Rank 13 - PHP File Inclusion: http://blogs.sans.org/appsecstreetfighter/2010/03/11/top-25-series-rank-13-php-file-inclusion/" CWE-188,EN-Reliance on Data/Memory Layout (Type: Base),"The software makes invalid assumptions about how protocol data or memory is organized at a lower level, resulting in unintended program behavior. -For example, an attacker might succeed in authentication by providing a small password that matches the associated portion of the larger, correct password.",,low,"The Art of Software Security Assessment: Chapter 6, ""Structure Padding"", Page 284." +For example, an attacker might succeed in authentication by providing a small password that matches the associated portion of the larger, correct password.",,low,"The Art of Software Security Assessment: Chapter 6, ""Structure Padding"", Page 284." CWE-197,EN-Numeric Truncation Error (Type: Base),"Truncation errors occur when a primitive is cast to a primitive of a smaller size and data is lost in the conversion. -When a primitive is cast to a smaller primitive, the high order bits of the large value are lost in the conversion, potentially resulting in an unexpected value that is not equal to the original value. This value may be required as an index into a buffer, a loop iterator, or simply necessary state data. In any case, the value cannot be trusted and the system will be in an undefined state. While this method may be employed viably to isolate the low bits of a value, this usage is rare, and truncation usually implies that an implementation error has occurred.",,low,"The Art of Software Security Assessment: Chapter 6, ""Truncation"", Page 259." +When a primitive is cast to a smaller primitive, the high order bits of the large value are lost in the conversion, potentially resulting in an unexpected value that is not equal to the original value. This value may be required as an index into a buffer, a loop iterator, or simply necessary state data. In any case, the value cannot be trusted and the system will be in an undefined state. While this method may be employed viably to isolate the low bits of a value, this usage is rare, and truncation usually implies that an implementation error has occurred.",,low,"The Art of Software Security Assessment: Chapter 6, ""Truncation"", Page 259." CWE-252,EN-Unchecked Return Value (Type: Base),"The software does not check the return value from a method or function, which can prevent it from detecting unexpected states and conditions. Two common programmer assumptions are ""this function call can never fail"" and ""it doesn't matter if this function call fails"". If an attacker can force the function to fail or otherwise return a value that is not expected, then the subsequent program logic could lead to a vulnerability, because the software is not in a state that the programmer assumes. For example, if the program calls a function to drop privileges but does not check the return code to ensure that privileges were successfully dropped, then the program will continue to operate with the higher privileges.",,low,"The Art of Software Security Assessment: Chapter 7, ""Program Building Blocks"" Page 341. Writing Secure Code: Chapter 20, ""Checking Returns"" Page 624 24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183 -ERR10-CPP. Check for error conditions: https://www.securecoding.cert.org/confluence/display/cplusplus/ERR10-CPP.+Check+for+error+conditions" +ERR10-CPP. Check for error conditions: https://www.securecoding.cert.org/confluence/display/cplusplus/ERR10-CPP.+Check+for+error+conditions" CWE-253,EN-Incorrect Check of Function Return Value (Type: Base),"The software incorrectly checks a return value from a function, which prevents the software from detecting errors or exceptional conditions. -Two common programmer assumptions are ""this function call can never fail"" and ""it doesn't matter if this function call fails"". If an attacker can force the function to fail or otherwise return a value that is not expected, then the subsequent program logic could lead to a vulnerability, because the software is not in a state that the programmer assumes. For example, if the program calls a function to drop privileges but does not check the return code to ensure that privileges were successfully dropped, then the program will continue to operate with the higher privileges.",,low,"The Art of Software Security Assessment: Chapter 7, ""Return Value Testing and Interpretation"", Page 340." +Two common programmer assumptions are ""this function call can never fail"" and ""it doesn't matter if this function call fails"". If an attacker can force the function to fail or otherwise return a value that is not expected, then the subsequent program logic could lead to a vulnerability, because the software is not in a state that the programmer assumes. For example, if the program calls a function to drop privileges but does not check the return code to ensure that privileges were successfully dropped, then the program will continue to operate with the higher privileges.",,low,"The Art of Software Security Assessment: Chapter 7, ""Return Value Testing and Interpretation"", Page 340." CWE-296,EN-Improper Following of a Certificates Chain of Trust (Type: Base),"The software does not follow, or incorrectly follows, the chain of trust for a certificate back to a trusted root certificate, resulting in incorrect trust of any resource that is associated with that certificate. If a system does not follow the chain of trust of a certificate to a root server, the certificate loses all usefulness as a metric of trust. Essentially, the trust gained from a certificate is derived from a chain of trust -- with a reputable trusted entity at the end of that list. The end user must trust that reputable source, and this reputable source must vouch for the resource in question through the medium of the certificate. In some cases, this trust traverses several entities who vouch for one another. The entity trusted by the end user is at one end of this trust chain, while the certificate-wielding resource is at the other end of the chain. If the user receives a certificate at the end of one of these trust chains and then proceeds to check only that the first link in the chain, no real trust has been derived, since the entire chain must be traversed back to a trusted source to verify the certificate. @@ -309,80 +309,80 @@ Any certificate in the chain is self-signed, unless it the root. Not every intermediate certificate is checked, starting from the original certificate all the way up to the root certificate. An intermediate, CA-signed certificate does not have the expected Basic Constraints or other important extensions. The root certificate has been compromised or authorized to the wrong party.",,low,"The Most Dangerous Code in the World: Validating SSL Certificates in Non-Browser Software: http://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf -24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347" +24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347" CWE-298,EN-Improper Validation of Certificate Expiration (Type: Variant),"A certificate expiration is not validated or is incorrectly validated, so trust may be assigned to certificates that have been abandoned due to age. -When the expiration of a certificate is not taken into account, no trust has necessarily been conveyed through it. Therefore, the validity of the certificate cannot be verified and all benefit of the certificate is lost.",,low,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347" +When the expiration of a certificate is not taken into account, no trust has necessarily been conveyed through it. Therefore, the validity of the certificate cannot be verified and all benefit of the certificate is lost.",,low,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347" CWE-324,EN-Use of a Key Past its Expiration Date (Type: Base),"The product uses a cryptographic key or password past its expiration date, which diminishes its safety significantly by increasing the timing window for cracking attacks against that key. -While the expiration of keys does not necessarily ensure that they are compromised, it is a significant concern that keys which remain in use for prolonged periods of time have a decreasing probability of integrity. For this reason, it is important to replace keys within a period of time proportional to their strength.",,low,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347" +While the expiration of keys does not necessarily ensure that they are compromised, it is a significant concern that keys which remain in use for prolonged periods of time have a decreasing probability of integrity. For this reason, it is important to replace keys within a period of time proportional to their strength.",,low,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347" CWE-379,EN-Creation of Temporary File in Directory with Incorrect Permissions (Type: Base),"The software creates a temporary file in a directory whose permissions allow unintended actors to determine the file's existence or otherwise access that file. -On some operating systems, the fact that the temporary file exists may be apparent to any user with sufficient privileges to access that directory. Since the file is visible, the application that is using the temporary file could be known. If one has access to list the processes on the system, the attacker has gained information about what the user is doing at that time. By correlating this with the applications the user is running, an attacker could potentially discover what a user's actions are. From this, higher levels of security could be breached.",,low,"The Art of Software Security Assessment: Chapter 9, ""Temporary Files"", Page 538." +On some operating systems, the fact that the temporary file exists may be apparent to any user with sufficient privileges to access that directory. Since the file is visible, the application that is using the temporary file could be known. If one has access to list the processes on the system, the attacker has gained information about what the user is doing at that time. By correlating this with the applications the user is running, an attacker could potentially discover what a user's actions are. From this, higher levels of security could be breached.",,low,"The Art of Software Security Assessment: Chapter 9, ""Temporary Files"", Page 538." CWE-462,EN-Duplicate Key in Associative List (Alist) (Type: Base),"Duplicate keys in associative lists can lead to non-unique keys being mistaken for an error. -A duplicate key entry -- if the alist is designed properly -- could be used as a constant time replace function. However, duplicate key entries could be inserted by mistake. Because of this ambiguity, duplicate key entries in an association list are not recommended and should not be allowed.",,low, +A duplicate key entry -- if the alist is designed properly -- could be used as a constant time replace function. However, duplicate key entries could be inserted by mistake. Because of this ambiguity, duplicate key entries in an association list are not recommended and should not be allowed.",,low, CWE-479,EN-Signal Handler Use of a Non-reentrant Function (Type: Variant),"The program defines a signal handler that calls a non-reentrant function. Non-reentrant functions are functions that cannot safely be called, interrupted, and then recalled before the first call has finished without resulting in memory corruption. This can lead to an unexpected system state an unpredictable results with a variety of potential consequences depending on context, including denial of service and code execution. -Many functions are not reentrant, but some of them can result in the corruption of memory if they are used in a signal handler. The function call syslog() is an example of this. In order to perform its functionality, it allocates a small amount of memory as ""scratch space."" If syslog() is suspended by a signal call and the signal handler calls syslog(), the memory used by both of these functions enters an undefined, and possibly, exploitable state. Implementations of malloc() and free() manage metadata in global structures in order to track which memory is allocated versus which memory is available, but they are non-reentrant. Simultaneous calls to these functions can cause corruption of the metadata.",,low,"The Art of Software Security Assessment: Chapter 13, ""Signal Vulnerabilities"", Page 791." +Many functions are not reentrant, but some of them can result in the corruption of memory if they are used in a signal handler. The function call syslog() is an example of this. In order to perform its functionality, it allocates a small amount of memory as ""scratch space."" If syslog() is suspended by a signal call and the signal handler calls syslog(), the memory used by both of these functions enters an undefined, and possibly, exploitable state. Implementations of malloc() and free() manage metadata in global structures in order to track which memory is allocated versus which memory is available, but they are non-reentrant. Simultaneous calls to these functions can cause corruption of the metadata.",,low,"The Art of Software Security Assessment: Chapter 13, ""Signal Vulnerabilities"", Page 791." CWE-480,EN-Use of Incorrect Operator (Type: Base),"The programmer accidentally uses the wrong operator, which changes the application logic in security-relevant ways. Non-reentrant functions are functions that cannot safely be called, interrupted, and then recalled before the first call has finished without resulting in memory corruption. This can lead to an unexpected system state an unpredictable results with a variety of potential consequences depending on context, including denial of service and code execution. -Many functions are not reentrant, but some of them can result in the corruption of memory if they are used in a signal handler. The function call syslog() is an example of this. In order to perform its functionality, it allocates a small amount of memory as ""scratch space."" If syslog() is suspended by a signal call and the signal handler calls syslog(), the memory used by both of these functions enters an undefined, and possibly, exploitable state. Implementations of malloc() and free() manage metadata in global structures in order to track which memory is allocated versus which memory is available, but they are non-reentrant. Simultaneous calls to these functions can cause corruption of the metadata.",,low,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289." +Many functions are not reentrant, but some of them can result in the corruption of memory if they are used in a signal handler. The function call syslog() is an example of this. In order to perform its functionality, it allocates a small amount of memory as ""scratch space."" If syslog() is suspended by a signal call and the signal handler calls syslog(), the memory used by both of these functions enters an undefined, and possibly, exploitable state. Implementations of malloc() and free() manage metadata in global structures in order to track which memory is allocated versus which memory is available, but they are non-reentrant. Simultaneous calls to these functions can cause corruption of the metadata.",,low,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289." CWE-481,EN-Assigning instead of Comparing (Type: Variant),"The code uses an operator for assignment when the intention was to perform a comparison. -In many languages the compare statement is very close in appearance to the assignment statement and are often confused. This bug is generally the result of a typo and usually causes obvious problems with program execution. If the comparison is in an if statement, the if statement will usually evaluate the value of the right-hand side of the predicate.",,low,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289." +In many languages the compare statement is very close in appearance to the assignment statement and are often confused. This bug is generally the result of a typo and usually causes obvious problems with program execution. If the comparison is in an if statement, the if statement will usually evaluate the value of the right-hand side of the predicate.",,low,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289." CWE-482,EN-Comparing instead of Assigning (Type: Variant),"The code uses an operator for comparison when the intention was to perform an assignment. -In many languages, the compare statement is very close in appearance to the assignment statement; they are often confused.",,low,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289." +In many languages, the compare statement is very close in appearance to the assignment statement; they are often confused.",,low,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289." CWE-483,EN-Incorrect Block Delimitation (Type: Variant),"The code does not explicitly delimit a block that is intended to contain 2 or more statements, creating a logic error. -In some languages, braces (or other delimiters) are optional for blocks. When the delimiter is omitted, it is possible to insert a logic error in which a statement is thought to be in a block but is not. In some cases, the logic error can have security implications.",,low, +In some languages, braces (or other delimiters) are optional for blocks. When the delimiter is omitted, it is possible to insert a logic error in which a statement is thought to be in a block but is not. In some cases, the logic error can have security implications.",,low, CWE-641,EN-Improper Restriction of Names for Files and Other Resources (Type: Base),"The application constructs the name of a file or other resource using input from an upstream component, but does not restrict or incorrectly restricts the resulting name. -This may produce resultant weaknesses. For instance, if the names of these resources contain scripting characters, it is possible that a script may get executed in the client's browser if the application ever displays the name of the resource on a dynamically generated web page. Alternately, if the resources are consumed by some application parser, a specially crafted name can exploit some vulnerability internal to the parser, potentially resulting in execution of arbitrary code on the server machine. The problems will vary based on the context of usage of such malformed resource names and whether vulnerabilities are present in or assumptions are made by the targeted technology that would make code execution possible.",,low, +This may produce resultant weaknesses. For instance, if the names of these resources contain scripting characters, it is possible that a script may get executed in the client's browser if the application ever displays the name of the resource on a dynamically generated web page. Alternately, if the resources are consumed by some application parser, a specially crafted name can exploit some vulnerability internal to the parser, potentially resulting in execution of arbitrary code on the server machine. The problems will vary based on the context of usage of such malformed resource names and whether vulnerabilities are present in or assumptions are made by the targeted technology that would make code execution possible.",,low, CWE-648,EN-Incorrect Use of Privileged APIs (Type: Base),"The application does not conform to the API requirements for a function call that requires extra privileges. This could allow attackers to gain privileges by causing the function to be called incorrectly. When an application contains certain functions that perform operations requiring an elevated level of privilege, the caller of a privileged API must be careful to: ensure that assumptions made by the APIs are valid, such as validity of arguments account for known weaknesses in the design/implementation of the API call the API from a safe context If the caller of the API does not follow these requirements, then it may allow a malicious user or process to elevate their privilege, hijack the process, or steal sensitive data. -For instance, it is important to know if privileged APIs do not shed their privileges before returning to the caller or if the privileged function might make certain assumptions about the data, context or state information passed to it by the caller. It is important to always know when and how privileged APIs can be called in order to ensure that their elevated level of privilege cannot be exploited.",,low, +For instance, it is important to know if privileged APIs do not shed their privileges before returning to the caller or if the privileged function might make certain assumptions about the data, context or state information passed to it by the caller. It is important to always know when and how privileged APIs can be called in order to ensure that their elevated level of privilege cannot be exploited.",,low, CWE-762,EN-Mismatched Memory Management Routines (Type: Variant),"The application attempts to return a memory resource to the system, but it calls a release function that is not compatible with the function that was originally used to allocate that resource. This weakness can be generally described as mismatching memory management routines, such as: The memory was allocated on the stack (automatically), but it was deallocated using the memory management routine free() (CWE-590), which is intended for explicitly allocated heap memory. The memory was allocated explicitly using one set of memory management functions, and deallocated using a different set. For example, memory might be allocated with malloc() in C++ instead of the new operator, and then deallocated with the delete operator. When the memory management functions are mismatched, the consequences may be as severe as code execution, memory corruption, or program crash. Consequences and ease of exploit will vary depending on the implementation of the routines and the object being managed.",,low,"boost C++ Library Smart Pointers: http://www.boost.org/doc/libs/1_38_0/libs/smart_ptr/smart_ptr.htm -Valgrind: http://valgrind.org/" +Valgrind: http://valgrind.org/" CWE-783,EN-Operator Precedence Logic Error (Type: Variant),"The program uses an expression in which operator precedence causes incorrect logic to be used. While often just a bug, operator precedence logic errors can have serious consequences if they are used in security-critical code, such as making an authentication decision.",,low,"EXP00-C. Use parentheses for precedence of operation: https://www.securecoding.cert.org/confluence/display/seccode/EXP00-C.+Use+parentheses+for+precedence+of+operation -The Art of Software Security Assessment: Chapter 6, ""Precedence"", Page 287." +The Art of Software Security Assessment: Chapter 6, ""Precedence"", Page 287." CWE-789,EN-Uncontrolled Memory Allocation (Type: Variant),"The product allocates memory based on an untrusted size value, but it does not validate or incorrectly validates the size, allowing arbitrary amounts of memory to be allocated. -This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. These problems may be resultant from missing sentinel values (CWE-463) or trusting a user-influenced input length variable.",,low,"The Art of Software Security Assessment: Chapter 10, ""Resource Limits"", Page 574." +This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. These problems may be resultant from missing sentinel values (CWE-463) or trusting a user-influenced input length variable.",,low,"The Art of Software Security Assessment: Chapter 10, ""Resource Limits"", Page 574." CWE-333,EN-Improper Handling of Insufficient Entropy in TRNG (Type: Variant),"True random number generators (TRNG) generally have a limited source of entropy and therefore can fail or block. -The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,medium, +The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,medium, CWE-367,EN-Time-of-check Time-of-use (TOCTOU) Race Condition (Type: Base),"The software checks the state of a resource before using that resource, but the resource's state can change between the check and the use in a way that invalidates the results of the check. This can cause the software to perform invalid actions when the resource is in an unexpected state. This weakness can be security-relevant when an attacker can influence the state of the resource between check and use. This can happen with shared resources such as files, memory, or even variables in multithreaded programs.",,medium,"Portably Solving File TOCTTOU Races with Hardness Amplification: http://www.usenix.org/events/fast08/tech/tsafrir.html 24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205 -The Art of Software Security Assessment: Chapter 9, ""TOCTOU"", Page 527." +The Art of Software Security Assessment: Chapter 9, ""TOCTOU"", Page 527." CWE-404,EN-Improper Resource Shutdown or Release (Type: Base),"The program does not release or incorrectly releases a resource before it is made available for re-use. -When a resource is created or allocated, the developer is responsible for properly releasing the resource as well as accounting for all potential paths of expiration or invalidation, such as a set period of time or revocation.",,medium,"24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143" +When a resource is created or allocated, the developer is responsible for properly releasing the resource as well as accounting for all potential paths of expiration or invalidation, such as a set period of time or revocation.",,medium,"24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143" CWE-407,EN-Algorithmic Complexity (Type: Base),"An algorithm in a product has an inefficient worst-case computational complexity that may be detrimental to system performance and can be triggered by an attacker, typically using crafted manipulations that ensure that the worst case is being reached. -In the absence of a policy to restrict asymmetric resource consumption, the application or system cannot distinguish between legitimate transmissions and traffic intended to serve as an amplifying attack on target systems. Systems can often be configured to restrict the amount of traffic sent out on behalf of a client, based on the client's origin or access level. This is usually defined in a resource allocation policy. In the absence of a mechanism to keep track of transmissions, the system or application can be easily abused to transmit asymmetrically greater traffic than the request or client should be permitted to.",,medium,Algorithmic Complexity Attacks: http://www.cs.rice.edu/~scrosby/hash/CrosbyWallach_UsenixSec2003/index.html +In the absence of a policy to restrict asymmetric resource consumption, the application or system cannot distinguish between legitimate transmissions and traffic intended to serve as an amplifying attack on target systems. Systems can often be configured to restrict the amount of traffic sent out on behalf of a client, based on the client's origin or access level. This is usually defined in a resource allocation policy. In the absence of a mechanism to keep track of transmissions, the system or application can be easily abused to transmit asymmetrically greater traffic than the request or client should be permitted to.",,medium,Algorithmic Complexity Attacks: http://www.cs.rice.edu/~scrosby/hash/CrosbyWallach_UsenixSec2003/index.html CWE-415,EN-Double Free (Type: Variant),"The product calls free() twice on the same memory address, potentially leading to modification of unexpected memory locations. When a program calls free() twice with the same argument, the program's memory management data structures become corrupted. This corruption can cause the program to crash or, in some circumstances, cause two later calls to malloc() to return the same pointer. If malloc() returns the same value twice and the program later gives the attacker control over the data that is written into this doubly-allocated memory, the program becomes vulnerable to a buffer overflow attack.",,medium,"24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143 -The Art of Software Security Assessment: Chapter 7, ""Double Frees"", Page 379." +The Art of Software Security Assessment: Chapter 7, ""Double Frees"", Page 379." CWE-59,EN-Improper Link Resolution Before File Access (Link Following) (Type: Base),"The software attempts to access a file based on the filename, but it does not properly prevent that filename from identifying a link or shortcut that resolves to an unintended resource. -Some functions that offer security features supported by the OS are not available on all versions of the OS in common use. Likewise, functions are often deprecated or made obsolete for security reasons and should not be used.",,medium,"The Art of Software Security Assessment: Chapter 9, ""Symbolic Link Attacks"", Page 518." +Some functions that offer security features supported by the OS are not available on all versions of the OS in common use. Likewise, functions are often deprecated or made obsolete for security reasons and should not be used.",,medium,"The Art of Software Security Assessment: Chapter 9, ""Symbolic Link Attacks"", Page 518." CWE-601,EN-URL Redirection to Untrusted Site (Open Redirect) (Type: Variant),"A web application accepts a user-controlled input that specifies a link to an external site, and uses that link in a Redirect. This simplifies phishing attacks. An http parameter may contain a URL value and could cause the web application to redirect the request to the specified URL. By modifying the URL value to a malicious site, an attacker may successfully launch a phishing scam and steal user credentials. Because the server name in the modified link is identical to the original site, phishing attempts have a more trustworthy appearance.",,medium,"Exploitable Redirects on the Web: Identification, Prevalence, and Defense: http://www.cs.indiana.edu/cgi-pub/cshue/research/woot08.pdf Open redirect vulnerabilities: definition and prevention: http://www.net-security.org/dl/insecure/INSECURE-Mag-17.pdf Top 25 Series - Rank 23 - Open Redirect: http://software-security.sans.org/blog/2010/03/25/top-25-series-rank-23-open-redirect -OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI" +OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI" CWE-749,EN-Exposed Dangerous Method or Function (Type: Base),"The software provides an Applications Programming Interface (API) or similar interface for interaction with external actors, but the interface includes a dangerous method or function that is not properly restricted. This weakness can lead to a wide variety of resultant weaknesses, depending on the behavior of the exposed method. It can apply to any number of technologies and approaches, such as ActiveX controls, Java functions, IOCTLs, and so on. The exposure can occur in a few different ways: 1) The function/method was never intended to be exposed to outside actors. 2) The function/method was only intended to be accessible to a limited set of actors, such as Internet-based access from a single web site.",,medium,"No description: http://msdn.microsoft.com/workshop/components/activex/safety.asp -No description: http://msdn.microsoft.com/workshop/components/activex/security.asp" +No description: http://msdn.microsoft.com/workshop/components/activex/security.asp" CWE-755,EN-Improper Handling of Exceptional Conditions (Type: Class),"The software does not handle or incorrectly handles an exceptional condition. The programmer may assume that certain events or conditions will never occur or do not need to be worried about, such as low memory conditions, lack of access to resources due to restrictive permissions, or misbehaving clients or components. However, attackers may intentionally trigger these unusual conditions, thus violating the programmer's assumptions, possibly introducing instability, incorrect behavior, or a vulnerability. -Note that this entry is not exclusively about the use of exceptions and exception handling, which are mechanisms for both checking and handling unusual or unexpected conditions.",,medium, +Note that this entry is not exclusively about the use of exceptions and exception handling, which are mechanisms for both checking and handling unusual or unexpected conditions.",,medium, CWE-766,EN-Critical Variable Declared Public (Type: Variant),"The software declares a critical variable or field to be public when intended security policy requires it to be private. -When software is operating in a concurrent environment and repeatedly unlocks a critical resource, the consequences will vary based on the type of lock, the lock's implementation, and the resource being protected. In some situations such as with semaphores, the resources are pooled and extra calls to unlock will increase the count for the number of available resources, likely resulting in a crash or unpredictable behavior when the system nears capacity.",,medium, +When software is operating in a concurrent environment and repeatedly unlocks a critical resource, the consequences will vary based on the type of lock, the lock's implementation, and the resource being protected. In some situations such as with semaphores, the resources are pooled and extra calls to unlock will increase the count for the number of available resources, likely resulting in a crash or unpredictable behavior when the system nears capacity.",,medium, CWE-767,EN-Access to Critical Private Variable via Public Method (Type: Variant),"The software defines a public method that reads or modifies a private variable. -If an attacker modifies the variable to contain unexpected values, this could violate assumptions from other parts of the code. Additionally, if an attacker can read the private variable, it may expose sensitive information or make it easier to launch further attacks.",,medium, +If an attacker modifies the variable to contain unexpected values, this could violate assumptions from other parts of the code. Additionally, if an attacker can read the private variable, it may expose sensitive information or make it easier to launch further attacks.",,medium, CWE-776,EN-Improper Restriction of Recursive Entity References in DTDs (XML Entity Expansion) (Type: Variant),"The software uses XML documents and allows their structure to be defined with a Document Type Definition (DTD), but it does not properly control the number of recursive definitions of entities. If the DTD contains a large number of nested or recursive entities, this can lead to explosive growth of data when parsed, causing a denial of service.",,medium,"Multiple vendors XML parser (and SOAP/WebServices server) Denial of Service attack using DTD: http://www.securityfocus.com/archive/1/303509 XML security: Preventing XML bombs: http://searchsoftwarequality.techtarget.com/expert/KnowledgebaseAnswer/0,289625,sid92_gci1168442,00.html?asrc=SS_CLA_302%20%20558&psrc=CLT_92# @@ -390,11 +390,11 @@ Dismantling an XML-Bomb: http://blog.didierstevens.com/2008/09/23/dismantling-an XML Entity Expansion: http://projects.webappsec.org/XML-Entity-Expansion Tip: Configure SAX parsers for secure processing: http://www.ibm.com/developerworks/xml/library/x-tipcfsx.html XML Denial of Service Attacks and Defenses: http://msdn.microsoft.com/en-us/magazine/ee335713.aspx -Preventing Entity Expansion Attacks in JAXB: http://blog.bdoughan.com/2011/03/preventing-entity-expansion-attacks-in.html" +Preventing Entity Expansion Attacks in JAXB: http://blog.bdoughan.com/2011/03/preventing-entity-expansion-attacks-in.html" CWE-777,EN-Regular Expression without Anchors (Type: Variant),"The software uses a regular expression to perform neutralization, but the regular expression is not anchored and may allow malicious or malformed data to slip through. -When performing tasks such as whitelist validation, data is examined and possibly modified to ensure that it is well-formed and adheres to a list of safe values. If the regular expression is not anchored, malicious or malformed data may be included before or after any string matching the regular expression. The type of malicious data that is allowed will depend on the context of the application and which anchors are omitted from the regular expression.",,medium, +When performing tasks such as whitelist validation, data is examined and possibly modified to ensure that it is well-formed and adheres to a list of safe values. If the regular expression is not anchored, malicious or malformed data may be included before or after any string matching the regular expression. The type of malicious data that is allowed will depend on the context of the application and which anchors are omitted from the regular expression.",,medium, CWE-779,EN-Logging of Excessive Data (Type: Base),"The software logs too much information, making log files hard to process and possibly hindering recovery efforts or forensic analysis after an attack. -While logging is a good practice in general, and very high levels of logging are appropriate for debugging stages of development, too much logging in a production environment might hinder a system administrator's ability to detect anomalous conditions. This can provide cover for an attacker while attempting to penetrate a system, clutter the audit trail for forensic analysis, or make it more difficult to debug problems in a production environment.",,medium, +While logging is a good practice in general, and very high levels of logging are appropriate for debugging stages of development, too much logging in a production environment might hinder a system administrator's ability to detect anomalous conditions. This can provide cover for an attacker while attempting to penetrate a system, clutter the audit trail for forensic analysis, or make it more difficult to debug problems in a production environment.",,medium, CWE-781,EN-Improper Address Validation in IOCTL with METHOD_NEITHER I/O Control Code (Type: Variant),"The software defines an IOCTL that uses METHOD_NEITHER for I/O, but it does not validate or incorrectly validates the addresses that are provided. When an IOCTL uses the METHOD_NEITHER option for I/O control, it is the responsibility of the IOCTL to validate the addresses that have been supplied to it. If validation is missing or incorrect, attackers can supply arbitrary memory addresses, leading to code execution or a denial of service.",,medium,"Exploiting Common Flaws in Drivers: http://reversemode.com/index.php?option=com_content&task=view&id=38&Itemid=1 Remote and Local Exploitation of Network Drivers: https://www.blackhat.com/presentations/bh-usa-07/Bulygin/Presentation/bh-usa-07-bulygin.pdf @@ -402,25 +402,25 @@ Windows driver vulnerabilities: the METHOD_NEITHER odyssey: http://www.net-secur Buffer Descriptions for I/O Control Codes: http://msdn.microsoft.com/en-us/library/ms795857.aspx Using Neither Buffered Nor Direct I/O: http://msdn.microsoft.com/en-us/library/cc264614.aspx Securing Device Objects: http://msdn.microsoft.com/en-us/library/ms794722.aspx -No description: http://www.piotrbania.com/all/articles/ewdd.pdf" +No description: http://www.piotrbania.com/all/articles/ewdd.pdf" CWE-782,EN-Exposed IOCTL with Insufficient Access Control (Type: Variant),"The software implements an IOCTL with functionality that should be restricted, but it does not properly enforce access control for the IOCTL. When an IOCTL contains privileged functionality and is exposed unnecessarily, attackers may be able to access this functionality by invoking the IOCTL. Even if the functionality is benign, if the programmer has assumed that the IOCTL would only be accessed by a trusted process, there may be little or no validation of the incoming data, exposing weaknesses that would never be reachable if the attacker cannot call the IOCTL directly. -The implementations of IOCTLs will differ between operating system types and versions, so the methods of attack and prevention may vary widely.",,medium,Securing Device Objects: http://msdn.microsoft.com/en-us/library/ms794722.aspx +The implementations of IOCTLs will differ between operating system types and versions, so the methods of attack and prevention may vary widely.",,medium,Securing Device Objects: http://msdn.microsoft.com/en-us/library/ms794722.aspx CWE-117,EN-Improper Output Neutralization for Logs (Type: Base),"The software does not neutralize or incorrectly neutralizes output that is written to logs. This can allow an attacker to forge log entries or inject malicious content into logs. Log forging vulnerabilities occur when: Data enters an application from an untrusted source. The data is written to an application or system log file.",,medium,"Exploiting Software: How to Break Code The night the log was forged: http://doc.novsu.ac.ru/oreilly/tcpip/puis/ch10_05.htm -OWASP TOP 10: http://www.owasp.org/index.php/Top_10_2007" +OWASP TOP 10: http://www.owasp.org/index.php/Top_10_2007" CWE-124,EN-Buffer Underwrite (Buffer Underflow) (Type: Base),"The software writes to a buffer using an index or pointer that references a memory location prior to the beginning of the buffer. This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used.",,medium,"Buffer UNDERFLOWS: What do you know about it?: http://seclists.org/vuln-dev/2004/Jan/0022.html -24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89" +24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89" CWE-128,EN-Wrap-around Error (Type: Base),"Wrap around errors occur whenever a value is incremented past the maximum value for its type and therefore ""wraps around"" to a very small, negative, or undefined value. This typically occurs when the pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. This may result in exposure of sensitive information or possibly a crash.",,medium,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89 -The Art of Software Security Assessment: Chapter 6, ""Signed Integer Boundaries"", Page 220." +The Art of Software Security Assessment: Chapter 6, ""Signed Integer Boundaries"", Page 220." CWE-170,EN-Improper Null Termination (Type: Base),"The software does not terminate or incorrectly terminates a string or array with a null character or equivalent terminator. -Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,medium, +Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,medium, CWE-190,EN-Integer Overflow or Wraparound (Type: Base),"The software performs a calculation that can produce an integer overflow or wraparound, when the logic assumes that the resulting value will always be larger than the original value. This can introduce other weaknesses when the calculation is used for resource management or execution control. An integer overflow or wraparound occurs when an integer value is incremented to a value that is too large to store in the associated representation. When this occurs, the value may wrap to become a very small or negative number. While this may be intended behavior in circumstances that rely on wrapping, it can have security consequences if the wrap is unexpected. This is especially the case if the integer overflow can be triggered using user-supplied inputs. This becomes security-critical when the result is used to control looping, make a security decision, or determine the offset or size in behaviors such as memory allocation, copying, concatenation, etc.",,medium,"An overview of common programming security vulnerabilities and possible solutions: http://fort-knox.org/thesis.pdf Basic Integer Overflows: http://www.phrack.org/issues.html?issue=60&id=10#article @@ -428,11 +428,11 @@ Writing Secure Code: Chapter 20, ""Integer Overflows"" Page 620 24 Deadly Sins of Software Security: ""Sin 7: Integer Overflows."" Page 119 SafeInt: http://safeint.codeplex.com/ Top 25 Series - Rank 17 - Integer Overflow Or Wraparound: http://software-security.sans.org/blog/2010/03/18/top-25-series-rank-17-integer-overflow-or-wraparound -The Art of Software Security Assessment: Chapter 6, ""Signed Integer Boundaries"", Page 220." +The Art of Software Security Assessment: Chapter 6, ""Signed Integer Boundaries"", Page 220." CWE-196,EN-Unsigned to Signed Conversion Error (Type: Variant),"An unsigned-to-signed conversion error takes place when a large unsigned primitive is used as a signed value. -It is dangerous to rely on implicit casts between signed and unsigned numbers because the result can take on an unexpected value and violate assumptions made by the program.",,medium,"The Art of Software Security Assessment: Chapter 6, ""Type Conversions"", Page 223." +It is dangerous to rely on implicit casts between signed and unsigned numbers because the result can take on an unexpected value and violate assumptions made by the program.",,medium,"The Art of Software Security Assessment: Chapter 6, ""Type Conversions"", Page 223." CWE-202,EN-Exposure of Sensitive Data Through Data Queries (Type: Variant),"When trying to keep information confidential, an attacker can often infer some of the information by using statistics. -In situations where data should not be tied to individual users, but a large number of users should be able to make queries that ""scrub"" the identity of users, it may be possible to get information about a user -- e.g., by specifying search terms that are known to be unique to that user.",,medium, +In situations where data should not be tied to individual users, but a large number of users should be able to make queries that ""scrub"" the identity of users, it may be possible to get information about a user -- e.g., by specifying search terms that are known to be unique to that user.",,medium, CWE-250,EN-Execution with Unnecessary Privileges (Type: Class),"The software performs an operation at a privilege level that is higher than the minimum level required, which creates new weaknesses or amplifies the consequences of other weaknesses. New weaknesses can be exposed because running with extra privileges, such as root or Administrator, can disable the normal security checks being performed by the operating system or surrounding environment. Other pre-existing weaknesses can turn into security vulnerabilities if they occur while operating at raised privileges. Privilege management functions can behave in some less-than-obvious ways, and they have different quirks on different platforms. These inconsistencies are particularly pronounced if you are transitioning from one non-root user to another. Signal handlers and spawned processes run at the privilege of the owning process, so if a process is running as root when a signal fires or a sub-process is executed, the signal handler or sub-process will operate with root privileges.",,medium,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ @@ -440,28 +440,28 @@ Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledg Writing Secure Code: Chapter 7, ""Running with Least Privilege"" Page 207 Federal Desktop Core Configuration: http://nvd.nist.gov/fdcc/index.cfm 24 Deadly Sins of Software Security: ""Sin 16: Executing Code With Too Much Privilege."" Page 243 -The Art of Software Security Assessment: Chapter 9, ""Privilege Vulnerabilities"", Page 477." +The Art of Software Security Assessment: Chapter 9, ""Privilege Vulnerabilities"", Page 477." CWE-269,EN-Improper Privilege Management (Type: Base),"The software does not properly assign, modify, track, or check privileges for an actor, creating an unintended sphere of control for that actor. Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,medium,"24 Deadly Sins of Software Security: ""Sin 16: Executing Code With Too Much Privilege."" Page 243 -The Art of Software Security Assessment: Chapter 9, ""Dropping Privileges Permanently"", Page 479." +The Art of Software Security Assessment: Chapter 9, ""Dropping Privileges Permanently"", Page 479." CWE-273,EN-Improper Check for Dropped Privileges (Type: Base),"The software attempts to drop privileges but does not check or incorrectly checks to see if the drop succeeded. -If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,medium, +If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,medium, CWE-276,EN-Incorrect Default Permissions (Type: Variant),"The software, upon installation, sets incorrect permissions for an object that exposes it to an unintended actor. -If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,medium,"The Art of Software Security Assessment: Chapter 3, ""Insecure Defaults"", Page 69." +If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,medium,"The Art of Software Security Assessment: Chapter 3, ""Insecure Defaults"", Page 69." CWE-299,EN-Improper Check for Certificate Revocation (Type: Variant),"The software does not check or incorrectly checks the revocation status of a certificate, which may cause it to use a certificate that has been compromised. -An improper check for certificate revocation is a far more serious flaw than related certificate failures. This is because the use of any revoked certificate is almost certainly malicious. The most common reason for certificate revocation is compromise of the system in question, with the result that no legitimate servers will be using a revoked certificate, unless they are sorely out of sync.",,medium,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347" +An improper check for certificate revocation is a far more serious flaw than related certificate failures. This is because the use of any revoked certificate is almost certainly malicious. The most common reason for certificate revocation is compromise of the system in question, with the result that no legitimate servers will be using a revoked certificate, unless they are sorely out of sync.",,medium,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347" CWE-301,EN-Reflection Attack in an Authentication Protocol (Type: Variant),"Simple authentication protocols are subject to reflection attacks if a malicious user can use the target machine to impersonate a trusted user. -A mutual authentication protocol requires each party to respond to a random challenge by the other party by encrypting it with a pre-shared key. Often, however, such protocols employ the same pre-shared key for communication with a number of different entities. A malicious user or an attacker can easily compromise this protocol without possessing the correct key by employing a reflection attack on the protocol.",,medium,"The Art of Software Security Assessment: Chapter 2, ""Insufficient Validation"", Page 38." +A mutual authentication protocol requires each party to respond to a random challenge by the other party by encrypting it with a pre-shared key. Often, however, such protocols employ the same pre-shared key for communication with a number of different entities. A malicious user or an attacker can easily compromise this protocol without possessing the correct key by employing a reflection attack on the protocol.",,medium,"The Art of Software Security Assessment: Chapter 2, ""Insufficient Validation"", Page 38." CWE-329,EN-Not Using a Random IV with CBC Mode (Type: Variant),"Not using a random initialization Vector (IV) with Cipher Block Chaining (CBC) Mode causes algorithms to be susceptible to dictionary attacks. -This weakness is especially dangerous when the hash is used in security algorithms that require the one-way property to hold. For example, if an authentication system takes an incoming password and generates a hash, then compares the hash to another hash that it has stored in its authentication database, then the ability to create a collision could allow an attacker to provide an alternate password that produces the same target hash, bypassing authentication.",,medium,"The Art of Software Security Assessment: Chapter 2, ""Initialization Vectors"", Page 42." +This weakness is especially dangerous when the hash is used in security algorithms that require the one-way property to hold. For example, if an authentication system takes an incoming password and generates a hash, then compares the hash to another hash that it has stored in its authentication database, then the ability to create a collision could allow an attacker to provide an alternate password that produces the same target hash, bypassing authentication.",,medium,"The Art of Software Security Assessment: Chapter 2, ""Initialization Vectors"", Page 42." CWE-332,EN-Insufficient Entropy in PRNG (Type: Variant),"The lack of entropy available for, or used by, a Pseudo-Random Number Generator (PRNG) can be a stability and security threat. -When software generates predictable values in a context requiring unpredictability, it may be possible for an attacker to guess the next value that will be generated, and use this guess to impersonate another user or access sensitive information.",,medium,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf +When software generates predictable values in a context requiring unpredictability, it may be possible for an attacker to guess the next value that will be generated, and use this guess to impersonate another user or access sensitive information.",,medium,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf CWE-338,EN-Use of Cryptographically Weak PRNG (Type: Base),"The product uses a Pseudo-Random Number Generator (PRNG) in a security context, but the PRNG is not cryptographically strong. -The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,medium,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" +The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,medium,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-353,EN-Missing Support for Integrity Check (Type: Base),"The software uses a transmission protocol that does not include a mechanism for verifying the integrity of the data during transmission, such as a checksum. -If integrity check values or ""checksums"" are omitted from a protocol, there is no way of determining if data has been corrupted in transmission. The lack of checksum functionality in a protocol removes the first application-level check of data that can be used. The end-to-end philosophy of checks states that integrity checks should be performed at the lowest level that they can be completely implemented. Excluding further sanity checks and input validation performed by applications, the protocol's checksum is the most important level of checksum, since it can be performed more completely than at any previous level and takes into account entire messages, as opposed to single packets.",,medium,"24 Deadly Sins of Software Security: ""Sin 15: Not Updating Easily."" Page 231" +If integrity check values or ""checksums"" are omitted from a protocol, there is no way of determining if data has been corrupted in transmission. The lack of checksum functionality in a protocol removes the first application-level check of data that can be used. The end-to-end philosophy of checks states that integrity checks should be performed at the lowest level that they can be completely implemented. Excluding further sanity checks and input validation performed by applications, the protocol's checksum is the most important level of checksum, since it can be performed more completely than at any previous level and takes into account entire messages, as opposed to single packets.",,medium,"24 Deadly Sins of Software Security: ""Sin 15: Not Updating Easily."" Page 231" CWE-354,EN-Improper Validation of Integrity Check Value (Type: Base),"The software does not validate or incorrectly validates the integrity check values or ""checksums"" of a message. This may prevent it from detecting if the data has been modified or corrupted in transmission. -Improper validation of checksums before use results in an unnecessary risk that can easily be mitigated. The protocol specification describes the algorithm used for calculating the checksum. It is then a simple matter of implementing the calculation and verifying that the calculated checksum and the received checksum match. Improper verification of the calculated checksum and the received checksum can lead to far greater consequences.",,medium, +Improper validation of checksums before use results in an unnecessary risk that can easily be mitigated. The protocol specification describes the algorithm used for calculating the checksum. It is then a simple matter of implementing the calculation and verifying that the calculated checksum and the received checksum match. Improper verification of the calculated checksum and the received checksum can lead to far greater consequences.",,medium, CWE-362,EN-Concurrent Execution using Shared Resource with Improper Synchronization (Race Condition) (Type: Class),"The program contains a code sequence that can run concurrently with other code, and the code sequence requires temporary, exclusive access to a shared resource, but a timing window exists in which the shared resource can be modified by another code sequence that is operating concurrently. This can have security implications when the expected synchronization is in security-critical code, such as recording whether a user is authenticated or modifying important state information that should not be influenced by an outsider. A race condition occurs within concurrent environments, and is effectively a property of a code sequence. Depending on the context, a code sequence may be in the form of a function call, a small number of instructions, a series of program invocations, etc. @@ -479,7 +479,7 @@ Discovering and Exploiting Named Pipe Security Flaws for Fun and Profit: http:// On Race Vulnerabilities in Web Applications: http://security.dico.unimi.it/~roberto/pubs/dimva08-web.pdf Avoiding Race Conditions and Insecure File Operations: http://developer.apple.com/documentation/Security/Conceptual/SecureCodingGuide/Articles/RaceConditions.html Top 25 Series - Rank 25 - Race Conditions: http://blogs.sans.org/appsecstreetfighter/2010/03/26/top-25-series-rank-25-race-conditions/ -Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html" +Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html" CWE-364,EN-Signal Handler Race Condition (Type: Base),"The software uses a signal handler that introduces a race condition. Race conditions frequently occur in signal handlers, since signal handlers support asynchronous actions. These race conditions have a variety of root causes and symptoms. Attackers may be able to exploit a signal handler race condition to cause the software state to be corrupted, possibly leading to a denial of service or even code execution. These issues occur when non-reentrant functions, or state-sensitive actions occur in the signal handler, where they may be called at any time. These behaviors can violate assumptions being made by the ""regular"" code that is interrupted, or by other signal handlers that may also be invoked. If these functions are called at an inopportune moment - such as while a non-reentrant function is already running - memory corruption could occur that may be exploitable for code execution. Another signal race condition commonly found occurs when free is called within a signal handler, resulting in a double free and therefore a write-what-where condition. Even if a given pointer is set to NULL after it has been freed, a race condition still exists between the time the memory was freed and the pointer was set to NULL. This is especially problematic if the same signal handler has been set for more than one signal -- since it means that the signal handler itself may be reentered. @@ -497,7 +497,7 @@ Using synchronization in the regular code Disabling or masking other signals, which provides atomicity (which effectively ensures exclusivity)",,medium,"Delivering Signals for Fun and Profit: http://lcamtuf.coredump.cx/signals.txt Race Condition: Signal Handling: http://www.fortify.com/vulncat/en/vulncat/cpp/race_condition_signal_handling.html 24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205 -The Art of Software Security Assessment: Chapter 13, ""Signal Vulnerabilities"", Page 791." +The Art of Software Security Assessment: Chapter 13, ""Signal Vulnerabilities"", Page 791." CWE-365,EN-Race Condition in Switch (Type: Base),"The code contains a switch statement in which the switched variable can be modified while the switch is still executing, resulting in unexpected behavior. Race conditions frequently occur in signal handlers, since signal handlers support asynchronous actions. These race conditions have a variety of root causes and symptoms. Attackers may be able to exploit a signal handler race condition to cause the software state to be corrupted, possibly leading to a denial of service or even code execution. These issues occur when non-reentrant functions, or state-sensitive actions occur in the signal handler, where they may be called at any time. These behaviors can violate assumptions being made by the ""regular"" code that is interrupted, or by other signal handlers that may also be invoked. If these functions are called at an inopportune moment - such as while a non-reentrant function is already running - memory corruption could occur that may be exploitable for code execution. Another signal race condition commonly found occurs when free is called within a signal handler, resulting in a double free and therefore a write-what-where condition. Even if a given pointer is set to NULL after it has been freed, a race condition still exists between the time the memory was freed and the pointer was set to NULL. This is especially problematic if the same signal handler has been set for more than one signal -- since it means that the signal handler itself may be reentered. @@ -512,7 +512,7 @@ Signal handler vulnerabilities are often classified based on the absence of a sp Avoiding shared state Using synchronization in the signal handler Using synchronization in the regular code -Disabling or masking other signals, which provides atomicity (which effectively ensures exclusivity)",,medium,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205" +Disabling or masking other signals, which provides atomicity (which effectively ensures exclusivity)",,medium,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205" CWE-366,EN-Race Condition within a Thread (Type: Base),"If two threads of execution use a resource simultaneously, there exists the possibility that resources may be used while invalid, in turn making the state of execution undefined. Race conditions frequently occur in signal handlers, since signal handlers support asynchronous actions. These race conditions have a variety of root causes and symptoms. Attackers may be able to exploit a signal handler race condition to cause the software state to be corrupted, possibly leading to a denial of service or even code execution. These issues occur when non-reentrant functions, or state-sensitive actions occur in the signal handler, where they may be called at any time. These behaviors can violate assumptions being made by the ""regular"" code that is interrupted, or by other signal handlers that may also be invoked. If these functions are called at an inopportune moment - such as while a non-reentrant function is already running - memory corruption could occur that may be exploitable for code execution. Another signal race condition commonly found occurs when free is called within a signal handler, resulting in a double free and therefore a write-what-where condition. Even if a given pointer is set to NULL after it has been freed, a race condition still exists between the time the memory was freed and the pointer was set to NULL. This is especially problematic if the same signal handler has been set for more than one signal -- since it means that the signal handler itself may be reentered. @@ -528,43 +528,43 @@ Avoiding shared state Using synchronization in the signal handler Using synchronization in the regular code Disabling or masking other signals, which provides atomicity (which effectively ensures exclusivity)",,medium,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205 -The Art of Software Security Assessment: Chapter 13, ""Race Conditions"", Page 759." +The Art of Software Security Assessment: Chapter 13, ""Race Conditions"", Page 759." CWE-369,EN-Divide By Zero (Type: Base),"The product divides a value by zero. This weakness typically occurs when an unexpected value is provided to the product, or if an error occurs that is not properly detected. It frequently occurs in calculations involving physical dimensions such as size, length, width, and height.",,medium,"No description: http://www.cprogramming.com/tutorial/exceptions.html -No description: http://msdn.microsoft.com/en-us/library/ms173160(VS.80).aspx" +No description: http://msdn.microsoft.com/en-us/library/ms173160(VS.80).aspx" CWE-370,EN-Missing Check for Certificate Revocation after Initial Check (Type: Base),"The software does not check the revocation status of a certificate after its initial revocation check, which can cause the software to perform privileged actions even after the certificate is revoked at a later time. -If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,medium,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205" +If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,medium,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205" CWE-374,EN-Passing Mutable Objects to an Untrusted Method (Type: Base),"Sending non-cloned mutable data as an argument may result in that data being altered or deleted by the called function, thereby putting the calling function into an undefined state. If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,medium,"Does Java pass by reference or pass by value?: http://www.javaworld.com/javaworld/javaqa/2000-05/03-qa-0526-pass.html -Java: The Complete Reference, J2SE 5th Edition" +Java: The Complete Reference, J2SE 5th Edition" CWE-375,EN-Returning a Mutable Object to an Untrusted Caller (Type: Base),"Sending non-cloned mutable data as a return value may result in that data being altered or deleted by the calling function, thereby putting the class in an undefined state. -If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,medium, +If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,medium, CWE-385,EN-Covert Timing Channel (Type: Base),"Covert timing channels convey information by modulating some aspect of system behavior over time, so that the program receiving the information can observe system behavior and infer protected information. In some instances, knowing when data is transmitted between parties can provide a malicious user with privileged information. Also, externally monitoring the timing of operations can potentially reveal sensitive data. For example, a cryptographic operation can expose its internal state if the time it takes to perform the operation varies, based on the state. -Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,medium, +Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,medium, CWE-390,EN-Detection of Error Condition Without Action (Type: Class),"The software detects a specific error, but takes no actions to handle the error. In some instances, knowing when data is transmitted between parties can provide a malicious user with privileged information. Also, externally monitoring the timing of operations can potentially reveal sensitive data. For example, a cryptographic operation can expose its internal state if the time it takes to perform the operation varies, based on the state. -Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,medium,"24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183" +Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,medium,"24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183" CWE-391,EN-Unchecked Error Condition (Type: Base),"Ignoring exceptions and other error conditions may allow an attacker to induce unexpected behavior unnoticed. In some instances, knowing when data is transmitted between parties can provide a malicious user with privileged information. Also, externally monitoring the timing of operations can potentially reveal sensitive data. For example, a cryptographic operation can expose its internal state if the time it takes to perform the operation varies, based on the state. -Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,medium, +Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,medium, CWE-401,EN-Improper Release of Memory Before Removing Last Reference (Memory Leak) (Type: Base),"The software does not sufficiently track and release allocated memory after it has been used, which slowly consumes remaining memory. -This is often triggered by improper handling of malformed data or unexpectedly interrupted sessions.",,medium,How to Break Software Security +This is often triggered by improper handling of malformed data or unexpectedly interrupted sessions.",,medium,How to Break Software Security CWE-460,EN-Improper Cleanup on Thrown Exception (Type: Variant),"The product does not clean up its state or incorrectly cleans up its state when an exception is thrown, leading to unexpected state or control flow. -In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,medium, +In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,medium, CWE-468,EN-Incorrect Pointer Scaling (Type: Base),"In C and C++, one may often accidentally refer to the wrong memory due to the semantics of when math operations are implicitly scaled. -Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,medium,"The Art of Software Security Assessment: Chapter 6, ""Pointer Arithmetic"", Page 277." +Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,medium,"The Art of Software Security Assessment: Chapter 6, ""Pointer Arithmetic"", Page 277." CWE-469,EN-Use of Pointer Subtraction to Determine Size (Type: Base),"The application subtracts one pointer from another in order to determine size, but this calculation can be incorrect if the pointers do not exist in the same memory chunk. -Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,medium, +Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,medium, CWE-476,EN-NULL Pointer Dereference (Type: Base),"A NULL pointer dereference occurs when the application dereferences a pointer that it expects to be valid, but is NULL, typically causing a crash or exit. -NULL pointer dereference issues can occur through a number of flaws, including race conditions, and simple programming omissions.",,medium, +NULL pointer dereference issues can occur through a number of flaws, including race conditions, and simple programming omissions.",,medium, CWE-484,EN-Omitted Break Statement in Switch (Type: Base),"The program omits a break statement within a switch or similar construct, causing code associated with multiple conditions to execute. This can cause problems when the programmer only intended to execute code associated with one condition. -This can lead to critical code executing in situations where it should not.",,medium,"The Art of Software Security Assessment: Chapter 7, ""Switch Statements"", Page 337." +This can lead to critical code executing in situations where it should not.",,medium,"The Art of Software Security Assessment: Chapter 7, ""Switch Statements"", Page 337." CWE-487,EN-Reliance on Package-level Scope (Type: Variant),"Java packages are not inherently closed; therefore, relying on them for code security is not a good practice. -If the decision to trust the methods and data of an object is based on the name of a class, it is possible for malicious users to send objects of the same name as trusted classes and thereby gain the trust afforded to known classes and types.",,medium, +If the decision to trust the methods and data of an object is based on the name of a class, it is possible for malicious users to send objects of the same name as trusted classes and thereby gain the trust afforded to known classes and types.",,medium, CWE-492,EN-Use of Inner Class Containing Sensitive Data (Type: Variant),"Inner classes are translated into classes that are accessible at package scope and may expose code that the programmer intended to keep private to attackers. Data can ""bleed"" from one session to another through member variables of singleton objects, such as Servlets, and objects from a shared pool. -In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,medium, +In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,medium, CWE-494,EN-Download of Code Without Integrity Check (Type: Base),"The product downloads source code or an executable from a remote location and executes the code without sufficiently verifying the origin and integrity of the code. An attacker can execute malicious code by compromising the host server, performing DNS spoofing, or modifying the code in transit.",,medium,"Introduction to Code Signing: http://msdn.microsoft.com/en-us/library/ms537361(VS.85).aspx Authenticode: http://msdn.microsoft.com/en-us/library/ms537359(v=VS.85).aspx @@ -572,61 +572,61 @@ Code Signing Guide: http://developer.apple.com/documentation/Security/Conceptual Secure Software Updates: Disappointments and New Challenges: http://prisms.cs.umass.edu/~kevinfu/papers/secureupdates-hotsec06.pdf 24 Deadly Sins of Software Security: ""Sin 18: The Sins of Mobile Code."" Page 267 Top 25 Series - Rank 20 - Download of Code Without Integrity Check: http://blogs.sans.org/appsecstreetfighter/2010/04/05/top-25-series-rank-20-download-code-integrity-check/ -Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html" +Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html" CWE-498,EN-Cloneable Class Containing Sensitive Information (Type: Variant),"The code contains a class with sensitive data, but the class is cloneable. The data can then be accessed by cloning the class. -Cloneable classes are effectively open classes, since data cannot be hidden in them. Classes that do not explicitly deny cloning can be cloned by any other class without running the constructor.",,medium, +Cloneable classes are effectively open classes, since data cannot be hidden in them. Classes that do not explicitly deny cloning can be cloned by any other class without running the constructor.",,medium, CWE-502,EN-Deserialization of Untrusted Data (Type: Variant),"The application deserializes untrusted data without sufficiently verifying that the resulting data will be valid. It is often convenient to serialize objects for communication or to save them for later use. However, deserialized data or code can often be modified without using the provided accessor functions if it does not use cryptography to protect itself. Furthermore, any cryptography would still be client-side security -- which is a dangerous security assumption. Data that is untrusted can not be trusted to be well-formed.",,medium,"Unserializing user-supplied data, a bad idea: http://heine.familiedeelstra.com/security/unserialize -Why Python Pickle is Insecure: http://nadiana.com/python-pickle-insecure" +Why Python Pickle is Insecure: http://nadiana.com/python-pickle-insecure" CWE-532,EN-Information Exposure Through Log Files (Type: Variant),"Information written to log files can be of a sensitive nature and give valuable guidance to an attacker or expose sensitive user information. -While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,medium, +While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,medium, CWE-602,EN-Client-Side Enforcement of Server-Side Security (Type: Base),"The software is composed of a server that relies on the client to implement a mechanism that is intended to protect the server. -When the server relies on protection mechanisms placed on the client side, an attacker can modify the client-side behavior to bypass the protection mechanisms resulting in potentially unexpected interactions between the client and server. The consequences will vary, depending on what the mechanisms are trying to protect.",,medium,"Writing Secure Code: Chapter 23, ""Client-Side Security Is an Oxymoron"" Page 687" +When the server relies on protection mechanisms placed on the client side, an attacker can modify the client-side behavior to bypass the protection mechanisms resulting in potentially unexpected interactions between the client and server. The consequences will vary, depending on what the mechanisms are trying to protect.",,medium,"Writing Secure Code: Chapter 23, ""Client-Side Security Is an Oxymoron"" Page 687" CWE-665,EN-Improper Initialization (Type: Base),"The software does not initialize or incorrectly initializes a resource, which might leave the resource in an unexpected state when it is accessed or used. This can have security implications when the associated resource is expected to have certain properties or values, such as a variable that determines whether a user has been authenticated or not.",,medium,"Exploiting Uninitialized Data: http://www.felinemenace.org/~mercy/papers/UBehavior/UBehavior.zip MS08-014 : The Case of the Uninitialized Stack Variable Vulnerability: http://blogs.technet.com/swi/archive/2008/03/11/the-case-of-the-uninitialized-stack-variable-vulnerability.aspx -The Art of Software Security Assessment: Chapter 7, ""Variable Initialization"", Page 312." +The Art of Software Security Assessment: Chapter 7, ""Variable Initialization"", Page 312." CWE-754,EN-Improper Check for Unusual or Exceptional Conditions (Type: Class),"The software does not check or improperly checks for unusual or exceptional conditions that are not expected to occur frequently during day to day operation of the software. The programmer may assume that certain events or conditions will never occur or do not need to be worried about, such as low memory conditions, lack of access to resources due to restrictive permissions, or misbehaving clients or components. However, attackers may intentionally trigger these unusual conditions, thus violating the programmer's assumptions, possibly introducing instability, incorrect behavior, or a vulnerability. Note that this entry is not exclusively about the use of exceptions and exception handling, which are mechanisms for both checking and handling unusual or unexpected conditions.",,medium,"The Art of Software Security Assessment: Chapter 7, ""Program Building Blocks"" Page 341 The Art of Software Security Assessment: Chapter 1, ""Exceptional Conditions,"" Page 22 24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183 -Top 25 Series - Rank 15 - Improper Check for Unusual or Exceptional Conditions: http://blogs.sans.org/appsecstreetfighter/2010/03/15/top-25-series-rank-15-improper-check-for-unusual-or-exceptional-conditions/" +Top 25 Series - Rank 15 - Improper Check for Unusual or Exceptional Conditions: http://blogs.sans.org/appsecstreetfighter/2010/03/15/top-25-series-rank-15-improper-check-for-unusual-or-exceptional-conditions/" CWE-778,EN-Insufficient Logging (Type: Base),"When a security-critical event occurs, the software either does not record the event or omits important details about the event when logging it. -When security-critical events are not logged properly, such as a failed login attempt, this can make malicious behavior more difficult to detect and may hinder forensic analysis after an attack succeeds.",,medium,"The Art of Software Security Assessment: Chapter 2, ""Accountability"", Page 40." +When security-critical events are not logged properly, such as a failed login attempt, this can make malicious behavior more difficult to detect and may hinder forensic analysis after an attack succeeds.",,medium,"The Art of Software Security Assessment: Chapter 2, ""Accountability"", Page 40." CWE-780,EN-Use of RSA Algorithm without OAEP (Type: Variant),"The software uses the RSA algorithm but does not incorporate Optimal Asymmetric Encryption Padding (OAEP), which might weaken the encryption. Padding schemes are often used with cryptographic algorithms to make the plaintext less predictable and complicate attack efforts. The OAEP scheme is often used with RSA to nullify the impact of predictable common text.",,medium,"RSA Problem: http://people.csail.mit.edu/rivest/RivestKaliski-RSAProblem.pdf -Optimal Asymmetric Encryption Padding: http://en.wikipedia.org/wiki/Optimal_Asymmetric_Encryption_Padding" +Optimal Asymmetric Encryption Padding: http://en.wikipedia.org/wiki/Optimal_Asymmetric_Encryption_Padding" CWE-908,EN-Use of Uninitialized Resource (Type: Base),"The software uses a resource that has not been properly initialized. -This can have security implications when the associated resource is expected to have certain properties or values.",,medium,Exploiting Uninitialized Data: http://www.felinemenace.org/~mercy/papers/UBehavior/UBehavior.zip +This can have security implications when the associated resource is expected to have certain properties or values.",,medium,Exploiting Uninitialized Data: http://www.felinemenace.org/~mercy/papers/UBehavior/UBehavior.zip CWE-909,EN-Missing Initialization of Resource (Type: Base),"The software does not initialize a critical resource. -Many resources require initialization before they can be properly used. If a resource is not initialized, it could contain unpredictable or expired data, or it could be initialized to defaults that are invalid. This can have security implications when the resource is expected to have certain properties or values.",,medium, +Many resources require initialization before they can be properly used. If a resource is not initialized, it could contain unpredictable or expired data, or it could be initialized to defaults that are invalid. This can have security implications when the resource is expected to have certain properties or values.",,medium, CWE-910,EN-Use of Expired File Descriptor (Type: Base),"The software uses or accesses a file descriptor after it has been closed. -After a file descriptor for a particular file or device has been released, it can be reused. The code might not write to the original file, since the reused file descriptor might reference a different file or device.",,medium, +After a file descriptor for a particular file or device has been released, it can be reused. The code might not write to the original file, since the reused file descriptor might reference a different file or device.",,medium, CWE-911,EN-Improper Update of Reference Count (Type: Base),"The software uses a reference count to manage a resource, but it does not update or incorrectly updates the reference count. -Reference counts can be used when tracking how many objects contain a reference to a particular resource, such as in memory management or garbage collection. When the reference count reaches zero, the resource can be de-allocated or reused because there are no more objects that use it. If the reference count accidentally reaches zero, then the resource might be released too soon, even though it is still in use. If all objects no longer use the resource, but the reference count is not zero, then the resource might not ever be released.",,medium,Windows Kernel Reference Count Vulnerabilities - Case Study: http://j00ru.vexillium.org/dump/zn_slides.pdf +Reference counts can be used when tracking how many objects contain a reference to a particular resource, such as in memory management or garbage collection. When the reference count reaches zero, the resource can be de-allocated or reused because there are no more objects that use it. If the reference count accidentally reaches zero, then the resource might be released too soon, even though it is still in use. If all objects no longer use the resource, but the reference count is not zero, then the resource might not ever be released.",,medium,Windows Kernel Reference Count Vulnerabilities - Case Study: http://j00ru.vexillium.org/dump/zn_slides.pdf CWE-94,EN-Improper Control of Generation of Code (Code Injection) (Type: Class),"The software constructs all or part of a code segment using externally-influenced input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could modify the syntax or behavior of the intended code segment. When software allows a user's input to contain code syntax, it might be possible for an attacker to craft the code in such a way that it will alter the intended control flow of the software. Such an alteration could lead to arbitrary code execution. -Injection problems encompass a wide variety of issues -- all mitigated in very different ways. For this reason, the most effective way to discuss these weaknesses is to note the distinct features which classify them as injection weaknesses. The most important issue to note is that all injection problems share one thing in common -- i.e., they allow for the injection of control plane data into the user-controlled data plane. This means that the execution of the process may be altered by sending code in through legitimate data channels, using no other mechanism. While buffer overflows, and many other flaws, involve the use of some further issue to gain execution, injection problems need only for the data to be parsed. The most classic instantiations of this category of weakness are SQL injection and format string vulnerabilities.",,medium,"24 Deadly Sins of Software Security: ""Sin 3: Web-Client Related Vulnerabilities (XSS)."" Page 63" +Injection problems encompass a wide variety of issues -- all mitigated in very different ways. For this reason, the most effective way to discuss these weaknesses is to note the distinct features which classify them as injection weaknesses. The most important issue to note is that all injection problems share one thing in common -- i.e., they allow for the injection of control plane data into the user-controlled data plane. This means that the execution of the process may be altered by sending code in through legitimate data channels, using no other mechanism. While buffer overflows, and many other flaws, involve the use of some further issue to gain execution, injection problems need only for the data to be parsed. The most classic instantiations of this category of weakness are SQL injection and format string vulnerabilities.",,medium,"24 Deadly Sins of Software Security: ""Sin 3: Web-Client Related Vulnerabilities (XSS)."" Page 63" CWE-95,EN-Improper Neutralization of Directives in Dynamically Evaluated Code (Eval Injection) (Type: Base),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes code syntax before using the input in a dynamic evaluation call (e.g. ""eval""). This may allow an attacker to execute arbitrary code, or at least modify what code can be executed.",,medium,"No description: http://www.rubycentral.com/book/taint.html -The Art of Software Security Assessment: Chapter 18, ""Inline Evaluation"", Page 1095." +The Art of Software Security Assessment: Chapter 18, ""Inline Evaluation"", Page 1095." CWE-287,EN-Improper Authentication (Type: Class),"When an actor claims to have a given identity, the software does not prove or insufficiently proves that the claim is correct. Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,high,"Weak Password Brings 'Happiness' to Twitter Hacker: http://www.wired.com/threatlevel/2009/01/professed-twitt/ Top 10 2007-Broken Authentication and Session Management: http://www.owasp.org/index.php/Top_10_2007-A7 Guide to Authentication: http://www.owasp.org/index.php/Guide_to_Authentication Authentication: http://msdn.microsoft.com/en-us/library/aa374735(VS.85).aspx -Writing Secure Code: Chapter 4, ""Authentication"" Page 109" +Writing Secure Code: Chapter 4, ""Authentication"" Page 109" CWE-306,EN-Missing Authentication for Critical Function (Type: Variant),"The software does not perform any authentication for functionality that requires a provable user identity or consumes a significant amount of resources. Authentication techniques should follow the algorithms that define them exactly, otherwise authentication can be bypassed or more easily subjected to brute force attacks.",,high,"The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Authentication,"" Page 36 Top 25 Series - Rank 19 - Missing Authentication for Critical Function: http://blogs.sans.org/appsecstreetfighter/2010/02/23/top-25-series-rank-19-missing-authentication-for-critical-function/ -OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI" +OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI" CWE-319,EN-Cleartext Transmission of Sensitive Information (Type: Base),"The software transmits sensitive or security-critical data in cleartext in a communication channel that can be sniffed by unauthorized actors. Many communication channels can be ""sniffed"" by attackers during data transmission. For example, network traffic can often be sniffed by any attacker who has access to a network interface. This significantly lowers the difficulty of exploitation by attackers.",,high,"Top 10 2007-Insecure Communications: http://www.owasp.org/index.php/Top_10_2007-A9 Writing Secure Code: Chapter 9, ""Protecting Secret Data"" Page 299 24 Deadly Sins of Software Security: ""Sin 22: Failing to Protect Network Traffic."" Page 337 -Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/" +Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/" CWE-327,EN-Use of a Broken or Risky Cryptographic Algorithm (Type: Base),"The use of a broken or risky cryptographic algorithm is an unnecessary risk that may result in the exposure of sensitive information. The use of a non-standard algorithm is dangerous because a determined attacker may be able to break the algorithm and compromise whatever data has been protected. Well-known techniques may exist to break the algorithm.",,high,"Applied Cryptography: http://www.schneier.com/book-applied.html Handbook of Applied Cryptography: http://www.cacr.math.uwaterloo.ca/hac/ @@ -636,12 +636,12 @@ Microsoft Scraps Old Encryption in New Code: http://www.eweek.com/c/a/Security/M Writing Secure Code: Chapter 8, ""Cryptographic Foibles"" Page 259 24 Deadly Sins of Software Security: ""Sin 21: Using the Wrong Cryptography."" Page 315 Top 25 Series - Rank 24 - Use of a Broken or Risky Cryptographic Algorithm: http://blogs.sans.org/appsecstreetfighter/2010/03/25/top-25-series-rank-24-use-of-a-broken-or-risky-cryptographic-algorithm/ -The Art of Software Security Assessment: Chapter 2, ""Insufficient or Obsolete Encryption"", Page 44." +The Art of Software Security Assessment: Chapter 2, ""Insufficient or Obsolete Encryption"", Page 44." CWE-330,EN-Use of Insufficiently Random Values (Type: Class),"The software may use insufficiently random numbers or values in a security context that depends on unpredictable numbers. When software generates predictable values in a context requiring unpredictability, it may be possible for an attacker to guess the next value that will be generated, and use this guess to impersonate another user or access sensitive information.",,high,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf Building Secure Software: How to Avoid Security Problems the Right Way Writing Secure Code: Chapter 8, ""Using Poor Random Numbers"" Page 259 -24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" +24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-400,EN-Uncontrolled Resource Consumption (Resource Exhaustion) (Type: Base),"The software does not properly restrict the size or amount of resources that are requested or influenced by an actor, which can be used to consume more resources than intended. Limited resources include memory, file system storage, database connection pool entries, or CPU. If an attacker can trigger the allocation of these limited resources, but the number or size of the resources is not controlled, then the attacker could cause a denial of service that consumes all available resources. This would prevent valid users from accessing the software, and it could potentially have an impact on the surrounding environment. For example, a memory exhaustion attack against an application could slow down the application as well as its host operating system. Resource exhaustion problems have at least two common causes: @@ -649,22 +649,22 @@ Error conditions and other exceptional circumstances Confusion over which part of the program is responsible for releasing the resource",,high,"Detection and Prediction of Resource-Exhaustion Vulnerabilities: http://homepages.di.fc.ul.pt/~nuno/PAPERS/ISSRE08.pdf Resource exhaustion: http://cr.yp.to/docs/resources.html Resource exhaustion: http://homes.cerias.purdue.edu/~pmeunier/secprog/sanitized/class1/6.resource%20exhaustion.ppt -Writing Secure Code: Chapter 17, ""Protecting Against Denial of Service Attacks"" Page 517" +Writing Secure Code: Chapter 17, ""Protecting Against Denial of Service Attacks"" Page 517" CWE-434,EN-Unrestricted Upload of File with Dangerous Type (Type: Base),"The software allows the attacker to upload or transfer files of dangerous types that can be automatically processed within the product's environment. If code is stored in a file with an extension such as "".inc"" or "".pl"", and the web server does not have a handler for that extension, then the server will likely send the contents of the file directly to the requester without the pre-processing that was expected. When that file contains sensitive information such as database credentials, this may allow the attacker to compromise the application or associated components.",,high,"Dynamic File Uploads, Security and You: http://shsc.info/FileUploadSecurity 8 Basic Rules to Implement Secure File Uploads: http://blogs.sans.org/appsecstreetfighter/2009/12/28/8-basic-rules-to-implement-secure-file-uploads/ Top 25 Series - Rank 8 - Unrestricted Upload of Dangerous File Type: http://blogs.sans.org/appsecstreetfighter/2010/02/25/top-25-series-rank-8-unrestricted-upload-of-dangerous-file-type/ Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html -The Art of Software Security Assessment: Chapter 17, ""File Uploading"", Page 1068." +The Art of Software Security Assessment: Chapter 17, ""File Uploading"", Page 1068." CWE-64,EN-Windows Shortcut Following (.LNK) (Type: Variant),"The software, when opening a file or directory, does not sufficiently handle when the file is a Windows shortcut (.LNK) whose target is outside of the intended control sphere. This could allow an attacker to cause the software to operate on unauthorized files. -The shortcut (file with the .lnk extension) can permit an attacker to read/write a file that they originally did not have permissions to access.",,high, +The shortcut (file with the .lnk extension) can permit an attacker to read/write a file that they originally did not have permissions to access.",,high, CWE-681,EN-Incorrect Conversion between Numeric Types (Type: Base),"When converting from one data type to another, such as long to integer, data can be omitted or translated in a way that produces unexpected values. If the resulting values are used in a sensitive context, then dangerous behaviors may occur. -Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,high,"The Art of Software Security Assessment: Chapter 6, ""Type Conversions"", Page 223." +Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,high,"The Art of Software Security Assessment: Chapter 6, ""Type Conversions"", Page 223." CWE-732,EN-Incorrect Permission Assignment for Critical Resource (Type: Class),"The software specifies permissions for a security-critical resource in a way that allows that resource to be read or modified by unintended actors. When a resource is given a permissions setting that provides access to a wider range of actors than required, it could lead to the exposure of sensitive information, or the modification of that resource by unintended parties. This is especially dangerous when the resource is related to program configuration, execution or sensitive user data.",,high,"The Art of Software Security Assessment: Chapter 9, ""File Permissions."" Page 495. Building Secure Software: How to Avoid Security Problems the Right Way: Chapter 8, ""Access Control."" Page 194. Top 25 Series - Rank 21 - Incorrect Permission Assignment for Critical Response: http://software-security.sans.org/blog/2010/03/24/top-25-series-rank-21-incorrect-permission-assignment-for-critical-response -Federal Desktop Core Configuration: http://nvd.nist.gov/fdcc/index.cfm" +Federal Desktop Core Configuration: http://nvd.nist.gov/fdcc/index.cfm" CWE-770,EN-Allocation of Resources Without Limits or Throttling (Type: Base),"The software allocates a reusable resource or group of resources on behalf of an actor without imposing any restrictions on how many resources can be allocated, in violation of the intended security policy for that actor. Command injection vulnerabilities typically occur when: 1. Data enters the application from an untrusted source. @@ -675,17 +675,17 @@ Resource exhaustion: http://cr.yp.to/docs/resources.html Resource exhaustion: http://homes.cerias.purdue.edu/~pmeunier/secprog/sanitized/class1/6.resource%20exhaustion.ppt Writing Secure Code: Chapter 17, ""Protecting Against Denial of Service Attacks"" Page 517 Top 25 Series - Rank 22 - Allocation of Resources Without Limits or Throttling: http://blogs.sans.org/appsecstreetfighter/2010/03/23/top-25-series-rank-22-allocation-of-resources-without-limits-or-throttling/ -The Art of Software Security Assessment: Chapter 10, ""Resource Limits"", Page 574." +The Art of Software Security Assessment: Chapter 10, ""Resource Limits"", Page 574." CWE-771,EN-Missing Reference to Active Allocated Resource (Type: Base),"The software does not properly maintain a reference to a resource that has been allocated, which prevents the resource from being reclaimed. -This does not necessarily apply in languages or frameworks that automatically perform garbage collection, since the removal of all references may act as a signal that the resource is ready to be reclaimed.",,high, +This does not necessarily apply in languages or frameworks that automatically perform garbage collection, since the removal of all references may act as a signal that the resource is ready to be reclaimed.",,high, CWE-772,EN-Missing Release of Resource after Effective Lifetime (Type: Base),"The software does not release a resource after its effective lifetime has ended, i.e., after the resource is no longer needed. -When a resource is not released after use, it can allow attackers to cause a denial of service.",,high, +When a resource is not released after use, it can allow attackers to cause a denial of service.",,high, CWE-773,EN-Missing Reference to Active File Descriptor or Handle (Type: Variant),"The software does not properly maintain references to a file descriptor or handle, which prevents that file descriptor/handle from being reclaimed. -This can cause the software to consume all available file descriptors or handles, which can prevent other processes from performing critical file processing operations.",,high, +This can cause the software to consume all available file descriptors or handles, which can prevent other processes from performing critical file processing operations.",,high, CWE-774,EN-Allocation of File Descriptors or Handles Without Limits or Throttling (Type: Variant),"The software allocates file descriptors or handles on behalf of an actor without imposing any restrictions on how many descriptors can be allocated, in violation of the intended security policy for that actor. -This can cause the software to consume all available file descriptors or handles, which can prevent other processes from performing critical file processing operations.",,high,"The Art of Software Security Assessment: Chapter 10, ""Resource Limits"", Page 574." +This can cause the software to consume all available file descriptors or handles, which can prevent other processes from performing critical file processing operations.",,high,"The Art of Software Security Assessment: Chapter 10, ""Resource Limits"", Page 574." CWE-775,EN-Missing Release of File Descriptor or Handle after Effective Lifetime (Type: Variant),"The software does not release a file descriptor or handle after its effective lifetime has ended, i.e., after the file descriptor/handle is no longer needed. -When a file descriptor or handle is not released after use (typically by explicitly closing it), attackers can cause a denial of service by consuming all available file descriptors/handles, or otherwise preventing other system processes from obtaining their own file descriptors/handles.",,high,"The Art of Software Security Assessment: Chapter 10, ""File Descriptor Leaks"", Page 582." +When a file descriptor or handle is not released after use (typically by explicitly closing it), attackers can cause a denial of service by consuming all available file descriptors/handles, or otherwise preventing other system processes from obtaining their own file descriptors/handles.",,high,"The Art of Software Security Assessment: Chapter 10, ""File Descriptor Leaks"", Page 582." CWE-804,EN-Guessable CAPTCHA (Type: Base),"The software uses a CAPTCHA challenge, but the challenge can be guessed or automatically recognized by a non-human actor. An automated attacker could bypass the intended protection of the CAPTCHA challenge and perform actions at a higher frequency than humanly possible, such as launching spam attacks. There can be several different causes of a guessable CAPTCHA: @@ -693,7 +693,7 @@ An audio or visual image that does not have sufficient distortion from the unobf A question is generated that with a format that can be automatically recognized, such as a math question. A question for which the number of possible answers is limited, such as birth years or favorite sports teams. A general-knowledge or trivia question for which the answer can be accessed using a data base, such as country capitals or popular actors. -Other data associated with the CAPTCHA may provide hints about its contents, such as an image whose filename contains the word that is used in the CAPTCHA.",,high,Insufficient Anti-automation: http://projects.webappsec.org/Insufficient+Anti-automation +Other data associated with the CAPTCHA may provide hints about its contents, such as an image whose filename contains the word that is used in the CAPTCHA.",,high,Insufficient Anti-automation: http://projects.webappsec.org/Insufficient+Anti-automation CWE-805,EN-Buffer Access with Incorrect Length Value (Type: Base),"The software uses a sequential operation to read or write a buffer, but it uses an incorrect length value that causes it to access memory that is outside of the bounds of the buffer. When the length value exceeds the size of the destination, a buffer overflow could occur.",,high,"Writing Secure Code: Chapter 6, ""Why ACLs Are Important"" Page 171 Address Space Layout Randomization in Windows Vista: http://blogs.msdn.com/michael_howard/archive/2006/05/26/address-space-layout-randomization-in-windows-vista.aspx @@ -703,84 +703,84 @@ Top 25 Series - Rank 12 - Buffer Access with Incorrect Length Value: http://blog Safe C String Library v1.0.3: http://www.zork.org/safestr/ Using the Strsafe.h Functions: http://msdn.microsoft.com/en-us/library/ms647466.aspx Understanding DEP as a mitigation technology part 1: http://blogs.technet.com/b/srd/archive/2009/06/12/understanding-dep-as-a-mitigation-technology-part-1.aspx -Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html" +Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html" CWE-806,EN-Buffer Access Using Size of Source Buffer (Type: Variant),"The software uses the size of a source buffer when reading from or writing to a destination buffer, which may cause it to access memory that is outside of the bounds of the buffer. When the size of the destination is smaller than the size of the source, a buffer overflow could occur.",,high,"Using the Strsafe.h Functions: http://msdn.microsoft.com/en-us/library/ms647466.aspx Safe C String Library v1.0.3: http://www.zork.org/safestr/ Address Space Layout Randomization in Windows Vista: http://blogs.msdn.com/michael_howard/archive/2006/05/26/address-space-layout-randomization-in-windows-vista.aspx Limiting buffer overflows with ExecShield: http://www.redhat.com/magazine/009jul05/features/execshield/ PaX: http://en.wikipedia.org/wiki/PaX -Understanding DEP as a mitigation technology part 1: http://blogs.technet.com/b/srd/archive/2009/06/12/understanding-dep-as-a-mitigation-technology-part-1.aspx" +Understanding DEP as a mitigation technology part 1: http://blogs.technet.com/b/srd/archive/2009/06/12/understanding-dep-as-a-mitigation-technology-part-1.aspx" CWE-807,EN-Reliance on Untrusted Inputs in a Security Decision (Type: Base),"The application uses a protection mechanism that relies on the existence or values of an input, but the input can be modified by an untrusted actor in a way that bypasses the protection mechanism. Developers may assume that inputs such as cookies, environment variables, and hidden form fields cannot be modified. However, an attacker could change these inputs using customized clients or other attacks. This change might not be detected. When security decisions such as authentication and authorization are made based on the values of these inputs, attackers can bypass the security of the software. Without sufficient encryption, integrity checking, or other mechanism, any input that originates from an outsider cannot be trusted.",,high,"Top 25 Series - Rank 6 - Reliance on Untrusted Inputs in a Security Decision: http://blogs.sans.org/appsecstreetfighter/2010/03/05/top-25-series-rank-6-reliance-on-untrusted-inputs-in-a-security-decision/ HMAC: http://en.wikipedia.org/wiki/Hmac Understanding ASP.NET View State: http://msdn.microsoft.com/en-us/library/ms972976.aspx -OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI" +OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI" CWE-93,EN-Improper Neutralization of CRLF Sequences (CRLF Injection) (Type: Base),"The software uses CRLF (carriage return line feeds) as a special element, e.g. to separate lines or records, but it does not neutralize or incorrectly neutralizes CRLF sequences from inputs. -Since an implicit intent does not specify a particular application to receive the data, any application can process the intent by using an Intent Filter for that intent. This can allow untrusted applications to obtain sensitive data.",,high,CRLF Injection: http://marc.info/?l=bugtraq&m=102088154213630&w=2 +Since an implicit intent does not specify a particular application to receive the data, any application can process the intent by using an Intent Filter for that intent. This can allow untrusted applications to obtain sensitive data.",,high,CRLF Injection: http://marc.info/?l=bugtraq&m=102088154213630&w=2 CWE-102,EN-Struts: Duplicate Validation Forms (Type: Variant),"The application uses multiple validation forms with the same name, which might cause the Struts Validator to validate a form that the programmer does not expect. -If two validation forms have the same name, the Struts Validator arbitrarily chooses one of the forms to use for input validation and discards the other. This decision might not correspond to the programmer's expectations, possibly leading to resultant weaknesses. Moreover, it indicates that the validation logic is not up-to-date, and can indicate that other, more subtle validation errors are present.",,unclassified, +If two validation forms have the same name, the Struts Validator arbitrarily chooses one of the forms to use for input validation and discards the other. This decision might not correspond to the programmer's expectations, possibly leading to resultant weaknesses. Moreover, it indicates that the validation logic is not up-to-date, and can indicate that other, more subtle validation errors are present.",,unclassified, CWE-103,EN-Struts: Incomplete validate() Method Definition (Type: Variant),"The application has a validator form that either does not define a validate() method, or defines a validate() method but does not call super.validate(). -If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,unclassified, +If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,unclassified, CWE-104,EN-Struts: Form Bean Does Not Extend Validation Class (Type: Variant),"If a form bean does not extend an ActionForm subclass of the Validator framework, it can expose the application to other weaknesses related to insufficient input validation. -If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,unclassified, +If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,unclassified, CWE-105,EN-Struts: Form Field Without Validator (Type: Variant),"The application has a form field that is not validated by a corresponding validation form, which can introduce other weaknesses related to insufficient input validation. -If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,unclassified, +If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,unclassified, CWE-106,EN-Struts: Plug-in Framework not in Use (Type: Variant),"When an application does not use an input validation framework such as the Struts Validator, there is a greater risk of introducing weaknesses related to insufficient input validation. -If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,unclassified, +If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,unclassified, CWE-107,EN-Struts: Unused Validation Form (Type: Variant),"An unused validation form indicates that validation logic is not up-to-date. -It is easy for developers to forget to update validation logic when they remove or rename action form mappings. One indication that validation logic is not being properly maintained is the presence of an unused validation form.",,unclassified, +It is easy for developers to forget to update validation logic when they remove or rename action form mappings. One indication that validation logic is not being properly maintained is the presence of an unused validation form.",,unclassified, CWE-108,EN-Struts: Unvalidated Action Form (Type: Variant),"Every Action Form must have a corresponding validation form. -If a Struts Action Form Mapping specifies a form, it must have a validation form defined under the Struts Validator.",,unclassified, +If a Struts Action Form Mapping specifies a form, it must have a validation form defined under the Struts Validator.",,unclassified, CWE-109,EN-Struts: Validator Turned Off (Type: Variant),"Automatic filtering via a Struts bean has been turned off, which disables the Struts Validator and custom validation logic. This exposes the application to other weaknesses related to insufficient input validation. -If a Struts Action Form Mapping specifies a form, it must have a validation form defined under the Struts Validator.",,unclassified, +If a Struts Action Form Mapping specifies a form, it must have a validation form defined under the Struts Validator.",,unclassified, CWE-11,EN-ASP.NET Misconfiguration: Creating Debug Binary (Type: Variant),"Debugging messages help attackers learn about the system and plan a form of attack. -ASP .NET applications can be configured to produce debug binaries. These binaries give detailed debugging messages and should not be used in production environments. Debug binaries are meant to be used in a development or testing environment and can pose a security risk if they are deployed to production.",,unclassified, +ASP .NET applications can be configured to produce debug binaries. These binaries give detailed debugging messages and should not be used in production environments. Debug binaries are meant to be used in a development or testing environment and can pose a security risk if they are deployed to production.",,unclassified, CWE-110,EN-Struts: Validator Without Form Field (Type: Variant),"Validation fields that do not appear in forms they are associated with indicate that the validation logic is out of date. -It is easy for developers to forget to update validation logic when they make changes to an ActionForm class. One indication that validation logic is not being properly maintained is inconsistencies between the action form and the validation form.",,unclassified, +It is easy for developers to forget to update validation logic when they make changes to an ActionForm class. One indication that validation logic is not being properly maintained is inconsistencies between the action form and the validation form.",,unclassified, CWE-111,EN-Direct Use of Unsafe JNI (Type: Base),"When a Java application uses the Java Native Interface (JNI) to call code written in another programming language, it can expose the application to weaknesses in that code, even if those weaknesses cannot occur in Java. Many safety features that programmers may take for granted simply do not apply for native code, so you must carefully review all such code for potential problems. The languages used to implement native code may be more susceptible to buffer overflows and other attacks. Native code is unprotected by the security features enforced by the runtime environment, such as strong typing and array bounds checking.",,unclassified,"Fortify Descriptions: http://vulncat.fortifysoftware.com -The Java(TM) Tutorial: The Java Native Interface: http://java.sun.com/docs/books/tutorial/native1.1/" +The Java(TM) Tutorial: The Java Native Interface: http://java.sun.com/docs/books/tutorial/native1.1/" CWE-112,EN-Missing XML Validation (Type: Base),"The software accepts XML from an untrusted source but does not validate the XML against the proper schema. -Most successful attacks begin with a violation of the programmer's assumptions. By accepting an XML document without validating it against a DTD or XML schema, the programmer leaves a door open for attackers to provide unexpected, unreasonable, or malicious input.",,unclassified, +Most successful attacks begin with a violation of the programmer's assumptions. By accepting an XML document without validating it against a DTD or XML schema, the programmer leaves a door open for attackers to provide unexpected, unreasonable, or malicious input.",,unclassified, CWE-113,EN-Improper Neutralization of CRLF Sequences in HTTP Headers (HTTP Response Splitting) (Type: Base),"The software receives data from an upstream component, but does not neutralize or incorrectly neutralizes CR and LF characters before the data is included in outgoing HTTP headers. Including unvalidated data in an HTTP header allows an attacker to specify the entirety of the HTTP response rendered by the browser. When an HTTP request contains unexpected CR (carriage return, also given by %0d or \r) and LF (line feed, also given by %0a or \n) characters the server may respond with an output stream that is interpreted as two different HTTP responses (instead of one). An attacker can control the second response and mount attacks such as cross-site scripting and cache poisoning attacks. HTTP response splitting weaknesses may be present when: Data enters a web application through an untrusted source, most frequently an HTTP request. The data is included in an HTTP response header sent to a web user without being validated for malicious characters.",,unclassified,"OWASP TOP 10: http://www.owasp.org/index.php/Top_10_2007 -24 Deadly Sins of Software Security: ""Sin 2: Web-Server Related Vulnerabilities (XSS, XSRF, and Response Splitting)."" Page 31" +24 Deadly Sins of Software Security: ""Sin 2: Web-Server Related Vulnerabilities (XSS, XSRF, and Response Splitting)."" Page 31" CWE-114,EN-Process Control (Type: Base),"Executing commands or loading libraries from an untrusted source or in an untrusted environment can cause an application to execute malicious commands (and payloads) on behalf of an attacker. -Process control vulnerabilities take two forms: 1. An attacker can change the command that the program executes: the attacker explicitly controls what the command is. 2. An attacker can change the environment in which the command executes: the attacker implicitly controls what the command means. Process control vulnerabilities of the first type occur when either data enters the application from an untrusted source and the data is used as part of a string representing a command that is executed by the application. By executing the command, the application gives an attacker a privilege or capability that the attacker would not otherwise have.",,unclassified, +Process control vulnerabilities take two forms: 1. An attacker can change the command that the program executes: the attacker explicitly controls what the command is. 2. An attacker can change the environment in which the command executes: the attacker implicitly controls what the command means. Process control vulnerabilities of the first type occur when either data enters the application from an untrusted source and the data is used as part of a string representing a command that is executed by the application. By executing the command, the application gives an attacker a privilege or capability that the attacker would not otherwise have.",,unclassified, CWE-115,EN-Misinterpretation of Input (Type: Base),"The software misinterprets an input, whether from an attacker or another product, in a security-relevant fashion. -Process control vulnerabilities take two forms: 1. An attacker can change the command that the program executes: the attacker explicitly controls what the command is. 2. An attacker can change the environment in which the command executes: the attacker implicitly controls what the command means. Process control vulnerabilities of the first type occur when either data enters the application from an untrusted source and the data is used as part of a string representing a command that is executed by the application. By executing the command, the application gives an attacker a privilege or capability that the attacker would not otherwise have.",,unclassified, +Process control vulnerabilities take two forms: 1. An attacker can change the command that the program executes: the attacker explicitly controls what the command is. 2. An attacker can change the environment in which the command executes: the attacker implicitly controls what the command means. Process control vulnerabilities of the first type occur when either data enters the application from an untrusted source and the data is used as part of a string representing a command that is executed by the application. By executing the command, the application gives an attacker a privilege or capability that the attacker would not otherwise have.",,unclassified, CWE-118,EN-Improper Access of Indexable Resource (Range Error) (Type: Class),"The software does not restrict or incorrectly restricts operations within the boundaries of a resource that is accessed using an index or pointer, such as memory or files. This can allow an attacker to forge log entries or inject malicious content into logs. Log forging vulnerabilities occur when: Data enters an application from an untrusted source. -The data is written to an application or system log file.",,unclassified, +The data is written to an application or system log file.",,unclassified, CWE-12,EN-ASP.NET Misconfiguration: Missing Custom Error Page (Type: Variant),"An ASP .NET application must enable custom error pages in order to prevent attackers from mining information from the framework's built-in responses. Certain languages allow direct addressing of memory locations and do not automatically ensure that these locations are valid for the memory buffer that is being referenced. This can cause read or write operations to be performed on memory locations that may be associated with other variables, data structures, or internal program data. As a result, an attacker may be able to execute arbitrary code, alter the intended control flow, read sensitive information, or cause the system to crash.",,unclassified,"19 Deadly Sins of Software Security -ASP.NET Misconfiguration: Missing Custom Error Handling: http://www.owasp.org/index.php/ASP.NET_Misconfiguration:_Missing_Custom_Error_Handling" +ASP.NET Misconfiguration: Missing Custom Error Handling: http://www.owasp.org/index.php/ASP.NET_Misconfiguration:_Missing_Custom_Error_Handling" CWE-125,EN-Out-of-bounds Read (Type: Base),"The software reads data past the end, or before the beginning, of the intended buffer. -This typically occurs when the pointer or its index is incremented or decremented to a position beyond the bounds of the buffer or when pointer arithmetic results in a position outside of the valid memory location to name a few. This may result in corruption of sensitive information, a crash, or code execution among other things.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89" +This typically occurs when the pointer or its index is incremented or decremented to a position beyond the bounds of the buffer or when pointer arithmetic results in a position outside of the valid memory location to name a few. This may result in corruption of sensitive information, a crash, or code execution among other things.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89" CWE-126,EN-Buffer Over-read (Type: Variant),"The software reads from a buffer using buffer access mechanisms such as indexes or pointers that reference memory locations after the targeted buffer. -This typically occurs when the pointer or its index is incremented to a position beyond the bounds of the buffer or when pointer arithmetic results in a position outside of the valid memory location to name a few. This may result in exposure of sensitive information or possibly a crash.",,unclassified, +This typically occurs when the pointer or its index is incremented to a position beyond the bounds of the buffer or when pointer arithmetic results in a position outside of the valid memory location to name a few. This may result in exposure of sensitive information or possibly a crash.",,unclassified, CWE-127,EN-Buffer Under-read (Type: Variant),"The software reads from a buffer using buffer access mechanisms such as indexes or pointers that reference memory locations prior to the targeted buffer. -This typically occurs when the pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. This may result in exposure of sensitive information or possibly a crash.",,unclassified, +This typically occurs when the pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. This may result in exposure of sensitive information or possibly a crash.",,unclassified, CWE-13,EN-ASP.NET Misconfiguration: Password in Configuration File (Type: Variant),"Storing a plaintext password in a configuration file allows anyone who can read the file access to the password-protected resource making them an easy target for attackers. This typically occurs when the pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. This may result in exposure of sensitive information or possibly a crash.",,unclassified,"How To: Encrypt Configuration Sections in ASP.NET 2.0 Using DPAPI: http://msdn.microsoft.com/en-us/library/ms998280.aspx How To: Encrypt Configuration Sections in ASP.NET 2.0 Using RSA: http://msdn.microsoft.com/en-us/library/ms998283.aspx -.NET Framework Developer's Guide - Securing Connection Strings: http://msdn.microsoft.com/en-us/library/89211k9b(VS.80).aspx" +.NET Framework Developer's Guide - Securing Connection Strings: http://msdn.microsoft.com/en-us/library/89211k9b(VS.80).aspx" CWE-130,EN-Improper Handling of Length Parameter Inconsistency (Type: Variant),"The software parses a formatted message or structure, but it does not handle or incorrectly handles a length field that is inconsistent with the actual length of the associated data. -If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,unclassified, +If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,unclassified, CWE-132,EN-DEPRECATED (Duplicate): Miscalculated Null Termination (Type: Base),"This entry has been deprecated because it was a duplicate of CWE-170. All content has been transferred to CWE-170. -If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,unclassified, +If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,unclassified, CWE-135,EN-Incorrect Calculation of Multi-Byte String Length (Type: Base),"The software does not correctly calculate the length of strings that can contain wide or multi-byte characters. -If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,unclassified,"Writing Secure Code: Chapter 5, ""Unicode and ANSI Buffer Size Mismatches"" Page 153" +If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,unclassified,"Writing Secure Code: Chapter 5, ""Unicode and ANSI Buffer Size Mismatches"" Page 153" CWE-138,EN-Improper Neutralization of Special Elements (Type: Class),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as control elements or syntactic markers when they are sent to a downstream component. -Most languages and protocols have their own special elements such as characters and reserved words. These special elements can carry control implications. If software does not prevent external control or influence over the inclusion of such special elements, the control flow of the program may be altered from what was intended. For example, both Unix and Windows interpret the symbol < (""less than"") as meaning ""read input from a file"".",,unclassified, +Most languages and protocols have their own special elements such as characters and reserved words. These special elements can carry control implications. If software does not prevent external control or influence over the inclusion of such special elements, the control flow of the program may be altered from what was intended. For example, both Unix and Windows interpret the symbol < (""less than"") as meaning ""read input from a file"".",,unclassified, CWE-14,EN-Compiler Removal of Code to Clear Buffers (Type: Base),"Sensitive memory is cleared according to the source code, but compiler optimizations leave the memory untouched when it is not read from again, aka ""dead store removal."" This compiler optimization error occurs when: 1. Secret data are stored in memory. @@ -788,337 +788,337 @@ This compiler optimization error occurs when: 3. The source code is compiled using an optimizing compiler, which identifies and removes the function that overwrites the contents as a dead store because the memory is not used subsequently.",,unclassified,"Writing Secure Code: Chapter 9, ""A Compiler Optimization Caveat"" Page 322 When scrubbing secrets in memory doesn't work: http://cert.uni-stuttgart.de/archive/bugtraq/2002/11/msg00046.html Some Bad News and Some Good News: http://msdn.microsoft.com/library/default.asp?url=/library/en-us/dncode/html/secure10102002.asp -GNU GCC: Optimizer Removes Code Necessary for Security: http://www.derkeiler.com/Mailing-Lists/securityfocus/bugtraq/2002-11/0257.html" +GNU GCC: Optimizer Removes Code Necessary for Security: http://www.derkeiler.com/Mailing-Lists/securityfocus/bugtraq/2002-11/0257.html" CWE-140,EN-Improper Neutralization of Delimiters (Type: Base),"The software does not neutralize or incorrectly neutralizes delimiters. This compiler optimization error occurs when: 1. Secret data are stored in memory. 2. The secret data are scrubbed from memory by overwriting its contents. -3. The source code is compiled using an optimizing compiler, which identifies and removes the function that overwrites the contents as a dead store because the memory is not used subsequently.",,unclassified, +3. The source code is compiled using an optimizing compiler, which identifies and removes the function that overwrites the contents as a dead store because the memory is not used subsequently.",,unclassified, CWE-141,EN-Improper Neutralization of Parameter/Argument Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as parameter or argument delimiters when they are sent to a downstream component. As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408. -The Art of Software Security Assessment: Chapter 10, ""IFS"", Page 604." +The Art of Software Security Assessment: Chapter 10, ""IFS"", Page 604." CWE-142,EN-Improper Neutralization of Value Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as value delimiters when they are sent to a downstream component. -As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." +As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." CWE-143,EN-Improper Neutralization of Record Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as record delimiters when they are sent to a downstream component. -As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." +As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." CWE-144,EN-Improper Neutralization of Line Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as line delimiters when they are sent to a downstream component. -As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." +As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." CWE-145,EN-Improper Neutralization of Section Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as section delimiters when they are sent to a downstream component. As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions. -One example of a section delimiter is the boundary string in a multipart MIME message. In many cases, doubled line delimiters can serve as a section delimiter.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." +One example of a section delimiter is the boundary string in a multipart MIME message. In many cases, doubled line delimiters can serve as a section delimiter.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." CWE-146,EN-Improper Neutralization of Expression/Command Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as expression or command delimiters when they are sent to a downstream component. -As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." +As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408." CWE-147,EN-Improper Neutralization of Input Terminators (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as input terminators when they are sent to a downstream component. -For example, a ""."" in SMTP signifies the end of mail message data, whereas a null character can be used for the end of a string.",,unclassified, +For example, a ""."" in SMTP signifies the end of mail message data, whereas a null character can be used for the end of a string.",,unclassified, CWE-148,EN-Improper Neutralization of Input Leaders (Type: Variant),"The application does not properly handle when a leading character or sequence (""leader"") is missing or malformed, or if multiple leaders are used when only one should be allowed. -For example, a ""."" in SMTP signifies the end of mail message data, whereas a null character can be used for the end of a string.",,unclassified, +For example, a ""."" in SMTP signifies the end of mail message data, whereas a null character can be used for the end of a string.",,unclassified, CWE-149,EN-Improper Neutralization of Quoting Syntax (Type: Variant),"Quotes injected into an application can be used to compromise a system. As data are parsed, an injected/absent/duplicate/malformed use of quotes may cause the process to take unexpected actions. -For example, a ""."" in SMTP signifies the end of mail message data, whereas a null character can be used for the end of a string.",,unclassified, +For example, a ""."" in SMTP signifies the end of mail message data, whereas a null character can be used for the end of a string.",,unclassified, CWE-15,EN-External Control of System or Configuration Setting (Type: Base),"One or more system settings or configuration elements can be externally controlled by a user. -Allowing external control of system settings can disrupt service or cause an application to behave in unexpected, and potentially malicious ways.",,unclassified, +Allowing external control of system settings can disrupt service or cause an application to behave in unexpected, and potentially malicious ways.",,unclassified, CWE-150,"EN-Improper Neutralization of Escape, Meta, or Control Sequences (Type: Variant)","The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as escape, meta, or control character sequences when they are sent to a downstream component. -As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified, +As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified, CWE-151,EN-Improper Neutralization of Comment Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as comment delimiters when they are sent to a downstream component. -As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified, +As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified, CWE-152,EN-Improper Neutralization of Macro Symbols (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as macro symbols when they are sent to a downstream component. -As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified, +As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified, CWE-153,EN-Improper Neutralization of Substitution Characters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as substitution characters when they are sent to a downstream component. -As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified, +As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified, CWE-154,EN-Improper Neutralization of Variable Name Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as variable name delimiters when they are sent to a downstream component. -As data is parsed, an injected delimiter may cause the process to take unexpected actions that result in an attack. Example: ""$"" for an environment variable.",,unclassified, +As data is parsed, an injected delimiter may cause the process to take unexpected actions that result in an attack. Example: ""$"" for an environment variable.",,unclassified, CWE-155,EN-Improper Neutralization of Wildcards or Matching Symbols (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as wildcards or matching symbols when they are sent to a downstream component. -As data is parsed, an injected element may cause the process to take unexpected actions.",,unclassified, +As data is parsed, an injected element may cause the process to take unexpected actions.",,unclassified, CWE-156,EN-Improper Neutralization of Whitespace (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as whitespace when they are sent to a downstream component. -This can include space, tab, etc.",,unclassified, +This can include space, tab, etc.",,unclassified, CWE-157,EN-Failure to Sanitize Paired Delimiters (Type: Variant),"The software does not properly handle the characters that are used to mark the beginning and ending of a group of entities, such as parentheses, brackets, and braces. -This can include space, tab, etc.",,unclassified, +This can include space, tab, etc.",,unclassified, CWE-158,EN-Improper Neutralization of Null Byte or NUL Character (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes NUL characters or null bytes when they are sent to a downstream component. -As data is parsed, an injected NUL character or null byte may cause the software to believe the input is terminated earlier than it actually is, or otherwise cause the input to be misinterpreted. This could then be used to inject potentially dangerous input that occurs after the null byte or otherwise bypass validation routines and other protection mechanisms.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""NUL Character Injection"", Page 411." +As data is parsed, an injected NUL character or null byte may cause the software to believe the input is terminated earlier than it actually is, or otherwise cause the input to be misinterpreted. This could then be used to inject potentially dangerous input that occurs after the null byte or otherwise bypass validation routines and other protection mechanisms.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""NUL Character Injection"", Page 411." CWE-159,EN-Failure to Sanitize Special Element (Type: Class),"Weaknesses in this attack-focused category do not properly filter and interpret special elements in user-controlled input which could cause adverse effect on the software behavior and integrity. -As data is parsed, an injected NUL character or null byte may cause the software to believe the input is terminated earlier than it actually is, or otherwise cause the input to be misinterpreted. This could then be used to inject potentially dangerous input that occurs after the null byte or otherwise bypass validation routines and other protection mechanisms.",,unclassified, +As data is parsed, an injected NUL character or null byte may cause the software to believe the input is terminated earlier than it actually is, or otherwise cause the input to be misinterpreted. This could then be used to inject potentially dangerous input that occurs after the null byte or otherwise bypass validation routines and other protection mechanisms.",,unclassified, CWE-160,EN-Improper Neutralization of Leading Special Elements (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes leading special elements that could be interpreted in unexpected ways when they are sent to a downstream component. -As data is parsed, improperly handled leading special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, +As data is parsed, improperly handled leading special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, CWE-161,EN-Improper Neutralization of Multiple Leading Special Elements (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes multiple leading special elements that could be interpreted in unexpected ways when they are sent to a downstream component. -As data is parsed, improperly handled multiple leading special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, +As data is parsed, improperly handled multiple leading special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, CWE-162,EN-Improper Neutralization of Trailing Special Elements (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes trailing special elements that could be interpreted in unexpected ways when they are sent to a downstream component. -As data is parsed, improperly handled trailing special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, +As data is parsed, improperly handled trailing special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, CWE-163,EN-Improper Neutralization of Multiple Trailing Special Elements (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes multiple trailing special elements that could be interpreted in unexpected ways when they are sent to a downstream component. -As data is parsed, improperly handled multiple trailing special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, +As data is parsed, improperly handled multiple trailing special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, CWE-164,EN-Improper Neutralization of Internal Special Elements (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes internal special elements that could be interpreted in unexpected ways when they are sent to a downstream component. -As data is parsed, improperly handled internal special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, +As data is parsed, improperly handled internal special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, CWE-165,EN-Improper Neutralization of Multiple Internal Special Elements (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes multiple internal special elements that could be interpreted in unexpected ways when they are sent to a downstream component. -As data is parsed, improperly handled multiple internal special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, +As data is parsed, improperly handled multiple internal special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, CWE-166,EN-Improper Handling of Missing Special Element (Type: Base),"The software receives input from an upstream component, but it does not handle or incorrectly handles when an expected special element is missing. -As data is parsed, improperly handled multiple internal special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, +As data is parsed, improperly handled multiple internal special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, CWE-167,EN-Improper Handling of Additional Special Element (Type: Base),"The software receives input from an upstream component, but it does not handle or incorrectly handles when an additional unexpected special element is missing. -As data is parsed, improperly handled multiple internal special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, +As data is parsed, improperly handled multiple internal special elements may cause the process to take unexpected actions that result in an attack.",,unclassified, CWE-168,EN-Improper Handling of Inconsistent Special Elements (Type: Base),"The software does not handle when an inconsistency exists between two or more special characters or reserved words. -An example of this problem would be if paired characters appear in the wrong order, or if the special characters are not properly nested.",,unclassified, +An example of this problem would be if paired characters appear in the wrong order, or if the special characters are not properly nested.",,unclassified, CWE-172,EN-Encoding Error (Type: Class),"The software does not properly encode or decode the data, resulting in unexpected values. -Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified, +Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified, CWE-173,EN-Improper Handling of Alternate Encoding (Type: Variant),"The software does not properly handle when an input uses an alternate encoding that is valid for the control sphere to which the input is being sent. -Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified, +Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified, CWE-174,EN-Double Decoding of the Same Data (Type: Variant),"The software decodes the same input twice, which can limit the effectiveness of any protection mechanism that occurs in between the decoding operations. -Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified, +Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified, CWE-175,EN-Improper Handling of Mixed Encoding (Type: Variant),"The software does not properly handle when the same input uses several different (mixed) encodings. -Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified, +Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified, CWE-176,EN-Improper Handling of Unicode Encoding (Type: Variant),"The software does not properly handle when an input contains Unicode encoding. -Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Character Sets and Unicode"", Page 446." +Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Character Sets and Unicode"", Page 446." CWE-177,EN-Improper Handling of URL Encoding (Hex Encoding) (Type: Variant),"The software does not properly handle when all or part of an input has been URL encoded. -Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified, +Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified, CWE-178,EN-Improper Handling of Case Sensitivity (Type: Base),"The software does not properly account for differences in case sensitivity when accessing or determining the properties of a resource, leading to inconsistent results. Improperly handled case sensitive data can lead to several possible consequences, including: case-insensitive passwords reducing the size of the key space, making brute force attacks easier bypassing filters or access controls using alternate names -multiple interpretation errors using alternate names.",,unclassified, +multiple interpretation errors using alternate names.",,unclassified, CWE-179,EN-Incorrect Behavior Order: Early Validation (Type: Base),"The software validates input before applying protection mechanisms that modify the input, which could allow an attacker to bypass the validation via dangerous inputs that only arise after the modification. -Software needs to validate data at the proper time, after data has been canonicalized and cleansed. Early validation is susceptible to various manipulations that result in dangerous inputs that are produced by canonicalization and cleansing.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Escaping Metacharacters"", Page 439." +Software needs to validate data at the proper time, after data has been canonicalized and cleansed. Early validation is susceptible to various manipulations that result in dangerous inputs that are produced by canonicalization and cleansing.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Escaping Metacharacters"", Page 439." CWE-180,EN-Incorrect Behavior Order: Validate Before Canonicalize (Type: Base),"The software validates input before it is canonicalized, which prevents the software from detecting data that becomes invalid after the canonicalization step. -This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,unclassified, +This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,unclassified, CWE-181,EN-Incorrect Behavior Order: Validate Before Filter (Type: Base),"The software validates data before it has been filtered, which prevents the software from detecting data that becomes invalid after the filtering step. -This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,unclassified, +This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,unclassified, CWE-182,EN-Collapse of Data into Unsafe Value (Type: Base),"The software filters data in a way that causes it to be reduced or ""collapsed"" into an unsafe value that violates an expected security property. -This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Character Stripping Vulnerabilities"", Page 437." +This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Character Stripping Vulnerabilities"", Page 437." CWE-183,EN-Permissive Whitelist (Type: Base),"An application uses a ""whitelist"" of acceptable values, but the whitelist includes at least one unsafe value, leading to resultant weaknesses. -This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Eliminating Metacharacters"", Page 435." +This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Eliminating Metacharacters"", Page 435." CWE-184,EN-Incomplete Blacklist (Type: Base),"An application uses a ""blacklist"" of prohibited values, but the blacklist is incomplete. If an incomplete blacklist is used as a security mechanism, then the software may allow unintended values to pass into the application logic.",,unclassified,"Exploiting Software: How to Break Code Blacklist defenses as a breeding ground for vulnerability variants: http://seclists.org/fulldisclosure/2006/Feb/0040.html -The Art of Software Security Assessment: Chapter 8, ""Eliminating Metacharacters"", Page 435." +The Art of Software Security Assessment: Chapter 8, ""Eliminating Metacharacters"", Page 435." CWE-185,EN-Incorrect Regular Expression (Type: Class),"The software specifies a regular expression in a way that causes data to be improperly matched or compared. -When the regular expression is used in protection mechanisms such as filtering or validation, this may allow an attacker to bypass the intended restrictions on the incoming data.",,unclassified,"Writing Secure Code: Chapter 10, ""Using Regular Expressions for Checking Input"" Page 350" +When the regular expression is used in protection mechanisms such as filtering or validation, this may allow an attacker to bypass the intended restrictions on the incoming data.",,unclassified,"Writing Secure Code: Chapter 10, ""Using Regular Expressions for Checking Input"" Page 350" CWE-186,EN-Overly Restrictive Regular Expression (Type: Base),"A regular expression is overly restrictive, which prevents dangerous values from being detected. -When the regular expression is used in protection mechanisms such as filtering or validation, this may allow an attacker to bypass the intended restrictions on the incoming data.",,unclassified, +When the regular expression is used in protection mechanisms such as filtering or validation, this may allow an attacker to bypass the intended restrictions on the incoming data.",,unclassified, CWE-187,EN-Partial Comparison (Type: Base),"The software performs a comparison that only examines a portion of a factor before determining whether there is a match, such as a substring, leading to resultant weaknesses. -For example, an attacker might succeed in authentication by providing a small password that matches the associated portion of the larger, correct password.",,unclassified, +For example, an attacker might succeed in authentication by providing a small password that matches the associated portion of the larger, correct password.",,unclassified, CWE-191,EN-Integer Underflow (Wrap or Wraparound) (Type: Base),"The product subtracts one value from another, such that the result is less than the minimum allowable integer value, which produces a value that is not equal to the correct result. -This can happen in signed and unsigned cases.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 7: Integer Overflows."" Page 119" +This can happen in signed and unsigned cases.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 7: Integer Overflows."" Page 119" CWE-193,EN-Off-by-one Error (Type: Base),"A product calculates or uses an incorrect maximum or minimum value that is 1 more, or 1 less, than the correct value. This can happen in signed and unsigned cases.",,unclassified,"Third Generation Exploits: http://www.blackhat.com/presentations/bh-europe-01/halvar-flake/bh-europe-01-halvarflake.ppt Off-by-one errors: a brief explanation: http://marc.theaimsgroup.com/?l=secprog&m=108379742110553&w=2 The Frame Pointer Overwrite: http://kaizo.org/mirrors/phrack/phrack55/P55-08 Exploiting Software: How to Break Code (The buffer overflow chapter) 24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89 -The Art of Software Security Assessment: Chapter 5, ""Off-by-One Errors"", Page 180." +The Art of Software Security Assessment: Chapter 5, ""Off-by-One Errors"", Page 180." CWE-195,EN-Signed to Unsigned Conversion Error (Type: Variant),"A signed-to-unsigned conversion error takes place when a signed primitive is used as an unsigned value, usually as a size variable. -It is dangerous to rely on implicit casts between signed and unsigned numbers because the result can take on an unexpected value and violate assumptions made by the program.",,unclassified,"The Art of Software Security Assessment: Chapter 6, ""Type Conversions"", Page 223." +It is dangerous to rely on implicit casts between signed and unsigned numbers because the result can take on an unexpected value and violate assumptions made by the program.",,unclassified,"The Art of Software Security Assessment: Chapter 6, ""Type Conversions"", Page 223." CWE-198,EN-Use of Incorrect Byte Ordering (Type: Base),"The software receives input from an upstream component, but it does not account for byte ordering (e.g. big-endian and little-endian) when processing the input, causing an incorrect number or value to be used. -When a primitive is cast to a smaller primitive, the high order bits of the large value are lost in the conversion, potentially resulting in an unexpected value that is not equal to the original value. This value may be required as an index into a buffer, a loop iterator, or simply necessary state data. In any case, the value cannot be trusted and the system will be in an undefined state. While this method may be employed viably to isolate the low bits of a value, this usage is rare, and truncation usually implies that an implementation error has occurred.",,unclassified, +When a primitive is cast to a smaller primitive, the high order bits of the large value are lost in the conversion, potentially resulting in an unexpected value that is not equal to the original value. This value may be required as an index into a buffer, a loop iterator, or simply necessary state data. In any case, the value cannot be trusted and the system will be in an undefined state. While this method may be employed viably to isolate the low bits of a value, this usage is rare, and truncation usually implies that an implementation error has occurred.",,unclassified, CWE-201,EN-Information Exposure Through Sent Data (Type: Variant),"The accidental exposure of sensitive information through sent data refers to the transmission of data which are either sensitive in and of itself or useful in the further exploitation of the system through standard data channels. The information either is regarded as sensitive within the product's own functionality, such as a private message; or provides information about the product or its environment that could be useful in an attack but is normally not available to the attacker, such as the installation path of a product that is remotely accessible. -Many information exposures are resultant (e.g. PHP script error revealing the full path of the program), but they can also be primary (e.g. timing discrepancies in cryptography). There are many different types of problems that involve information exposures. Their severity can range widely depending on the type of information that is revealed.",,unclassified, +Many information exposures are resultant (e.g. PHP script error revealing the full path of the program), but they can also be primary (e.g. timing discrepancies in cryptography). There are many different types of problems that involve information exposures. Their severity can range widely depending on the type of information that is revealed.",,unclassified, CWE-203,EN-Information Exposure Through Discrepancy (Type: Class),"The product behaves differently or sends different responses in a way that exposes security-relevant information about the state of the product, such as whether a particular operation was successful or not. -In situations where data should not be tied to individual users, but a large number of users should be able to make queries that ""scrub"" the identity of users, it may be possible to get information about a user -- e.g., by specifying search terms that are known to be unique to that user.",,unclassified, +In situations where data should not be tied to individual users, but a large number of users should be able to make queries that ""scrub"" the identity of users, it may be possible to get information about a user -- e.g., by specifying search terms that are known to be unique to that user.",,unclassified, CWE-204,EN-Response Discrepancy Information Exposure (Type: Base),"The software provides different responses to incoming requests in a way that allows an actor to determine system state information that is outside of that actor's control sphere. -This issue frequently occurs during authentication, where a difference in failed-login messages could allow an attacker to determine if the username is valid or not. These exposures can be inadvertent (bug) or intentional (design).",,unclassified,"24 Deadly Sins of Software Security: ""Sin 12: Information Leakage."" Page 191" +This issue frequently occurs during authentication, where a difference in failed-login messages could allow an attacker to determine if the username is valid or not. These exposures can be inadvertent (bug) or intentional (design).",,unclassified,"24 Deadly Sins of Software Security: ""Sin 12: Information Leakage."" Page 191" CWE-205,EN-Information Exposure Through Behavioral Discrepancy (Type: Base),"The product's actions indicate important differences based on (1) the internal state of the product or (2) differences from other products in the same class. -For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,unclassified, +For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,unclassified, CWE-206,EN-Information Exposure of Internal State Through Behavioral Inconsistency (Type: Variant),"Two separate operations in a product cause the product to behave differently in a way that is observable to an attacker and reveals security-relevant information about the internal state of the product, such as whether a particular operation was successful or not. -For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,unclassified, +For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,unclassified, CWE-207,EN-Information Exposure Through an External Behavioral Inconsistency (Type: Variant),"The product behaves differently than other products like it, in a way that is observable to an attacker and exposes security-relevant information about which product is being used. -For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,unclassified, +For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,unclassified, CWE-208,EN-Information Exposure Through Timing Discrepancy (Type: Base),"Two separate operations in a product require different amounts of time to complete, in a way that is observable to an actor and reveals security-relevant information about the state of the product, such as whether a particular operation was successful or not. -For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,unclassified, +For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,unclassified, CWE-210,EN-Information Exposure Through Self-generated Error Message (Type: Base),"The software identifies an error condition and creates its own diagnostic or error messages that contain sensitive information. The sensitive information may be valuable information on its own (such as a password), or it may be useful for launching other, more deadly attacks. If an attack fails, an attacker may use error information provided by the server to launch another more focused attack. For example, an attempt to exploit a path traversal weakness (CWE-22) might yield the full pathname of the installed application. In turn, this could be used to select the proper number of "".."" sequences to navigate to the targeted file. An attack using SQL injection (CWE-89) might not initially succeed, but an error message could reveal the malformed query, which would expose query logic and possibly even passwords or other sensitive information used within the query.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 12: Information Leakage."" Page 191 -The Art of Software Security Assessment: Chapter 3, ""Overly Verbose Error Messages"", Page 75." +The Art of Software Security Assessment: Chapter 3, ""Overly Verbose Error Messages"", Page 75." CWE-211,EN-Information Exposure Through Externally-generated Error Message (Type: Base),"The software performs an operation that triggers an external diagnostic or error message that is not directly generated by the software, such as an error generated by the programming language interpreter that the software uses. The error can contain sensitive system information. -The sensitive information may be valuable information on its own (such as a password), or it may be useful for launching other, more deadly attacks. If an attack fails, an attacker may use error information provided by the server to launch another more focused attack. For example, an attempt to exploit a path traversal weakness (CWE-22) might yield the full pathname of the installed application. In turn, this could be used to select the proper number of "".."" sequences to navigate to the targeted file. An attack using SQL injection (CWE-89) might not initially succeed, but an error message could reveal the malformed query, which would expose query logic and possibly even passwords or other sensitive information used within the query.",,unclassified, +The sensitive information may be valuable information on its own (such as a password), or it may be useful for launching other, more deadly attacks. If an attack fails, an attacker may use error information provided by the server to launch another more focused attack. For example, an attempt to exploit a path traversal weakness (CWE-22) might yield the full pathname of the installed application. In turn, this could be used to select the proper number of "".."" sequences to navigate to the targeted file. An attack using SQL injection (CWE-89) might not initially succeed, but an error message could reveal the malformed query, which would expose query logic and possibly even passwords or other sensitive information used within the query.",,unclassified, CWE-212,EN-Improper Cross-boundary Removal of Sensitive Data (Type: Base),"The software uses a resource that contains sensitive data, but it does not properly remove that data before it stores, transfers, or shares the resource with actors in another control sphere. Resources that may contain sensitive data include documents, packets, messages, databases, etc. While this data may be useful to an individual user or small set of users who share the resource, it may need to be removed before the resource can be shared outside of the trusted group. The process of removal is sometimes called cleansing or scrubbing. -For example, software that is used for editing documents might not remove sensitive data such as reviewer comments or the local pathname where the document is stored. Or, a proxy might not remove an internal IP address from headers before making an outgoing request to an Internet site.",,unclassified, +For example, software that is used for editing documents might not remove sensitive data such as reviewer comments or the local pathname where the document is stored. Or, a proxy might not remove an internal IP address from headers before making an outgoing request to an Internet site.",,unclassified, CWE-213,EN-Intentional Information Exposure (Type: Base),"A product's design or configuration explicitly requires the publication of information that could be regarded as sensitive by an administrator. Resources that may contain sensitive data include documents, packets, messages, databases, etc. While this data may be useful to an individual user or small set of users who share the resource, it may need to be removed before the resource can be shared outside of the trusted group. The process of removal is sometimes called cleansing or scrubbing. -For example, software that is used for editing documents might not remove sensitive data such as reviewer comments or the local pathname where the document is stored. Or, a proxy might not remove an internal IP address from headers before making an outgoing request to an Internet site.",,unclassified, +For example, software that is used for editing documents might not remove sensitive data such as reviewer comments or the local pathname where the document is stored. Or, a proxy might not remove an internal IP address from headers before making an outgoing request to an Internet site.",,unclassified, CWE-214,EN-Information Exposure Through Process Environment (Type: Variant),"A process is invoked with sensitive arguments, environment variables, or other elements that can be seen by other processes on the operating system. -Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified, +Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified, CWE-215,EN-Information Exposure Through Debug Information (Type: Variant),"The application contains debugging code that can expose sensitive information to untrusted parties. -Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified, +Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified, CWE-216,EN-Containment Errors (Container Errors) (Type: Class),"This tries to cover various problems in which improper data are included within a ""container."" -Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified, +Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified, CWE-217,EN-DEPRECATED: Failure to Protect Stored Data from Modification (Type: Base),"This weakness has been deprecated because it incorporated and confused multiple weaknesses. The issues formerly covered in this weakness can be found at CWE-766 and CWE-767. -Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified, +Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified, CWE-218,EN-DEPRECATED (Duplicate): Failure to provide confidentiality for stored data (Type: Base),"This weakness has been deprecated because it was a duplicate of CWE-493. All content has been transferred to CWE-493. -Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified, +Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified, CWE-219,EN-Sensitive Data Under Web Root (Type: Variant),"The application stores sensitive data under the web document root with insufficient access control, which might make it accessible to untrusted parties. -Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified, +Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified, CWE-220,EN-Sensitive Data Under FTP Root (Type: Variant),"The application stores sensitive data under the FTP document root with insufficient access control, which might make it accessible to untrusted parties. Many file operations are intended to take place within a restricted directory. By using special elements such as "".."" and ""/"" separators, attackers can escape outside of the restricted location to access files or directories that are elsewhere on the system. One of the most common special elements is the ""../"" sequence, which in most modern operating systems is interpreted as the parent directory of the current location. This is referred to as relative path traversal. Path traversal also covers the use of absolute pathnames such as ""/usr/local/bin"", which may also be useful in accessing unexpected files. This is referred to as absolute path traversal. -In many programming languages, the injection of a null byte (the 0 or NUL) may allow an attacker to truncate a generated filename to widen the scope of attack. For example, the software may add "".txt"" to any pathname, thus limiting the attacker to text files, but a null injection may effectively remove this restriction.",,unclassified, +In many programming languages, the injection of a null byte (the 0 or NUL) may allow an attacker to truncate a generated filename to widen the scope of attack. For example, the software may add "".txt"" to any pathname, thus limiting the attacker to text files, but a null injection may effectively remove this restriction.",,unclassified, CWE-221,EN-Information Loss or Omission (Type: Class),"The software does not record, or improperly records, security-relevant information that leads to an incorrect decision or hampers later analysis. -This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified, +This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified, CWE-222,EN-Truncation of Security-relevant Information (Type: Base),"The application truncates the display, recording, or processing of security-relevant information in a way that can obscure the source or nature of an attack. -This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified, +This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified, CWE-223,EN-Omission of Security-relevant Information (Type: Base),"The application does not record or display information that would be important for identifying the source or nature of an attack, or determining if an action is safe. -This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified,"The Art of Software Security Assessment: Chapter 2, ""Accountability"", Page 40." +This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified,"The Art of Software Security Assessment: Chapter 2, ""Accountability"", Page 40." CWE-224,EN-Obscured Security-relevant Information by Alternate Name (Type: Base),"The software records security-relevant information according to an alternate name of the affected entity, instead of the canonical name. -This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified,Writing Secure Code +This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified,Writing Secure Code CWE-225,EN-DEPRECATED (Duplicate): General Information Management Problems (Type: Base),"This weakness can be found at CWE-199. -This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified, +This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified, CWE-226,EN-Sensitive Information Uncleared Before Release (Type: Base),"The software does not fully clear previously used information in a data structure, file, or other resource, before making that resource available to a party in another control sphere. -This typically results from new data that is not as long as the old data, which leaves portions of the old data still available. Equivalent errors can occur in other situations where the length of data is variable but the associated data structure is not. If memory is not cleared after use, it may allow unintended actors to read the data when the memory is reallocated.",,unclassified, +This typically results from new data that is not as long as the old data, which leaves portions of the old data still available. Equivalent errors can occur in other situations where the length of data is variable but the associated data structure is not. If memory is not cleared after use, it may allow unintended actors to read the data when the memory is reallocated.",,unclassified, CWE-227,EN-Improper Fulfillment of API Contract (API Abuse) (Type: Class),"The software uses an API in a manner contrary to its intended use. -An API is a contract between a caller and a callee. The most common forms of API misuse occurs when the caller does not honor its end of this contract. For example, if a program does not call chdir() after calling chroot(), it violates the contract that specifies how to change the active root directory in a secure fashion. Another good example of library abuse is expecting the callee to return trustworthy DNS information to the caller. In this case, the caller misuses the callee API by making certain assumptions about its behavior (that the return value can be used for authentication purposes). One can also violate the caller-callee contract from the other side. For example, if a coder subclasses SecureRandom and returns a non-random value, the contract is violated.",,unclassified, +An API is a contract between a caller and a callee. The most common forms of API misuse occurs when the caller does not honor its end of this contract. For example, if a program does not call chdir() after calling chroot(), it violates the contract that specifies how to change the active root directory in a secure fashion. Another good example of library abuse is expecting the callee to return trustworthy DNS information to the caller. In this case, the caller misuses the callee API by making certain assumptions about its behavior (that the return value can be used for authentication purposes). One can also violate the caller-callee contract from the other side. For example, if a coder subclasses SecureRandom and returns a non-random value, the contract is violated.",,unclassified, CWE-228,EN-Improper Handling of Syntactically Invalid Structure (Type: Class),"The product does not handle or incorrectly handles input that is not syntactically well-formed with respect to the associated specification. -An API is a contract between a caller and a callee. The most common forms of API misuse occurs when the caller does not honor its end of this contract. For example, if a program does not call chdir() after calling chroot(), it violates the contract that specifies how to change the active root directory in a secure fashion. Another good example of library abuse is expecting the callee to return trustworthy DNS information to the caller. In this case, the caller misuses the callee API by making certain assumptions about its behavior (that the return value can be used for authentication purposes). One can also violate the caller-callee contract from the other side. For example, if a coder subclasses SecureRandom and returns a non-random value, the contract is violated.",,unclassified, +An API is a contract between a caller and a callee. The most common forms of API misuse occurs when the caller does not honor its end of this contract. For example, if a program does not call chdir() after calling chroot(), it violates the contract that specifies how to change the active root directory in a secure fashion. Another good example of library abuse is expecting the callee to return trustworthy DNS information to the caller. In this case, the caller misuses the callee API by making certain assumptions about its behavior (that the return value can be used for authentication purposes). One can also violate the caller-callee contract from the other side. For example, if a coder subclasses SecureRandom and returns a non-random value, the contract is violated.",,unclassified, CWE-229,EN-Improper Handling of Values (Type: Base),"The software does not properly handle when the expected number of values for parameters, fields, or arguments is not provided in input, or if those values are undefined. -An API is a contract between a caller and a callee. The most common forms of API misuse occurs when the caller does not honor its end of this contract. For example, if a program does not call chdir() after calling chroot(), it violates the contract that specifies how to change the active root directory in a secure fashion. Another good example of library abuse is expecting the callee to return trustworthy DNS information to the caller. In this case, the caller misuses the callee API by making certain assumptions about its behavior (that the return value can be used for authentication purposes). One can also violate the caller-callee contract from the other side. For example, if a coder subclasses SecureRandom and returns a non-random value, the contract is violated.",,unclassified, +An API is a contract between a caller and a callee. The most common forms of API misuse occurs when the caller does not honor its end of this contract. For example, if a program does not call chdir() after calling chroot(), it violates the contract that specifies how to change the active root directory in a secure fashion. Another good example of library abuse is expecting the callee to return trustworthy DNS information to the caller. In this case, the caller misuses the callee API by making certain assumptions about its behavior (that the return value can be used for authentication purposes). One can also violate the caller-callee contract from the other side. For example, if a coder subclasses SecureRandom and returns a non-random value, the contract is violated.",,unclassified, CWE-23,EN-Relative Path Traversal (Type: Base),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize sequences such as "".."" that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified,"OWASP Attack listing: http://www.owasp.org/index.php/Relative_Path_Traversal -The Art of Software Security Assessment: Chapter 9, ""Filenames and Paths"", Page 503." +The Art of Software Security Assessment: Chapter 9, ""Filenames and Paths"", Page 503." CWE-230,EN-Improper Handling of Missing Values (Type: Variant),"The software does not handle or incorrectly handles when a parameter, field, or argument name is specified, but the associated value is missing, i.e. it is empty, blank, or null. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, CWE-231,EN-Improper Handling of Extra Values (Type: Variant),"The software does not handle or incorrectly handles when more values are provided than expected. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, CWE-232,EN-Improper Handling of Undefined Values (Type: Variant),"The software does not handle or incorrectly handles when a value is not defined or supported for the associated parameter, field, or argument name. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, CWE-233,EN-Improper Handling of Parameters (Type: Base),"The software does not properly handle when the expected number of parameters, fields, or arguments is not provided in input, or if those parameters are undefined. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, CWE-235,EN-Improper Handling of Extra Parameters (Type: Variant),"The software does not handle or incorrectly handles when the number of parameters, fields, or arguments with the same name exceeds the expected amount. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, CWE-236,EN-Improper Handling of Undefined Parameters (Type: Variant),"The software does not handle or incorrectly handles when a particular parameter, field, or argument name is not defined or supported by the product. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, CWE-237,EN-Improper Handling of Structural Elements (Type: Base),"The software does not handle or incorrectly handles inputs that are related to complex structures. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, CWE-238,EN-Improper Handling of Incomplete Structural Elements (Type: Variant),"The software does not handle or incorrectly handles when a particular structural element is not completely specified. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, CWE-239,EN-Failure to Handle Incomplete Element (Type: Variant),"The software does not properly handle when a particular element is not completely specified. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified, CWE-24,EN-Path Traversal: ../filedir (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize ""../"" sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The ""../"" manipulation is the canonical manipulation for operating systems that use ""/"" as directory separators, such as UNIX- and Linux-based systems. In some cases, it is useful for bypassing protection schemes in environments for which ""/"" is supported but not the primary separator, such as Windows, which uses ""\"" but can also accept ""/"".",,unclassified, +The ""../"" manipulation is the canonical manipulation for operating systems that use ""/"" as directory separators, such as UNIX- and Linux-based systems. In some cases, it is useful for bypassing protection schemes in environments for which ""/"" is supported but not the primary separator, such as Windows, which uses ""\"" but can also accept ""/"".",,unclassified, CWE-240,EN-Improper Handling of Inconsistent Structural Elements (Type: Variant),"The software does not handle or incorrectly handles when two or more structural elements should be consistent, but are not. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The ""../"" manipulation is the canonical manipulation for operating systems that use ""/"" as directory separators, such as UNIX- and Linux-based systems. In some cases, it is useful for bypassing protection schemes in environments for which ""/"" is supported but not the primary separator, such as Windows, which uses ""\"" but can also accept ""/"".",,unclassified, +The ""../"" manipulation is the canonical manipulation for operating systems that use ""/"" as directory separators, such as UNIX- and Linux-based systems. In some cases, it is useful for bypassing protection schemes in environments for which ""/"" is supported but not the primary separator, such as Windows, which uses ""\"" but can also accept ""/"".",,unclassified, CWE-241,EN-Improper Handling of Unexpected Data Type (Type: Base),"The software does not handle or incorrectly handles when a particular element is not the expected type, e.g. it expects a digit (0-9) but is provided with a letter (A-Z). This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The ""../"" manipulation is the canonical manipulation for operating systems that use ""/"" as directory separators, such as UNIX- and Linux-based systems. In some cases, it is useful for bypassing protection schemes in environments for which ""/"" is supported but not the primary separator, such as Windows, which uses ""\"" but can also accept ""/"".",,unclassified, +The ""../"" manipulation is the canonical manipulation for operating systems that use ""/"" as directory separators, such as UNIX- and Linux-based systems. In some cases, it is useful for bypassing protection schemes in environments for which ""/"" is supported but not the primary separator, such as Windows, which uses ""\"" but can also accept ""/"".",,unclassified, CWE-244,EN-Improper Clearing of Heap Memory Before Release (Heap Inspection) (Type: Variant),"Using realloc() to resize buffers that store sensitive information can leave the sensitive information exposed to attack, because it is not removed from memory. -When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,unclassified, +When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,unclassified, CWE-245,EN-J2EE Bad Practices: Direct Management of Connections (Type: Variant),"The J2EE application directly manages connections, instead of using the container's connection management facilities. -When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,unclassified, +When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,unclassified, CWE-246,EN-J2EE Bad Practices: Direct Use of Sockets (Type: Variant),"The J2EE application directly uses sockets instead of using framework method calls. -When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,unclassified, +When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,unclassified, CWE-247,EN-DEPRECATED (Duplicate): Reliance on DNS Lookups in a Security Decision (Type: Base),"This entry has been deprecated because it was a duplicate of CWE-350. All content has been transferred to CWE-350. -When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,unclassified, +When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,unclassified, CWE-248,EN-Uncaught Exception (Type: Base),"An exception is thrown from a function, but it is not caught. -When an exception is not caught, it may cause the program to crash or expose sensitive information.",,unclassified, +When an exception is not caught, it may cause the program to crash or expose sensitive information.",,unclassified, CWE-249,EN-DEPRECATED: Often Misused: Path Manipulation (Type: Variant),"This entry has been deprecated because of name confusion and an accidental combination of multiple weaknesses. Most of its content has been transferred to CWE-785. -When an exception is not caught, it may cause the program to crash or expose sensitive information.",,unclassified, +When an exception is not caught, it may cause the program to crash or expose sensitive information.",,unclassified, CWE-25,EN-Path Traversal: /../filedir (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize ""/../"" sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -Sometimes a program checks for ""../"" at the beginning of the input, so a ""/../"" can bypass that check.",,unclassified, +Sometimes a program checks for ""../"" at the beginning of the input, so a ""/../"" can bypass that check.",,unclassified, CWE-26,EN-Path Traversal: /dir/../filename (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize ""/dir/../filename"" sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '/dir/../filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only checks for ""../"" at the beginning of the input, so a ""/../"" can bypass that check.",,unclassified, +The '/dir/../filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only checks for ""../"" at the beginning of the input, so a ""/../"" can bypass that check.",,unclassified, CWE-260,EN-Password in Configuration File (Type: Variant),"The software stores a password in a configuration file that might be accessible to actors who do not know the password. -This can result in compromise of the system for which the password is used. An attacker could gain access to this file and learn the stored password or worse yet, change the password to one of their choosing.",,unclassified,Building Secure Software: How to Avoid Security Problems the Right Way +This can result in compromise of the system for which the password is used. An attacker could gain access to this file and learn the stored password or worse yet, change the password to one of their choosing.",,unclassified,Building Secure Software: How to Avoid Security Problems the Right Way CWE-261,EN-Weak Cryptography for Passwords (Type: Variant),"Obscuring a password with a trivial encoding does not protect the password. This can result in compromise of the system for which the password is used. An attacker could gain access to this file and learn the stored password or worse yet, change the password to one of their choosing.",,unclassified,"Building Secure Software: How to Avoid Security Problems the Right Way -24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" +24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" CWE-266,EN-Incorrect Privilege Assignment (Type: Base),"A product incorrectly assigns a privilege to a particular actor, creating an unintended sphere of control for that actor. -Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,unclassified,Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html +Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,unclassified,Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html CWE-267,EN-Privilege Defined With Unsafe Actions (Type: Base),"A particular privilege, role, capability, or right can be used to perform unsafe actions that were not intended, even when it is assigned to the correct entity. -Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,unclassified,Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html +Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,unclassified,Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html CWE-27,EN-Path Traversal: dir/../../filename (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize multiple internal ""../"" sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The 'directory/../../filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only removes one ""../"" sequence, so multiple ""../"" can bypass that check. Alternately, this manipulation could be used to bypass a check for ""../"" at the beginning of the pathname, moving up more than one directory level.",,unclassified, +The 'directory/../../filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only removes one ""../"" sequence, so multiple ""../"" can bypass that check. Alternately, this manipulation could be used to bypass a check for ""../"" at the beginning of the pathname, moving up more than one directory level.",,unclassified, CWE-270,EN-Privilege Context Switching Error (Type: Base),"The software does not properly manage privileges while it is switching between different contexts that have different privileges or spheres of control. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. The 'directory/../../filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only removes one ""../"" sequence, so multiple ""../"" can bypass that check. Alternately, this manipulation could be used to bypass a check for ""../"" at the beginning of the pathname, moving up more than one directory level.",,unclassified,"Writing Secure Code: Chapter 7, ""Running with Least Privilege"" Page 207 -Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html" +Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html" CWE-272,EN-Least Privilege Violation (Type: Base),"The elevated privilege level required to perform operations such as chroot() should be dropped immediately after the operation is performed. -In some contexts, a system executing with elevated permissions will hand off a process/file/etc. to another process or user. If the privileges of an entity are not reduced, then elevated privileges are spread throughout a system and possibly to an attacker.",,unclassified, +In some contexts, a system executing with elevated permissions will hand off a process/file/etc. to another process or user. If the privileges of an entity are not reduced, then elevated privileges are spread throughout a system and possibly to an attacker.",,unclassified, CWE-274,EN-Improper Handling of Insufficient Privileges (Type: Base),"The software does not handle or incorrectly handles when it has insufficient privileges to perform an operation, leading to resultant weaknesses. -If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,unclassified, +If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,unclassified, CWE-277,EN-Insecure Inherited Permissions (Type: Variant),"A product defines a set of insecure permissions that are inherited by objects that are created by the program. -If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,unclassified, +If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,unclassified, CWE-278,EN-Insecure Preserved Inherited Permissions (Type: Variant),"A product inherits a set of insecure permissions for an object, e.g. when copying from an archive file, without user awareness or involvement. -If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,unclassified, +If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,unclassified, CWE-279,EN-Incorrect Execution-Assigned Permissions (Type: Variant),"While it is executing, the software sets the permissions of an object in a way that violates the intended permissions that have been specified by the user. -If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,unclassified, +If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,unclassified, CWE-28,EN-Path Traversal: ..\filedir (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize ""..\"" sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified, +The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified, CWE-280,EN-Improper Handling of Insufficient Permissions or Privileges (Type: Base),"The application does not handle or incorrectly handles when it has insufficient privileges to access resources or functionality as specified by their permissions. This may cause it to follow unexpected code paths that may leave the application in an invalid state. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified, +The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified, CWE-281,EN-Improper Preservation of Permissions (Type: Base),"The software does not preserve permissions or incorrectly preserves permissions when copying, restoring, or sharing objects, which can cause them to have less restrictive permissions than intended. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified, +The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified, CWE-282,EN-Improper Ownership Management (Type: Class),"The software assigns the wrong ownership, or does not properly verify the ownership, of an object or resource. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified, +The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified, CWE-283,EN-Unverified Ownership (Type: Base),"The software does not properly verify that a critical resource is owned by the proper entity. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified, +The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified, CWE-284,EN-Improper Access Control (Type: Class),"The software does not restrict or incorrectly restricts access to a resource from an unauthorized actor. Access control involves the use of several protection mechanisms such as authentication (proving the identity of an actor) authorization (ensuring that a given actor can access a resource), and accountability (tracking of activities that were performed). When any mechanism is not applied or otherwise fails, attackers can compromise the security of the software by gaining privileges, reading sensitive information, executing commands, evading detection, etc. There are two distinct behaviors that can introduce access control weaknesses: Specification: incorrect privileges, permissions, ownership, etc. are explicitly specified for either the user or the resource (for example, setting a password file to be world-writable, or giving administrator capabilities to a guest user). This action could be performed by the program or the administrator. Enforcement: the mechanism contains errors that prevent it from properly enforcing the specified access control requirements (e.g., allowing the user to specify their own privileges, or allowing a syntactically-incorrect ACL to produce insecure settings). This problem occurs within the program itself, in that it does not actually enforce the intended security policy that the administrator specifies.",,unclassified,"Writing Secure Code: Chapter 6, ""Determining Appropriate Access Control"" Page 171 -24 Deadly Sins of Software Security: ""Sin 17: Failure to Protect Stored Data."" Page 253" +24 Deadly Sins of Software Security: ""Sin 17: Failure to Protect Stored Data."" Page 253" CWE-286,EN-Incorrect User Management (Type: Class),"The software does not properly manage a user within its environment. -Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,unclassified, +Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,unclassified, CWE-288,EN-Authentication Bypass Using an Alternate Path or Channel (Type: Base),"A product requires authentication, but the product has an alternate path or channel that does not require authentication. -Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,unclassified, +Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,unclassified, CWE-289,EN-Authentication Bypass by Alternate Name (Type: Variant),"The software performs authentication based on the name of a resource being accessed, or the name of the actor performing the access, but it does not properly check all possible names for that resource or actor. -Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,unclassified, +Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,unclassified, CWE-29,EN-Path Traversal: \..\filename (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '\..\filename' (leading backslash dot dot) sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -This is similar to CWE-25, except using ""\"" instead of ""/"". Sometimes a program checks for ""..\"" at the beginning of the input, so a ""\..\"" can bypass that check. It is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified, +This is similar to CWE-25, except using ""\"" instead of ""/"". Sometimes a program checks for ""..\"" at the beginning of the input, so a ""\..\"" can bypass that check. It is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified, CWE-290,EN-Authentication Bypass by Spoofing (Type: Base),"This attack-focused weakness is caused by improperly implemented authentication schemes that are subject to spoofing attacks. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -This is similar to CWE-25, except using ""\"" instead of ""/"". Sometimes a program checks for ""..\"" at the beginning of the input, so a ""\..\"" can bypass that check. It is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified,"The Art of Software Security Assessment: Chapter 3, ""Spoofing and Identification"", Page 72." +This is similar to CWE-25, except using ""\"" instead of ""/"". Sometimes a program checks for ""..\"" at the beginning of the input, so a ""\..\"" can bypass that check. It is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified,"The Art of Software Security Assessment: Chapter 3, ""Spoofing and Identification"", Page 72." CWE-295,EN-Improper Certificate Validation (Type: Base),"The software does not validate, or incorrectly validates, a certificate. When a certificate is invalid or malicious, it might allow an attacker to spoof a trusted entity by using a man-in-the-middle (MITM) attack. The software might connect to a malicious host while believing it is a trusted host, or the software might be deceived into accepting spoofed data that appears to originate from a trusted host.",,unclassified,"Why Eve and Mallory Love Android: An Analysis of Android SSL (In)Security: http://www2.dcsec.uni-hannover.de/files/android/p50-fahl.pdf -Computer Security: Art and Science" +Computer Security: Art and Science" CWE-30,EN-Path Traversal: \dir\..\filename (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '\dir\..\filename' (leading backslash dot dot) sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -This is similar to CWE-26, except using ""\"" instead of ""/"". The '\dir\..\filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only checks for ""..\"" at the beginning of the input, so a ""\..\"" can bypass that check.",,unclassified, +This is similar to CWE-26, except using ""\"" instead of ""/"". The '\dir\..\filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only checks for ""..\"" at the beginning of the input, so a ""\..\"" can bypass that check.",,unclassified, CWE-300,EN-Channel Accessible by Non-Endpoint (Man-in-the-Middle) (Type: Class),"The product does not adequately verify the identity of actors at both ends of a communication channel, or does not adequately ensure the integrity of the channel, in a way that allows the channel to be accessed or influenced by an actor that is not an endpoint. -In order to establish secure communication between two parties, it is often important to adequately verify the identity of entities at each end of the communication channel. Inadequate or inconsistent verification may result in insufficient or incorrect identification of either communicating entity. This can have negative consequences such as misplaced trust in the entity at the other end of the channel. An attacker can leverage this by interposing between the communicating entities and masquerading as the original entity. In the absence of sufficient verification of identity, such an attacker can eavesdrop and potentially modify the communication between the original entities.",,unclassified,Computer Security: Art and Science +In order to establish secure communication between two parties, it is often important to adequately verify the identity of entities at each end of the communication channel. Inadequate or inconsistent verification may result in insufficient or incorrect identification of either communicating entity. This can have negative consequences such as misplaced trust in the entity at the other end of the channel. An attacker can leverage this by interposing between the communicating entities and masquerading as the original entity. In the absence of sufficient verification of identity, such an attacker can eavesdrop and potentially modify the communication between the original entities.",,unclassified,Computer Security: Art and Science CWE-302,EN-Authentication Bypass by Assumed-Immutable Data (Type: Variant),"The authentication scheme or implementation uses key data elements that are assumed to be immutable, but can be controlled or modified by the attacker. -A mutual authentication protocol requires each party to respond to a random challenge by the other party by encrypting it with a pre-shared key. Often, however, such protocols employ the same pre-shared key for communication with a number of different entities. A malicious user or an attacker can easily compromise this protocol without possessing the correct key by employing a reflection attack on the protocol.",,unclassified, +A mutual authentication protocol requires each party to respond to a random challenge by the other party by encrypting it with a pre-shared key. Often, however, such protocols employ the same pre-shared key for communication with a number of different entities. A malicious user or an attacker can easily compromise this protocol without possessing the correct key by employing a reflection attack on the protocol.",,unclassified, CWE-303,EN-Incorrect Implementation of Authentication Algorithm (Type: Base),"The requirements for the software dictate the use of an established authentication algorithm, but the implementation of the algorithm is incorrect. -This incorrect implementation may allow authentication to be bypassed.",,unclassified, +This incorrect implementation may allow authentication to be bypassed.",,unclassified, CWE-304,EN-Missing Critical Step in Authentication (Type: Base),"The software implements an authentication technique, but it skips a step that weakens the technique. -Authentication techniques should follow the algorithms that define them exactly, otherwise authentication can be bypassed or more easily subjected to brute force attacks.",,unclassified, +Authentication techniques should follow the algorithms that define them exactly, otherwise authentication can be bypassed or more easily subjected to brute force attacks.",,unclassified, CWE-305,EN-Authentication Bypass by Primary Weakness (Type: Base),"The authentication algorithm is sound, but the implemented mechanism can be bypassed as the result of a separate weakness that is primary to the authentication error. -Authentication techniques should follow the algorithms that define them exactly, otherwise authentication can be bypassed or more easily subjected to brute force attacks.",,unclassified, +Authentication techniques should follow the algorithms that define them exactly, otherwise authentication can be bypassed or more easily subjected to brute force attacks.",,unclassified, CWE-307,EN-Improper Restriction of Excessive Authentication Attempts (Type: Base),"The software does not implement sufficient measures to prevent multiple failed authentication attempts within in a short time frame, making it more susceptible to brute force attacks. Authentication techniques should follow the algorithms that define them exactly, otherwise authentication can be bypassed or more easily subjected to brute force attacks.",,unclassified,"Weak Password Brings 'Happiness' to Twitter Hacker: http://www.wired.com/threatlevel/2009/01/professed-twitt/ -OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI" +OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI" CWE-31,EN-Path Traversal: dir\..\..\filename (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize 'dir\..\..\filename' (multiple internal backslash dot dot) sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The 'dir\..\..\filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only removes one ""..\"" sequence, so multiple ""..\"" can bypass that check. Alternately, this manipulation could be used to bypass a check for ""..\"" at the beginning of the pathname, moving up more than one directory level.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" +The 'dir\..\..\filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only removes one ""..\"" sequence, so multiple ""..\"" can bypass that check. Alternately, this manipulation could be used to bypass a check for ""..\"" at the beginning of the pathname, moving up more than one directory level.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-312,EN-Cleartext Storage of Sensitive Information (Type: Base),"The application stores sensitive information in cleartext within a resource that might be accessible to another control sphere. Because the information is stored in cleartext, attackers could potentially read it. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified,"Writing Secure Code: Chapter 9, ""Protecting Secret Data"" Page 299 The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Encryption"", Page 43. -Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/" +Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/" CWE-313,EN-Cleartext Storage in a File or on Disk (Type: Variant),"The application stores sensitive information in cleartext in a file, or on disk. -The sensitive information could be read by attackers with access to the file, or with physical or administrator access to the raw disk. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified, +The sensitive information could be read by attackers with access to the file, or with physical or administrator access to the raw disk. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified, CWE-314,EN-Cleartext Storage in the Registry (Type: Variant),"The application stores sensitive information in cleartext in the registry. -Attackers can read the information by accessing the registry key. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified, +Attackers can read the information by accessing the registry key. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified, CWE-315,EN-Cleartext Storage of Sensitive Information in a Cookie (Type: Variant),"The application stores sensitive information in cleartext in a cookie. -Attackers can use widely-available tools to view the cookie and read the sensitive information. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified, +Attackers can use widely-available tools to view the cookie and read the sensitive information. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified, CWE-316,EN-Cleartext Storage of Sensitive Information in Memory (Type: Variant),"The application stores sensitive information in cleartext in memory. The sensitive memory might be saved to disk, stored in a core dump, or remain uncleared if the application crashes, or if the programmer does not properly clear the memory before freeing it. -It could be argued that such problems are usually only exploitable by those with administrator privileges. However, swapping could cause the memory to be written to disk and leave it accessible to physical attack afterwards. Core dump files might have insecure permissions or be stored in archive files that are accessible to untrusted people. Or, uncleared sensitive memory might be inadvertently exposed to attackers due to another weakness.",,unclassified, +It could be argued that such problems are usually only exploitable by those with administrator privileges. However, swapping could cause the memory to be written to disk and leave it accessible to physical attack afterwards. Core dump files might have insecure permissions or be stored in archive files that are accessible to untrusted people. Or, uncleared sensitive memory might be inadvertently exposed to attackers due to another weakness.",,unclassified, CWE-317,EN-Cleartext Storage of Sensitive Information in GUI (Type: Variant),"The application stores sensitive information in cleartext within the GUI. -An attacker can often obtain data from a GUI, even if hidden, by using an API to directly access GUI objects such as windows and menus. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified, +An attacker can often obtain data from a GUI, even if hidden, by using an API to directly access GUI objects such as windows and menus. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified, CWE-318,EN-Cleartext Storage of Sensitive Information in Executable (Type: Variant),"The application stores sensitive information in cleartext in an executable. -Attackers can reverse engineer binary code to obtain secret data. This is especially easy when the cleartext is plain ASCII. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified, +Attackers can reverse engineer binary code to obtain secret data. This is especially easy when the cleartext is plain ASCII. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified, CWE-32,EN-Path Traversal: ... (Triple Dot) (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '...' (triple dot) sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '...' manipulation is useful for bypassing some path traversal protection schemes. On some Windows systems, it is equivalent to ""..\.."" and might bypass checks that assume only two dots are valid. Incomplete filtering, such as removal of ""./"" sequences, can ultimately produce valid "".."" sequences due to a collapse into unsafe value (CWE-182).",,unclassified, +The '...' manipulation is useful for bypassing some path traversal protection schemes. On some Windows systems, it is equivalent to ""..\.."" and might bypass checks that assume only two dots are valid. Incomplete filtering, such as removal of ""./"" sequences, can ultimately produce valid "".."" sequences due to a collapse into unsafe value (CWE-182).",,unclassified, CWE-325,EN-Missing Required Cryptographic Step (Type: Base),"The software does not implement a required step in a cryptographic algorithm, resulting in weaker encryption than advertised by that algorithm. -Cryptographic implementations should follow the algorithms that define them exactly, otherwise encryption can be weaker than expected.",,unclassified, +Cryptographic implementations should follow the algorithms that define them exactly, otherwise encryption can be weaker than expected.",,unclassified, CWE-326,EN-Inadequate Encryption Strength (Type: Class),"The software stores or transmits sensitive data using an encryption scheme that is theoretically sound, but is not strong enough for the level of protection required. A weak encryption scheme can be subjected to brute force attacks that have a reasonable chance of succeeding using current attack methods and resources.",,unclassified,"Writing Secure Code: Chapter 8, ""Cryptographic Foibles"" Page 259 -24 Deadly Sins of Software Security: ""Sin 21: Using the Wrong Cryptography."" Page 315" +24 Deadly Sins of Software Security: ""Sin 21: Using the Wrong Cryptography."" Page 315" CWE-328,EN-Reversible One-Way Hash (Type: Base),"The product uses a hashing algorithm that produces a hash value that can be used to determine the original input, or to find an input that can produce the same hash, more efficiently than brute force techniques. This weakness is especially dangerous when the hash is used in security algorithms that require the one-way property to hold. For example, if an authentication system takes an incoming password and generates a hash, then compares the hash to another hash that it has stored in its authentication database, then the ability to create a collision could allow an attacker to provide an alternate password that produces the same target hash, bypassing authentication.",,unclassified,"MD5 considered harmful today: http://www.phreedom.org/research/rogue-ca/ The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Integrity"", Page 47. @@ -1128,73 +1128,73 @@ Tarsnap - The scrypt key derivation function and encryption utility: http://www. How Companies Can Beef Up Password Security (interview with Thomas H. Ptacek): http://krebsonsecurity.com/2012/06/how-companies-can-beef-up-password-security/ Password security: past, present, future: http://www.openwall.com/presentations/PHDays2012-Password-Security/ Our password hashing has no clothes: http://www.troyhunt.com/2012/06/our-password-hashing-has-no-clothes.html -Should we really use bcrypt/scrypt?: http://www.analyticalengine.net/2012/06/should-we-really-use-bcryptscrypt/" +Should we really use bcrypt/scrypt?: http://www.analyticalengine.net/2012/06/should-we-really-use-bcryptscrypt/" CWE-33,EN-Path Traversal: .... (Multiple Dot) (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '....' (multiple dot) sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '....' manipulation is useful for bypassing some path traversal protection schemes. On some Windows systems, it is equivalent to ""..\..\.."" and might bypass checks that assume only two dots are valid. Incomplete filtering, such as removal of ""./"" sequences, can ultimately produce valid "".."" sequences due to a collapse into unsafe value (CWE-182).",,unclassified, +The '....' manipulation is useful for bypassing some path traversal protection schemes. On some Windows systems, it is equivalent to ""..\..\.."" and might bypass checks that assume only two dots are valid. Incomplete filtering, such as removal of ""./"" sequences, can ultimately produce valid "".."" sequences due to a collapse into unsafe value (CWE-182).",,unclassified, CWE-331,EN-Insufficient Entropy (Type: Base),"The software uses an algorithm or scheme that produces insufficient entropy, leaving patterns or clusters of values that are more likely to occur than others. -When software generates predictable values in a context requiring unpredictability, it may be possible for an attacker to guess the next value that will be generated, and use this guess to impersonate another user or access sensitive information.",,unclassified,Building Secure Software: How to Avoid Security Problems the Right Way +When software generates predictable values in a context requiring unpredictability, it may be possible for an attacker to guess the next value that will be generated, and use this guess to impersonate another user or access sensitive information.",,unclassified,Building Secure Software: How to Avoid Security Problems the Right Way CWE-334,EN-Small Space of Random Values (Type: Base),"The number of possible random values is smaller than needed by the product, making it more susceptible to brute force attacks. The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,unclassified,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf -24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" +24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-335,EN-PRNG Seed Error (Type: Class),"A Pseudo-Random Number Generator (PRNG) uses seeds incorrectly. -The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" +The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-336,EN-Same Seed in PRNG (Type: Base),"A PRNG uses the same seed each time the product is initialized. If an attacker can guess (or knows) the seed, then he/she may be able to determine the ""random"" number produced from the PRNG. -The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,unclassified,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf +The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,unclassified,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf CWE-337,EN-Predictable Seed in PRNG (Type: Base),"A PRNG is initialized from a predictable seed, e.g. using process ID or system time. The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,unclassified,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf -24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" +24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-339,EN-Small Seed Space in PRNG (Type: Base),"A PRNG uses a relatively small space of seeds. -The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,unclassified,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf +The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,unclassified,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf CWE-34,EN-Path Traversal: ....// (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '....//' (doubled dot dot slash) sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '....//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then ""....//"" can collapse into the ""../"" unsafe value (CWE-182). It could also be useful when "".."" is removed, if the operating system treats ""//"" and ""/"" as equivalent.",,unclassified, +The '....//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then ""....//"" can collapse into the ""../"" unsafe value (CWE-182). It could also be useful when "".."" is removed, if the operating system treats ""//"" and ""/"" as equivalent.",,unclassified, CWE-340,EN-Predictability Problems (Type: Class),"Weaknesses in this category are related to schemes that generate numbers or identifiers that are more predictable than required by the application. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '....//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then ""....//"" can collapse into the ""../"" unsafe value (CWE-182). It could also be useful when "".."" is removed, if the operating system treats ""//"" and ""/"" as equivalent.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" +The '....//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then ""....//"" can collapse into the ""../"" unsafe value (CWE-182). It could also be useful when "".."" is removed, if the operating system treats ""//"" and ""/"" as equivalent.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-341,EN-Predictable from Observable State (Type: Base),"A number or object is predictable based on observations that the attacker can make about the state of the system or network, such as time, process ID, etc. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. The '....//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then ""....//"" can collapse into the ""../"" unsafe value (CWE-182). It could also be useful when "".."" is removed, if the operating system treats ""//"" and ""/"" as equivalent.",,unclassified,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf -24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" +24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-342,EN-Predictable Exact Value from Previous Values (Type: Base),"An exact value or random number can be precisely predicted by observing previous values. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. The '....//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then ""....//"" can collapse into the ""../"" unsafe value (CWE-182). It could also be useful when "".."" is removed, if the operating system treats ""//"" and ""/"" as equivalent.",,unclassified,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf -24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" +24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-343,EN-Predictable Value Range from Previous Values (Type: Base),"The software's random number generator produces a series of values which, when observed, can be used to infer a relatively small range of possibilities for the next value that could be generated. The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf Strange Attractors and TCP/IP Sequence Number Analysis: http://www.bindview.com/Services/Razor/Papers/2001/tcpseq.cfm -24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" +24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299" CWE-344,EN-Use of Invariant Value in Dynamically Changing Context (Type: Base),"The product uses a constant value, name, or reference, but this value can (or should) vary across different environments. -The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf +The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf CWE-345,EN-Insufficient Verification of Data Authenticity (Type: Class),"The software does not sufficiently verify the origin or authenticity of data, in a way that causes it to accept invalid data. -The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 15: Not Updating Easily."" Page 231" +The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 15: Not Updating Easily."" Page 231" CWE-346,EN-Origin Validation Error (Type: Base),"The software does not properly verify that the source of data or communication is valid. -The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified, +The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified, CWE-347,EN-Improper Verification of Cryptographic Signature (Type: Base),"The software does not verify, or incorrectly verifies, the cryptographic signature for data. -The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified, +The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified, CWE-348,EN-Use of Less Trusted Source (Type: Base),"The software has two different sources of the same data or information, but it uses the source that has less support for verification, is less trusted, or is less resistant to attack. -The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified, +The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified, CWE-349,EN-Acceptance of Extraneous Untrusted Data With Trusted Data (Type: Base),"The software, when processing trusted data, accepts any untrusted data that is also included with the trusted data, treating the untrusted data as if it were trusted. -The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified, +The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified, CWE-35,EN-Path Traversal: .../...// (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '.../...//' (doubled triple dot slash) sequences that can resolve to a location that is outside of that directory. This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. -The '.../...//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then "".../...//"" can collapse into the ""../"" unsafe value (CWE-182). Removing the first ""../"" yields ""....//""; the second removal yields ""../"". Depending on the algorithm, the software could be susceptible to CWE-34 but not CWE-35, or vice versa.",,unclassified, +The '.../...//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then "".../...//"" can collapse into the ""../"" unsafe value (CWE-182). Removing the first ""../"" yields ""....//""; the second removal yields ""../"". Depending on the algorithm, the software could be susceptible to CWE-34 but not CWE-35, or vice versa.",,unclassified, CWE-350,EN-Reliance on Reverse DNS Resolution for a Security-Critical Action (Type: Variant),"The software performs reverse DNS resolution on an IP address to obtain the hostname and make a security decision, but it does not properly ensure that the IP address is truly associated with the hostname. When the software performs a reverse DNS resolution for an IP address, if an attacker controls the server for that IP address, then the attacker can cause the server to return an arbitrary hostname. As a result, the attacker may be able to bypass authentication, cause the wrong hostname to be recorded in log files to hide activities, or perform other attacks. Attackers can spoof DNS names by either (1) compromising a DNS server and modifying its records (sometimes called DNS cache poisoning), or (2) having legitimate control over a DNS server associated with their IP address. Since DNS names can be easily spoofed or misreported, and it may be difficult for the software to detect if a trusted DNS server has not been compromised, they do not constitute a valid authentication mechanism.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 15: Not Updating Easily."" Page 231 24 Deadly Sins of Software Security: ""Sin 24: Trusting Network Name Resolution."" Page 361 -The Art of Software Security Assessment: Chapter 16, ""DNS Spoofing"", Page 1002." +The Art of Software Security Assessment: Chapter 16, ""DNS Spoofing"", Page 1002." CWE-351,EN-Insufficient Type Distinction (Type: Base),"The software does not properly distinguish between different types of elements in a way that leads to insecure behavior. When the software performs a reverse DNS resolution for an IP address, if an attacker controls the server for that IP address, then the attacker can cause the server to return an arbitrary hostname. As a result, the attacker may be able to bypass authentication, cause the wrong hostname to be recorded in log files to hide activities, or perform other attacks. Attackers can spoof DNS names by either (1) compromising a DNS server and modifying its records (sometimes called DNS cache poisoning), or (2) having legitimate control over a DNS server associated with their IP address. -Since DNS names can be easily spoofed or misreported, and it may be difficult for the software to detect if a trusted DNS server has not been compromised, they do not constitute a valid authentication mechanism.",,unclassified, +Since DNS names can be easily spoofed or misreported, and it may be difficult for the software to detect if a trusted DNS server has not been compromised, they do not constitute a valid authentication mechanism.",,unclassified, CWE-356,EN-Product UI does not Warn User of Unsafe Actions (Type: Base),"The software's user interface does not warn the user before undertaking an unsafe action on behalf of that user. This makes it easier for attackers to trick users into inflicting damage to their system. -Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,unclassified, +Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,unclassified, CWE-357,EN-Insufficient UI Warning of Dangerous Operations (Type: Base),"The user interface provides a warning to a user regarding dangerous or sensitive operations, but the warning is not noticeable enough to warrant attention. -Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,unclassified, +Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,unclassified, CWE-358,EN-Improperly Implemented Security Check for Standard (Type: Base),"The software does not implement or incorrectly implements one or more security-relevant checks as specified by the design of a standardized algorithm, protocol, or technique. -Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,unclassified, +Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,unclassified, CWE-359,EN-Privacy Violation (Type: Class),"Mishandling private information, such as customer passwords or social security numbers, can compromise user privacy and is often illegal. Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,unclassified,"AOL man pleads guilty to selling 92m email addies: http://www.theregister.co.uk/2005/02/07/aol_email_theft/ Safe Harbor Privacy Framework: http://www.export.gov/safeharbor/ @@ -1202,106 +1202,106 @@ Financial Privacy: The Gramm-Leach Bliley Act (GLBA): http://www.ftc.gov/privacy Health Insurance Portability and Accountability Act (HIPAA): http://www.hhs.gov/ocr/hipaa/ California SB-1386: http://info.sen.ca.gov/pub/01-02/bill/sen/sb_1351-1400/sb_1386_bill_20020926_chaptered.html SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf -Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/" +Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/" CWE-36,EN-Absolute Path Traversal (Type: Base),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize absolute path sequences such as ""/abs/path"" that can resolve to a location that is outside of that directory. -This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified,"The Art of Software Security Assessment: Chapter 9, ""Filenames and Paths"", Page 503." +This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified,"The Art of Software Security Assessment: Chapter 9, ""Filenames and Paths"", Page 503." CWE-363,EN-Race Condition Enabling Link Following (Type: Base),"The software checks the status of a file or directory before accessing it, which produces a race condition in which the file can be replaced with a link before the access is performed, causing the software to access the wrong file. -While developers might expect that there is a very narrow time window between the time of check and time of use, there is still a race condition. An attacker could cause the software to slow down (e.g. with memory consumption), causing the time window to become larger. Alternately, in some situations, the attacker could win the race by performing a large number of attacks.",,unclassified,"The Art of Software Security Assessment: Chapter 9, ""Race Conditions"", Page 526." +While developers might expect that there is a very narrow time window between the time of check and time of use, there is still a race condition. An attacker could cause the software to slow down (e.g. with memory consumption), causing the time window to become larger. Alternately, in some situations, the attacker could win the race by performing a large number of attacks.",,unclassified,"The Art of Software Security Assessment: Chapter 9, ""Race Conditions"", Page 526." CWE-368,EN-Context Switching Race Condition (Type: Base),"A product performs a series of non-atomic actions to switch between contexts that cross privilege or other security boundaries, but a race condition allows an attacker to modify or misrepresent the product's behavior during the switch. -This is commonly seen in web browser vulnerabilities in which the attacker can perform certain actions while the browser is transitioning from a trusted to an untrusted domain, or vice versa, and the browser performs the actions on one domain using the trust level and resources of the other domain.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205" +This is commonly seen in web browser vulnerabilities in which the attacker can perform certain actions while the browser is transitioning from a trusted to an untrusted domain, or vice versa, and the browser performs the actions on one domain using the trust level and resources of the other domain.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205" CWE-37,EN-Path Traversal: /absolute/pathname/here (Type: Variant),"A software system that accepts input in the form of a slash absolute path ('/absolute/pathname/here') without appropriate validation can allow an attacker to traverse the file system to unintended locations or access arbitrary files. -This weakness typically occurs when an unexpected value is provided to the product, or if an error occurs that is not properly detected. It frequently occurs in calculations involving physical dimensions such as size, length, width, and height.",,unclassified, +This weakness typically occurs when an unexpected value is provided to the product, or if an error occurs that is not properly detected. It frequently occurs in calculations involving physical dimensions such as size, length, width, and height.",,unclassified, CWE-372,EN-Incomplete Internal State Distinction (Type: Base),"The software does not properly determine which state it is in, causing it to assume it is in state X when in fact it is in state Y, causing it to perform incorrect operations in a security-relevant manner. -If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,unclassified, +If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,unclassified, CWE-373,EN-DEPRECATED: State Synchronization Error (Type: Base),"This entry was deprecated because it overlapped the same concepts as race condition (CWE-362) and Improper Synchronization (CWE-662). -If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,unclassified, +If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,unclassified, CWE-377,EN-Insecure Temporary File (Type: Base),"Creating and using insecure temporary files can leave application and system data vulnerable to attack. If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,unclassified,"Writing Secure Code: Chapter 23, ""Creating Temporary Files Securely"" Page 682 The Art of Software Security Assessment: Chapter 9, ""Temporary Files"", Page 538. -The Art of Software Security Assessment: Chapter 11, ""File Squatting"", Page 662." +The Art of Software Security Assessment: Chapter 11, ""File Squatting"", Page 662." CWE-38,EN-Path Traversal: \absolute\pathname\here (Type: Variant),"A software system that accepts input in the form of a backslash absolute path ('\absolute\pathname\here') without appropriate validation can allow an attacker to traverse the file system to unintended locations or access arbitrary files. -On some operating systems, the fact that the temporary file exists may be apparent to any user with sufficient privileges to access that directory. Since the file is visible, the application that is using the temporary file could be known. If one has access to list the processes on the system, the attacker has gained information about what the user is doing at that time. By correlating this with the applications the user is running, an attacker could potentially discover what a user's actions are. From this, higher levels of security could be breached.",,unclassified, +On some operating systems, the fact that the temporary file exists may be apparent to any user with sufficient privileges to access that directory. Since the file is visible, the application that is using the temporary file could be known. If one has access to list the processes on the system, the attacker has gained information about what the user is doing at that time. By correlating this with the applications the user is running, an attacker could potentially discover what a user's actions are. From this, higher levels of security could be breached.",,unclassified, CWE-382,EN-J2EE Bad Practices: Use of System.exit() (Type: Variant),"A J2EE application uses System.exit(), which also shuts down its container. -On some operating systems, the fact that the temporary file exists may be apparent to any user with sufficient privileges to access that directory. Since the file is visible, the application that is using the temporary file could be known. If one has access to list the processes on the system, the attacker has gained information about what the user is doing at that time. By correlating this with the applications the user is running, an attacker could potentially discover what a user's actions are. From this, higher levels of security could be breached.",,unclassified, +On some operating systems, the fact that the temporary file exists may be apparent to any user with sufficient privileges to access that directory. Since the file is visible, the application that is using the temporary file could be known. If one has access to list the processes on the system, the attacker has gained information about what the user is doing at that time. By correlating this with the applications the user is running, an attacker could potentially discover what a user's actions are. From this, higher levels of security could be breached.",,unclassified, CWE-383,EN-J2EE Bad Practices: Direct Use of Threads (Type: Variant),"Thread management in a Web application is forbidden in some circumstances and is always highly error prone. -Thread management in a web application is forbidden by the J2EE standard in some circumstances and is always highly error prone. Managing threads is difficult and is likely to interfere in unpredictable ways with the behavior of the application container. Even without interfering with the container, thread management usually leads to bugs that are hard to detect and diagnose like deadlock, race conditions, and other synchronization errors.",,unclassified, +Thread management in a web application is forbidden by the J2EE standard in some circumstances and is always highly error prone. Managing threads is difficult and is likely to interfere in unpredictable ways with the behavior of the application container. Even without interfering with the container, thread management usually leads to bugs that are hard to detect and diagnose like deadlock, race conditions, and other synchronization errors.",,unclassified, CWE-386,EN-Symbolic Name not Mapping to Correct Object (Type: Base),"A constant symbolic reference to an object is used, even though the reference can resolve to a different object over time. In some instances, knowing when data is transmitted between parties can provide a malicious user with privileged information. Also, externally monitoring the timing of operations can potentially reveal sensitive data. For example, a cryptographic operation can expose its internal state if the time it takes to perform the operation varies, based on the state. -Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,unclassified, +Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,unclassified, CWE-39,EN-Path Traversal: C:dirname (Type: Variant),"An attacker can inject a drive letter or Windows volume letter ('C:dirname') into a software system to potentially redirect access to an unintended location or arbitrary file. In some instances, knowing when data is transmitted between parties can provide a malicious user with privileged information. Also, externally monitoring the timing of operations can potentially reveal sensitive data. For example, a cryptographic operation can expose its internal state if the time it takes to perform the operation varies, based on the state. -Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,unclassified, +Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,unclassified, CWE-392,EN-Missing Report of Error Condition (Type: Base),"The software encounters an error but does not provide a status code or return value to indicate that an error has occurred. In some instances, knowing when data is transmitted between parties can provide a malicious user with privileged information. Also, externally monitoring the timing of operations can potentially reveal sensitive data. For example, a cryptographic operation can expose its internal state if the time it takes to perform the operation varies, based on the state. -Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,unclassified, +Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,unclassified, CWE-393,EN-Return of Wrong Status Code (Type: Base),"A function or operation returns an incorrect return value or status code that does not indicate an error, but causes the product to modify its behavior based on the incorrect result. -This can lead to unpredictable behavior. If the function is used to make security-critical decisions or provide security-critical information, then the wrong status code can cause the software to assume that an action is safe, even when it is not.",,unclassified, +This can lead to unpredictable behavior. If the function is used to make security-critical decisions or provide security-critical information, then the wrong status code can cause the software to assume that an action is safe, even when it is not.",,unclassified, CWE-394,EN-Unexpected Status Code or Return Value (Type: Base),"The software does not properly check when a function or operation returns a value that is legitimate for the function, but is not expected by the software. -This can lead to unpredictable behavior. If the function is used to make security-critical decisions or provide security-critical information, then the wrong status code can cause the software to assume that an action is safe, even when it is not.",,unclassified, +This can lead to unpredictable behavior. If the function is used to make security-critical decisions or provide security-critical information, then the wrong status code can cause the software to assume that an action is safe, even when it is not.",,unclassified, CWE-395,EN-Use of NullPointerException Catch to Detect NULL Pointer Dereference (Type: Base),"Catching NullPointerException should not be used as an alternative to programmatic checks to prevent dereferencing a null pointer. -This can lead to unpredictable behavior. If the function is used to make security-critical decisions or provide security-critical information, then the wrong status code can cause the software to assume that an action is safe, even when it is not.",,unclassified, +This can lead to unpredictable behavior. If the function is used to make security-critical decisions or provide security-critical information, then the wrong status code can cause the software to assume that an action is safe, even when it is not.",,unclassified, CWE-396,EN-Declaration of Catch for Generic Exception (Type: Base),"Catching overly broad exceptions promotes complex error handling code that is more likely to contain security vulnerabilities. -Multiple catch blocks can get ugly and repetitive, but ""condensing"" catch blocks by catching a high-level class like Exception can obscure exceptions that deserve special treatment or that should not be caught at this point in the program. Catching an overly broad exception essentially defeats the purpose of Java's typed exceptions, and can become particularly dangerous if the program grows and begins to throw new types of exceptions. The new exception types will not receive any attention.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 9: Catching Exceptions."" Page 157" +Multiple catch blocks can get ugly and repetitive, but ""condensing"" catch blocks by catching a high-level class like Exception can obscure exceptions that deserve special treatment or that should not be caught at this point in the program. Catching an overly broad exception essentially defeats the purpose of Java's typed exceptions, and can become particularly dangerous if the program grows and begins to throw new types of exceptions. The new exception types will not receive any attention.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 9: Catching Exceptions."" Page 157" CWE-397,EN-Declaration of Throws for Generic Exception (Type: Base),"Throwing overly broad exceptions promotes complex error handling code that is more likely to contain security vulnerabilities. -Declaring a method to throw Exception or Throwable makes it difficult for callers to perform proper error handling and error recovery. Java's exception mechanism, for example, is set up to make it easy for callers to anticipate what can go wrong and write code to handle each specific exceptional circumstance. Declaring that a method throws a generic form of exception defeats this system.",,unclassified, +Declaring a method to throw Exception or Throwable makes it difficult for callers to perform proper error handling and error recovery. Java's exception mechanism, for example, is set up to make it easy for callers to anticipate what can go wrong and write code to handle each specific exceptional circumstance. Declaring that a method throws a generic form of exception defeats this system.",,unclassified, CWE-398,EN-Indicator of Poor Code Quality (Type: Class),"The code has features that do not directly introduce a weakness or vulnerability, but indicate that the product has not been carefully developed or maintained. -Programs are more likely to be secure when good development practices are followed. If a program is complex, difficult to maintain, not portable, or shows evidence of neglect, then there is a higher likelihood that weaknesses are buried in the code.",,unclassified, +Programs are more likely to be secure when good development practices are followed. If a program is complex, difficult to maintain, not portable, or shows evidence of neglect, then there is a higher likelihood that weaknesses are buried in the code.",,unclassified, CWE-40,EN-Path Traversal: \\UNC\share\name\ (Windows UNC Share) (Type: Variant),"An attacker can inject a Windows UNC share ('\\UNC\share\name') into a software system to potentially redirect access to an unintended location or arbitrary file. -Programs are more likely to be secure when good development practices are followed. If a program is complex, difficult to maintain, not portable, or shows evidence of neglect, then there is a higher likelihood that weaknesses are buried in the code.",,unclassified,"The Art of Software Security Assessment: Chapter 11, ""Filelike Objects"", Page 664." +Programs are more likely to be secure when good development practices are followed. If a program is complex, difficult to maintain, not portable, or shows evidence of neglect, then there is a higher likelihood that weaknesses are buried in the code.",,unclassified,"The Art of Software Security Assessment: Chapter 11, ""Filelike Objects"", Page 664." CWE-402,EN-Transmission of Private Resources into a New Sphere (Resource Leak) (Type: Class),"The software makes resources available to untrusted parties when those resources are only intended to be accessed by the software. -This is often triggered by improper handling of malformed data or unexpectedly interrupted sessions.",,unclassified, +This is often triggered by improper handling of malformed data or unexpectedly interrupted sessions.",,unclassified, CWE-403,EN-Exposure of File Descriptor to Unintended Control Sphere (File Descriptor Leak) (Type: Base),"A process does not close sensitive file descriptors before invoking a child process, which allows the child to perform unauthorized I/O operations using those descriptors. When a new process is forked or executed, the child process inherits any open file descriptors. When the child process has fewer privileges than the parent process, this might introduce a vulnerability if the child process can access the file descriptor but does not have the privileges to access the associated file.",,unclassified,"File descriptors and setuid applications: https://blogs.oracle.com/paulr/entry/file_descriptors_and_setuid_applications -Introduction to Secure Coding Guide: https://developer.apple.com/library/mac/#documentation/security/conceptual/SecureCodingGuide/Articles/AccessControl.html" +Introduction to Secure Coding Guide: https://developer.apple.com/library/mac/#documentation/security/conceptual/SecureCodingGuide/Articles/AccessControl.html" CWE-405,EN-Asymmetric Resource Consumption (Amplification) (Type: Class),"Software that does not appropriately monitor or control resource consumption can lead to adverse system performance. -This situation is amplified if the software allows malicious users or attackers to consume more resources than their access level permits. Exploiting such a weakness can lead to asymmetric resource consumption, aiding in amplification attacks against the system or the network.",,unclassified, +This situation is amplified if the software allows malicious users or attackers to consume more resources than their access level permits. Exploiting such a weakness can lead to asymmetric resource consumption, aiding in amplification attacks against the system or the network.",,unclassified, CWE-406,EN-Insufficient Control of Network Message Volume (Network Amplification) (Type: Base),"The software does not sufficiently monitor or control transmitted network traffic volume, so that an actor can cause the software to transmit more traffic than should be allowed for that actor. -In the absence of a policy to restrict asymmetric resource consumption, the application or system cannot distinguish between legitimate transmissions and traffic intended to serve as an amplifying attack on target systems. Systems can often be configured to restrict the amount of traffic sent out on behalf of a client, based on the client's origin or access level. This is usually defined in a resource allocation policy. In the absence of a mechanism to keep track of transmissions, the system or application can be easily abused to transmit asymmetrically greater traffic than the request or client should be permitted to.",,unclassified, +In the absence of a policy to restrict asymmetric resource consumption, the application or system cannot distinguish between legitimate transmissions and traffic intended to serve as an amplifying attack on target systems. Systems can often be configured to restrict the amount of traffic sent out on behalf of a client, based on the client's origin or access level. This is usually defined in a resource allocation policy. In the absence of a mechanism to keep track of transmissions, the system or application can be easily abused to transmit asymmetrically greater traffic than the request or client should be permitted to.",,unclassified, CWE-408,EN-Incorrect Behavior Order: Early Amplification (Type: Base),"The software allows an entity to perform a legitimate but expensive operation before authentication or authorization has taken place. -In the absence of a policy to restrict asymmetric resource consumption, the application or system cannot distinguish between legitimate transmissions and traffic intended to serve as an amplifying attack on target systems. Systems can often be configured to restrict the amount of traffic sent out on behalf of a client, based on the client's origin or access level. This is usually defined in a resource allocation policy. In the absence of a mechanism to keep track of transmissions, the system or application can be easily abused to transmit asymmetrically greater traffic than the request or client should be permitted to.",,unclassified, +In the absence of a policy to restrict asymmetric resource consumption, the application or system cannot distinguish between legitimate transmissions and traffic intended to serve as an amplifying attack on target systems. Systems can often be configured to restrict the amount of traffic sent out on behalf of a client, based on the client's origin or access level. This is usually defined in a resource allocation policy. In the absence of a mechanism to keep track of transmissions, the system or application can be easily abused to transmit asymmetrically greater traffic than the request or client should be permitted to.",,unclassified, CWE-409,EN-Improper Handling of Highly Compressed Data (Data Amplification) (Type: Base),"The software does not handle or incorrectly handles a compressed input with a very high compression ratio that produces a large output. -An example of data amplification is a ""decompression bomb,"" a small ZIP file that can produce a large amount of data when it is decompressed.",,unclassified, +An example of data amplification is a ""decompression bomb,"" a small ZIP file that can produce a large amount of data when it is decompressed.",,unclassified, CWE-41,EN-Improper Resolution of Path Equivalence (Type: Base),"The system or application is vulnerable to file system contents disclosure through path equivalence. Path equivalence involves the use of special characters in file and directory names. The associated manipulations are intended to generate multiple names for the same object. -Path equivalence is usually employed in order to circumvent access controls expressed using an incomplete set of file name or file path representations. This is different from path traversal, wherein the manipulations are performed to generate a name for a different object.",,unclassified, +Path equivalence is usually employed in order to circumvent access controls expressed using an incomplete set of file name or file path representations. This is different from path traversal, wherein the manipulations are performed to generate a name for a different object.",,unclassified, CWE-410,EN-Insufficient Resource Pool (Type: Base),"The software's resource pool is not large enough to handle peak demand, which allows an attacker to prevent others from accessing the resource by using a (relatively) large number of requests for resources. -Frequently the consequence is a ""flood"" of connection or sessions.",,unclassified,"Writing Secure Code: Chapter 17, ""Protecting Against Denial of Service Attacks"" Page 517" +Frequently the consequence is a ""flood"" of connection or sessions.",,unclassified,"Writing Secure Code: Chapter 17, ""Protecting Against Denial of Service Attacks"" Page 517" CWE-412,EN-Unrestricted Externally Accessible Lock (Type: Base),"The software properly checks for the existence of a lock, but the lock can be externally controlled or influenced by an actor that is outside of the intended sphere of control. -This prevents the software from acting on associated resources or performing other behaviors that are controlled by the presence of the lock. Relevant locks might include an exclusive lock or mutex, or modifying a shared resource that is treated as a lock. If the lock can be held for an indefinite period of time, then the denial of service could be permanent.",,unclassified, +This prevents the software from acting on associated resources or performing other behaviors that are controlled by the presence of the lock. Relevant locks might include an exclusive lock or mutex, or modifying a shared resource that is treated as a lock. If the lock can be held for an indefinite period of time, then the denial of service could be permanent.",,unclassified, CWE-413,EN-Improper Resource Locking (Type: Base),"The software does not lock or does not correctly lock a resource when the software must have exclusive access to the resource. -When a resource is not properly locked, an attacker could modify the resource while it is being operated on by the software. This might violate the software's assumption that the resource will not change, potentially leading to unexpected behaviors.",,unclassified, +When a resource is not properly locked, an attacker could modify the resource while it is being operated on by the software. This might violate the software's assumption that the resource will not change, potentially leading to unexpected behaviors.",,unclassified, CWE-414,EN-Missing Lock Check (Type: Base),"A product does not check to see if a lock is present before performing sensitive operations on a resource. -When a resource is not properly locked, an attacker could modify the resource while it is being operated on by the software. This might violate the software's assumption that the resource will not change, potentially leading to unexpected behaviors.",,unclassified, +When a resource is not properly locked, an attacker could modify the resource while it is being operated on by the software. This might violate the software's assumption that the resource will not change, potentially leading to unexpected behaviors.",,unclassified, CWE-419,EN-Unprotected Primary Channel (Type: Base),"The software uses a primary channel for administration or restricted functionality, but it does not properly protect the channel. The use of previously-freed memory can have any number of adverse consequences, ranging from the corruption of valid data to the execution of arbitrary code, depending on the instantiation and timing of the flaw. The simplest way data corruption may occur involves the system's reuse of the freed memory. Use-after-free errors have two common and sometimes overlapping causes: Error conditions and other exceptional circumstances. Confusion over which part of the program is responsible for freeing the memory. In this scenario, the memory in question is allocated to another pointer validly at some point after it has been freed. The original pointer to the freed memory is used again and points to somewhere within the new allocation. As the data is changed, it corrupts the validly used memory; this induces undefined behavior in the process. -If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,unclassified, +If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,unclassified, CWE-42,EN-Path Equivalence: filename. (Trailing Dot) (Type: Variant),"A software system that accepts path input in the form of trailing dot ('filedir.') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. The use of previously-freed memory can have any number of adverse consequences, ranging from the corruption of valid data to the execution of arbitrary code, depending on the instantiation and timing of the flaw. The simplest way data corruption may occur involves the system's reuse of the freed memory. Use-after-free errors have two common and sometimes overlapping causes: Error conditions and other exceptional circumstances. Confusion over which part of the program is responsible for freeing the memory. In this scenario, the memory in question is allocated to another pointer validly at some point after it has been freed. The original pointer to the freed memory is used again and points to somewhere within the new allocation. As the data is changed, it corrupts the validly used memory; this induces undefined behavior in the process. -If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,unclassified, +If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,unclassified, CWE-420,EN-Unprotected Alternate Channel (Type: Base),"The software protects a primary channel, but it does not use the same level of protection for an alternate channel. The use of previously-freed memory can have any number of adverse consequences, ranging from the corruption of valid data to the execution of arbitrary code, depending on the instantiation and timing of the flaw. The simplest way data corruption may occur involves the system's reuse of the freed memory. Use-after-free errors have two common and sometimes overlapping causes: Error conditions and other exceptional circumstances. Confusion over which part of the program is responsible for freeing the memory. In this scenario, the memory in question is allocated to another pointer validly at some point after it has been freed. The original pointer to the freed memory is used again and points to somewhere within the new allocation. As the data is changed, it corrupts the validly used memory; this induces undefined behavior in the process. -If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,unclassified, +If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,unclassified, CWE-421,EN-Race Condition During Access to Alternate Channel (Type: Base),"The product opens an alternate channel to communicate with an authorized user, but the channel is accessible to other actors. This creates a race condition that allows an attacker to access the channel before the authorized user does.",,unclassified,"Discovering and Exploiting Named Pipe Security Flaws for Fun and Profit: http://www.blakewatts.com/namedpipepaper.html -24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205" +24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205" CWE-422,EN-Unprotected Windows Messaging Channel (Shatter) (Type: Variant),"The software does not properly verify the source of a message in the Windows Messaging System while running at elevated privileges, creating an alternate channel through which an attacker can directly send a message to the product. This creates a race condition that allows an attacker to access the channel before the authorized user does.",,unclassified,"Exploiting design flaws in the Win32 API for privilege escalation. Or... Shatter Attacks - How to break Windows: http://web.archive.org/web/20060115174629/http://security.tombom.co.uk/shatter.html The Art of Software Security Assessment: Chapter 2, ""Design Review."" Page 34. -The Art of Software Security Assessment: Chapter 12, ""Shatter Attacks"", Page 694." +The Art of Software Security Assessment: Chapter 12, ""Shatter Attacks"", Page 694." CWE-423,EN-DEPRECATED (Duplicate): Proxied Trusted Channel (Type: Base),"This entry has been deprecated because it was a duplicate of CWE-441. All content has been transferred to CWE-441. -This creates a race condition that allows an attacker to access the channel before the authorized user does.",,unclassified, +This creates a race condition that allows an attacker to access the channel before the authorized user does.",,unclassified, CWE-424,EN-Improper Protection of Alternate Path (Type: Class),"The product does not sufficiently protect all possible paths that a user can take to access restricted functionality or resources. -This creates a race condition that allows an attacker to access the channel before the authorized user does.",,unclassified, +This creates a race condition that allows an attacker to access the channel before the authorized user does.",,unclassified, CWE-425,EN-Direct Request (Forced Browsing) (Type: Base),"The web application does not adequately enforce appropriate authorization on all restricted URLs, scripts, or files. -Web applications susceptible to direct request attacks often make the false assumption that such resources can only be reached through a given navigation path and so only apply authorization at certain points in the path.",,unclassified, +Web applications susceptible to direct request attacks often make the false assumption that such resources can only be reached through a given navigation path and so only apply authorization at certain points in the path.",,unclassified, CWE-427,EN-Uncontrolled Search Path Element (Type: Base),"The product uses a fixed or controlled search path to find resources, but one or more locations in that path can be under the control of unintended actors. Although this weakness can occur with any type of resource, it is frequently introduced when a product uses a directory search path to find executables or code libraries, but the path contains a directory that can be modified by an attacker, such as ""/tmp"" or the current working directory. In Windows-based systems, when the LoadLibrary or LoadLibraryEx function is called with a DLL name that does not contain a fully qualified path, the function follows a search order that includes two path elements that might be uncontrolled: @@ -1310,355 +1310,355 @@ the current working directory. In some cases, the attack can be conducted remotely, such as when SMB or WebDAV network shares are used. In some Unix-based systems, a PATH might be created that contains an empty element, e.g. by splicing an empty variable into the PATH. This empty element can be interpreted as equivalent to the current working directory, which might be an untrusted search element.",,unclassified,"Double clicking on MS Office documents from Windows Explorer may execute arbitrary programs in some cases ACROS Security: Remote Binary Planting in Apple iTunes for Windows (ASPR #2010-08-18-1) -Automatic Detection of Vulnerable Dynamic Component Loadings: http://www.cs.ucdavis.edu/research/tech-reports/2010/CSE-2010-2.pdf +Automatic Detection of Vulnerable Dynamic Component Loadings: http://www.cs.ucdavis.edu/research/tech-reports/2010/CSE-2010-2.pdf Dynamic-Link Library Search Order: http://msdn.microsoft.com/en-us/library/ms682586%28v=VS.85%29.aspx Dynamic-Link Library Security: http://msdn.microsoft.com/en-us/library/ff919712%28VS.85%29.aspx An update on the DLL-preloading remote attack vector: http://blogs.technet.com/b/srd/archive/2010/08/23/an-update-on-the-dll-preloading-remote-attack-vector.aspx Insecure Library Loading Could Allow Remote Code Execution: http://www.microsoft.com/technet/security/advisory/2269637.mspx Application DLL Load Hijacking: http://blog.rapid7.com/?p=5325 -DLL Hijacking: Facts and Fiction: http://threatpost.com/en_us/blogs/dll-hijacking-facts-and-fiction-082610" +DLL Hijacking: Facts and Fiction: http://threatpost.com/en_us/blogs/dll-hijacking-facts-and-fiction-082610" CWE-428,EN-Unquoted Search Path or Element (Type: Base),"The product uses a search path that contains an unquoted element, in which the element contains whitespace or other separators. This can cause the product to access resources in a parent path. -If a malicious individual has access to the file system, it is possible to elevate privileges by inserting such a file as ""C:\Program.exe"" to be run by a privileged program making use of WinExec.",,unclassified,"The Art of Software Security Assessment: Chapter 11, ""Process Loading"", Page 654." +If a malicious individual has access to the file system, it is possible to elevate privileges by inserting such a file as ""C:\Program.exe"" to be run by a privileged program making use of WinExec.",,unclassified,"The Art of Software Security Assessment: Chapter 11, ""Process Loading"", Page 654." CWE-43,EN-Path Equivalence: filename.... (Multiple Trailing Dot) (Type: Variant),"A software system that accepts path input in the form of multiple trailing dot ('filedir....') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -If a malicious individual has access to the file system, it is possible to elevate privileges by inserting such a file as ""C:\Program.exe"" to be run by a privileged program making use of WinExec.",,unclassified, +If a malicious individual has access to the file system, it is possible to elevate privileges by inserting such a file as ""C:\Program.exe"" to be run by a privileged program making use of WinExec.",,unclassified, CWE-430,EN-Deployment of Wrong Handler (Type: Base),"The wrong ""handler"" is assigned to process an object. -An example of deploying the wrong handler would be calling a servlet to reveal source code of a .JSP file, or automatically ""determining"" type of the object even if it is contradictory to an explicitly specified type.",,unclassified,"The Art of Software Security Assessment: Chapter 3, ""File Handlers"", Page 74." +An example of deploying the wrong handler would be calling a servlet to reveal source code of a .JSP file, or automatically ""determining"" type of the object even if it is contradictory to an explicitly specified type.",,unclassified,"The Art of Software Security Assessment: Chapter 3, ""File Handlers"", Page 74." CWE-431,EN-Missing Handler (Type: Base),"A handler is not available or implemented. -When an exception is thrown and not caught, the process has given up an opportunity to decide if a given failure or event is worth a change in execution.",,unclassified,"The Art of Software Security Assessment: Chapter 3, ""File Handlers"", Page 74." +When an exception is thrown and not caught, the process has given up an opportunity to decide if a given failure or event is worth a change in execution.",,unclassified,"The Art of Software Security Assessment: Chapter 3, ""File Handlers"", Page 74." CWE-432,EN-Dangerous Signal Handler not Disabled During Sensitive Operations (Type: Base),"The application uses a signal handler that shares state with other signal handlers, but it does not properly mask or prevent those signal handlers from being invoked while the original signal handler is still running. -During the execution of a signal handler, it can be interrupted by another handler when a different signal is sent. If the two handlers share state - such as global variables - then an attacker can corrupt the state by sending another signal before the first handler has completed execution.",,unclassified, +During the execution of a signal handler, it can be interrupted by another handler when a different signal is sent. If the two handlers share state - such as global variables - then an attacker can corrupt the state by sending another signal before the first handler has completed execution.",,unclassified, CWE-433,EN-Unparsed Raw Web Content Delivery (Type: Variant),"The software stores raw content or supporting code under the web document root with an extension that is not specifically handled by the server. -If code is stored in a file with an extension such as "".inc"" or "".pl"", and the web server does not have a handler for that extension, then the server will likely send the contents of the file directly to the requester without the pre-processing that was expected. When that file contains sensitive information such as database credentials, this may allow the attacker to compromise the application or associated components.",,unclassified,"The Art of Software Security Assessment: Chapter 3, ""File Handlers"", Page 74." +If code is stored in a file with an extension such as "".inc"" or "".pl"", and the web server does not have a handler for that extension, then the server will likely send the contents of the file directly to the requester without the pre-processing that was expected. When that file contains sensitive information such as database credentials, this may allow the attacker to compromise the application or associated components.",,unclassified,"The Art of Software Security Assessment: Chapter 3, ""File Handlers"", Page 74." CWE-435,EN-Interaction Error (Type: Class),"An interaction error occurs when two entities work correctly when running independently, but they interact in unexpected ways when they are run together. -This could apply to products, systems, components, etc.",,unclassified, +This could apply to products, systems, components, etc.",,unclassified, CWE-436,EN-Interpretation Conflict (Type: Base),"Product A handles inputs or steps differently than Product B, which causes A to perform incorrect actions based on its perception of B's state. This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified,"On Interpretation Conflict Vulnerabilities Insertion, Evasion, and Denial of Service: Eluding Network Intrusion Detection: http://www.insecure.org/stf/secnet_ids/secnet_ids.pdf 0x00 vs ASP file upload scripts: http://www.security-assessment.com/Whitepapers/0x00_vs_ASP_File_Uploads.pdf Poison NULL byte -Re: Corsaire Security Advisory - Multiple vendor MIME RFC2047 encoding: http://marc.theaimsgroup.com/?l=bugtraq&m=109525864717484&w=2" +Re: Corsaire Security Advisory - Multiple vendor MIME RFC2047 encoding: http://marc.theaimsgroup.com/?l=bugtraq&m=109525864717484&w=2" CWE-437,EN-Incomplete Model of Endpoint Features (Type: Base),"A product acts as an intermediary or monitor between two or more endpoints, but it does not have a complete model of an endpoint's features, behaviors, or state, potentially causing the product to perform incorrect actions based on this incomplete model. -This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified, +This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified, CWE-439,EN-Behavioral Change in New Version or Environment (Type: Base),"A's behavior or functionality changes with a new version of A, or a new environment, which is not known (or manageable) by B. -This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified, +This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified, CWE-44,EN-Path Equivalence: file.name (Internal Dot) (Type: Variant),"A software system that accepts path input in the form of internal dot ('file.ordir') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified, +This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified, CWE-440,EN-Expected Behavior Violation (Type: Base),"A feature, API, or function being used by a product behaves differently than the product expects. -This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified, +This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified, CWE-441,EN-Unintended Proxy or Intermediary (Confused Deputy) (Type: Class),"The software receives a request, message, or directive from an upstream component, but the software does not sufficiently preserve the original source of the request before forwarding the request to an external actor that is outside of the software's control sphere. This causes the software to appear to be the source of the request, leading it to act as a proxy or other intermediary between the upstream component and the external actor. If an attacker cannot directly contact a target, but the software has access to the target, then the attacker can send a request to the software and have it be forwarded from the target. The request would appear to be coming from the software's system, not the attacker's system. As a result, the attacker can bypass access controls (such as firewalls) or hide the source of malicious requests, since the requests would not be coming directly from the attacker. Since proxy functionality and message-forwarding often serve a legitimate purpose, this issue only becomes a vulnerability when: The software runs with different privileges or on a different system, or otherwise has different levels of access than the upstream component; The attacker is prevented from making the request directly to the target; and -The attacker can create a request that the proxy does not explicitly intend to be forwarded on the behalf of the requester. Such a request might point to an unexpected hostname, port number, or service. Or, the request might be sent to an allowed service, but the request could contain disallowed directives, commands, or resources.",,unclassified,The Confused Deputy (or why capabilities might have been invented): http://www.cap-lore.com/CapTheory/ConfusedDeputy.html +The attacker can create a request that the proxy does not explicitly intend to be forwarded on the behalf of the requester. Such a request might point to an unexpected hostname, port number, or service. Or, the request might be sent to an allowed service, but the request could contain disallowed directives, commands, or resources.",,unclassified,The Confused Deputy (or why capabilities might have been invented): http://www.cap-lore.com/CapTheory/ConfusedDeputy.html CWE-443,EN-DEPRECATED (Duplicate): HTTP response splitting (Type: Base),"This weakness can be found at CWE-113. If an attacker cannot directly contact a target, but the software has access to the target, then the attacker can send a request to the software and have it be forwarded from the target. The request would appear to be coming from the software's system, not the attacker's system. As a result, the attacker can bypass access controls (such as firewalls) or hide the source of malicious requests, since the requests would not be coming directly from the attacker. Since proxy functionality and message-forwarding often serve a legitimate purpose, this issue only becomes a vulnerability when: The software runs with different privileges or on a different system, or otherwise has different levels of access than the upstream component; The attacker is prevented from making the request directly to the target; and -The attacker can create a request that the proxy does not explicitly intend to be forwarded on the behalf of the requester. Such a request might point to an unexpected hostname, port number, or service. Or, the request might be sent to an allowed service, but the request could contain disallowed directives, commands, or resources.",,unclassified, +The attacker can create a request that the proxy does not explicitly intend to be forwarded on the behalf of the requester. Such a request might point to an unexpected hostname, port number, or service. Or, the request might be sent to an allowed service, but the request could contain disallowed directives, commands, or resources.",,unclassified, CWE-444,EN-Inconsistent Interpretation of HTTP Requests (HTTP Request Smuggling) (Type: Base),"When malformed or abnormal HTTP requests are interpreted by one or more entities in the data flow between the user and the web server, such as a proxy or firewall, they can be interpreted inconsistently, allowing the attacker to ""smuggle"" a request to one device without the other device being aware of it. If an attacker cannot directly contact a target, but the software has access to the target, then the attacker can send a request to the software and have it be forwarded from the target. The request would appear to be coming from the software's system, not the attacker's system. As a result, the attacker can bypass access controls (such as firewalls) or hide the source of malicious requests, since the requests would not be coming directly from the attacker. Since proxy functionality and message-forwarding often serve a legitimate purpose, this issue only becomes a vulnerability when: The software runs with different privileges or on a different system, or otherwise has different levels of access than the upstream component; The attacker is prevented from making the request directly to the target; and -The attacker can create a request that the proxy does not explicitly intend to be forwarded on the behalf of the requester. Such a request might point to an unexpected hostname, port number, or service. Or, the request might be sent to an allowed service, but the request could contain disallowed directives, commands, or resources.",,unclassified,HTTP Request Smuggling: http://www.cgisecurity.com/lib/HTTP-Request-Smuggling.pdf +The attacker can create a request that the proxy does not explicitly intend to be forwarded on the behalf of the requester. Such a request might point to an unexpected hostname, port number, or service. Or, the request might be sent to an allowed service, but the request could contain disallowed directives, commands, or resources.",,unclassified,HTTP Request Smuggling: http://www.cgisecurity.com/lib/HTTP-Request-Smuggling.pdf CWE-446,EN-UI Discrepancy for Security Feature (Type: Base),"The user interface does not correctly enable or configure a security feature, but the interface provides feedback that causes the user to believe that the feature is in a secure state. -When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, +When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, CWE-447,EN-Unimplemented or Unsupported Feature in UI (Type: Base),"A UI function for a security feature appears to be supported and gives feedback to the user that suggests that it is supported, but the underlying functionality is not implemented. -When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, +When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, CWE-448,EN-Obsolete Feature in UI (Type: Base),"A UI function is obsolete and the product does not warn the user. -When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, +When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, CWE-449,EN-The UI Performs the Wrong Action (Type: Base),"The UI performs the wrong action with respect to the user's request. -When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, +When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, CWE-45,EN-Path Equivalence: file...name (Multiple Internal Dot) (Type: Variant),"A software system that accepts path input in the form of multiple internal dot ('file...dir') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, +When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, CWE-450,EN-Multiple Interpretations of UI Input (Type: Base),"The UI has multiple interpretations of user input but does not prompt the user when it selects the less secure interpretation. -When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, +When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, CWE-451,EN-UI Misrepresentation of Critical Information (Type: Base),"The UI does not properly represent critical information to the user, allowing the information - or its source - to be obscured or spoofed. This is often a component in phishing attacks. -When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, +When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, CWE-453,EN-Insecure Default Variable Initialization (Type: Base),"The software, by default, initializes an internal variable with an insecure or less secure value than is possible. -When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, +When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified, CWE-454,EN-External Initialization of Trusted Variables or Data Stores (Type: Base),"The software initializes critical internal variables or data stores using inputs that can be modified by untrusted actors. -A software system should be reluctant to trust variables that have been initialized outside of its trust boundary, especially if they are initialized by users. They may have been initialized incorrectly. If an attacker can initialize the variable, then he/she can influence what the vulnerable system will do.",,unclassified, +A software system should be reluctant to trust variables that have been initialized outside of its trust boundary, especially if they are initialized by users. They may have been initialized incorrectly. If an attacker can initialize the variable, then he/she can influence what the vulnerable system will do.",,unclassified, CWE-455,EN-Non-exit on Failed Initialization (Type: Base),"The software does not exit or otherwise modify its operation when security-relevant errors occur during initialization, such as when a configuration file has a format error, which can cause the software to execute in a less secure fashion than intended by the administrator. -A software system should be reluctant to trust variables that have been initialized outside of its trust boundary, especially if they are initialized by users. They may have been initialized incorrectly. If an attacker can initialize the variable, then he/she can influence what the vulnerable system will do.",,unclassified, +A software system should be reluctant to trust variables that have been initialized outside of its trust boundary, especially if they are initialized by users. They may have been initialized incorrectly. If an attacker can initialize the variable, then he/she can influence what the vulnerable system will do.",,unclassified, CWE-456,EN-Missing Initialization of a Variable (Type: Base),"The software does not initialize critical variables, which causes the execution environment to use unexpected values. -A software system should be reluctant to trust variables that have been initialized outside of its trust boundary, especially if they are initialized by users. They may have been initialized incorrectly. If an attacker can initialize the variable, then he/she can influence what the vulnerable system will do.",,unclassified,"The Art of Software Security Assessment: Chapter 7, ""Variable Initialization"", Page 312." +A software system should be reluctant to trust variables that have been initialized outside of its trust boundary, especially if they are initialized by users. They may have been initialized incorrectly. If an attacker can initialize the variable, then he/she can influence what the vulnerable system will do.",,unclassified,"The Art of Software Security Assessment: Chapter 7, ""Variable Initialization"", Page 312." CWE-458,EN-DEPRECATED: Incorrect Initialization (Type: Base),"This weakness has been deprecated because its name and description did not match. The description duplicated CWE-454, while the name suggested a more abstract initialization problem. Please refer to CWE-665 for the more abstract problem. -In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,unclassified, +In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,unclassified, CWE-459,EN-Incomplete Cleanup (Type: Base),"The software does not properly ""clean up"" and remove temporary or supporting resources after they have been used. -In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,unclassified, +In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,unclassified, CWE-46,EN-Path Equivalence: filename (Trailing Space) (Type: Variant),"A software system that accepts path input in the form of trailing space ('filedir ') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,unclassified, +In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,unclassified, CWE-463,EN-Deletion of Data Structure Sentinel (Type: Base),"The accidental deletion of a data-structure sentinel can cause serious programming logic problems. -Often times data-structure sentinels are used to mark structure of the data structure. A common example of this is the null character at the end of strings. Another common example is linked lists which may contain a sentinel to mark the end of the list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the deletion or modification outside of some wrapper interface which provides safety.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""NUL-Termination Problems"", Page 452." +Often times data-structure sentinels are used to mark structure of the data structure. A common example of this is the null character at the end of strings. Another common example is linked lists which may contain a sentinel to mark the end of the list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the deletion or modification outside of some wrapper interface which provides safety.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""NUL-Termination Problems"", Page 452." CWE-466,EN-Return of Pointer Value Outside of Expected Range (Type: Base),"A function can return a pointer to memory that is outside of the buffer that the pointer is expected to reference. -Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89" +Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89" CWE-47,EN-Path Equivalence: filename (Leading Space) (Type: Variant),"A software system that accepts path input in the form of leading space (' filedir') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,unclassified, +Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,unclassified, CWE-470,EN-Use of Externally-Controlled Input to Select Classes or Code (Unsafe Reflection) (Type: Base),"The application uses external input with reflection to select which classes or code to use, but it does not sufficiently prevent the input from selecting improper classes or code. -If the application uses external inputs to determine which class to instantiate or which method to invoke, then an attacker could supply values to select unexpected classes or methods. If this occurs, then the attacker could create control flow paths that were not intended by the developer. These paths could bypass authentication or access control checks, or otherwise cause the application to behave in an unexpected manner. This situation becomes a doomsday scenario if the attacker can upload files into a location that appears on the application's classpath (CWE-427) or add new entries to the application's classpath (CWE-426). Under either of these conditions, the attacker can use reflection to introduce new, malicious behavior into the application.",,unclassified, +If the application uses external inputs to determine which class to instantiate or which method to invoke, then an attacker could supply values to select unexpected classes or methods. If this occurs, then the attacker could create control flow paths that were not intended by the developer. These paths could bypass authentication or access control checks, or otherwise cause the application to behave in an unexpected manner. This situation becomes a doomsday scenario if the attacker can upload files into a location that appears on the application's classpath (CWE-427) or add new entries to the application's classpath (CWE-426). Under either of these conditions, the attacker can use reflection to introduce new, malicious behavior into the application.",,unclassified, CWE-471,EN-Modification of Assumed-Immutable Data (MAID) (Type: Base),"The software does not properly protect an assumed-immutable element from being modified by an attacker. -If the application uses external inputs to determine which class to instantiate or which method to invoke, then an attacker could supply values to select unexpected classes or methods. If this occurs, then the attacker could create control flow paths that were not intended by the developer. These paths could bypass authentication or access control checks, or otherwise cause the application to behave in an unexpected manner. This situation becomes a doomsday scenario if the attacker can upload files into a location that appears on the application's classpath (CWE-427) or add new entries to the application's classpath (CWE-426). Under either of these conditions, the attacker can use reflection to introduce new, malicious behavior into the application.",,unclassified, +If the application uses external inputs to determine which class to instantiate or which method to invoke, then an attacker could supply values to select unexpected classes or methods. If this occurs, then the attacker could create control flow paths that were not intended by the developer. These paths could bypass authentication or access control checks, or otherwise cause the application to behave in an unexpected manner. This situation becomes a doomsday scenario if the attacker can upload files into a location that appears on the application's classpath (CWE-427) or add new entries to the application's classpath (CWE-426). Under either of these conditions, the attacker can use reflection to introduce new, malicious behavior into the application.",,unclassified, CWE-472,EN-External Control of Assumed-Immutable Web Parameter (Type: Base),"The web application does not sufficiently verify inputs that are assumed to be immutable but are actually externally controllable, such as hidden form fields. If a web product does not properly protect assumed-immutable values from modification in hidden form fields, parameters, cookies, or URLs, this can lead to modification of critical data. Web applications often mistakenly make the assumption that data passed to the client in hidden fields or cookies is not susceptible to tampering. Improper validation of data that are user-controllable can lead to the application processing incorrect, and often malicious, input. For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 4: Use of Magic URLs, Predictable Cookies, and Hidden Form Fields."" Page 75 -The Art of Software Security Assessment: Chapter 17, ""Embedding State in HTML and URLs"", Page 1032." +The Art of Software Security Assessment: Chapter 17, ""Embedding State in HTML and URLs"", Page 1032." CWE-473,EN-PHP External Variable Modification (Type: Variant),"A PHP application does not properly protect against the modification of variables from external sources, such as query parameters or cookies. This can expose the application to numerous weaknesses that would not exist otherwise. If a web product does not properly protect assumed-immutable values from modification in hidden form fields, parameters, cookies, or URLs, this can lead to modification of critical data. Web applications often mistakenly make the assumption that data passed to the client in hidden fields or cookies is not susceptible to tampering. Improper validation of data that are user-controllable can lead to the application processing incorrect, and often malicious, input. -For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,unclassified, +For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,unclassified, CWE-474,EN-Use of Function with Inconsistent Implementations (Type: Base),"The code uses a function that has inconsistent implementations across operating systems and versions, which might cause security-relevant portability problems. If a web product does not properly protect assumed-immutable values from modification in hidden form fields, parameters, cookies, or URLs, this can lead to modification of critical data. Web applications often mistakenly make the assumption that data passed to the client in hidden fields or cookies is not susceptible to tampering. Improper validation of data that are user-controllable can lead to the application processing incorrect, and often malicious, input. -For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,unclassified, +For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,unclassified, CWE-475,EN-Undefined Behavior for Input to API (Type: Base),"The behavior of this function is undefined unless its control parameter is set to a specific value. If a web product does not properly protect assumed-immutable values from modification in hidden form fields, parameters, cookies, or URLs, this can lead to modification of critical data. Web applications often mistakenly make the assumption that data passed to the client in hidden fields or cookies is not susceptible to tampering. Improper validation of data that are user-controllable can lead to the application processing incorrect, and often malicious, input. -For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,unclassified, +For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,unclassified, CWE-477,EN-Use of Obsolete Functions (Type: Base),"The code uses deprecated or obsolete functions, which suggests that the code has not been actively reviewed or maintained. -NULL pointer dereference issues can occur through a number of flaws, including race conditions, and simple programming omissions.",,unclassified, +NULL pointer dereference issues can occur through a number of flaws, including race conditions, and simple programming omissions.",,unclassified, CWE-478,EN-Missing Default Case in Switch Statement (Type: Variant),"The code does not have a default case in a switch statement, which might lead to complex logical errors and resultant weaknesses. -NULL pointer dereference issues can occur through a number of flaws, including race conditions, and simple programming omissions.",,unclassified,"The Art of Software Security Assessment: Chapter 7, ""Switch Statements"", Page 337." +NULL pointer dereference issues can occur through a number of flaws, including race conditions, and simple programming omissions.",,unclassified,"The Art of Software Security Assessment: Chapter 7, ""Switch Statements"", Page 337." CWE-48,EN-Path Equivalence: file name (Internal Whitespace) (Type: Variant),"A software system that accepts path input in the form of internal space ('file(SPACE)name') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. Non-reentrant functions are functions that cannot safely be called, interrupted, and then recalled before the first call has finished without resulting in memory corruption. This can lead to an unexpected system state an unpredictable results with a variety of potential consequences depending on context, including denial of service and code execution. -Many functions are not reentrant, but some of them can result in the corruption of memory if they are used in a signal handler. The function call syslog() is an example of this. In order to perform its functionality, it allocates a small amount of memory as ""scratch space."" If syslog() is suspended by a signal call and the signal handler calls syslog(), the memory used by both of these functions enters an undefined, and possibly, exploitable state. Implementations of malloc() and free() manage metadata in global structures in order to track which memory is allocated versus which memory is available, but they are non-reentrant. Simultaneous calls to these functions can cause corruption of the metadata.",,unclassified, +Many functions are not reentrant, but some of them can result in the corruption of memory if they are used in a signal handler. The function call syslog() is an example of this. In order to perform its functionality, it allocates a small amount of memory as ""scratch space."" If syslog() is suspended by a signal call and the signal handler calls syslog(), the memory used by both of these functions enters an undefined, and possibly, exploitable state. Implementations of malloc() and free() manage metadata in global structures in order to track which memory is allocated versus which memory is available, but they are non-reentrant. Simultaneous calls to these functions can cause corruption of the metadata.",,unclassified, CWE-485,EN-Insufficient Encapsulation (Type: Class),"The product does not sufficiently encapsulate critical data or functionality. -Encapsulation is about drawing strong boundaries. In a web browser that might mean ensuring that your mobile code cannot be abused by other mobile code. On the server it might mean differentiation between validated data and unvalidated data, between one user's data and another's, or between data users are allowed to see and data that they are not.",,unclassified, +Encapsulation is about drawing strong boundaries. In a web browser that might mean ensuring that your mobile code cannot be abused by other mobile code. On the server it might mean differentiation between validated data and unvalidated data, between one user's data and another's, or between data users are allowed to see and data that they are not.",,unclassified, CWE-488,EN-Exposure of Data Element to Wrong Session (Type: Variant),"The product does not sufficiently enforce boundaries between the states of different sessions, causing data to be provided to, or used by, the wrong session. Data can ""bleed"" from one session to another through member variables of singleton objects, such as Servlets, and objects from a shared pool. -In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,unclassified, +In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,unclassified, CWE-489,EN-Leftover Debug Code (Type: Base),"The application can be deployed with active debugging code that can create unintended entry points. Data can ""bleed"" from one session to another through member variables of singleton objects, such as Servlets, and objects from a shared pool. -In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,unclassified, +In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,unclassified, CWE-49,EN-Path Equivalence: filename/ (Trailing Slash) (Type: Variant),"A software system that accepts path input in the form of trailing slash ('filedir/') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. Data can ""bleed"" from one session to another through member variables of singleton objects, such as Servlets, and objects from a shared pool. -In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,unclassified, +In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,unclassified, CWE-491,EN-Public cloneable() Method Without Final (Object Hijack) (Type: Variant),"A class has a cloneable() method that is not declared final, which allows an object to be created without calling the constructor. This can cause the object to be in an unexpected state. Data can ""bleed"" from one session to another through member variables of singleton objects, such as Servlets, and objects from a shared pool. -In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,unclassified,"OWASP , Attack Category : Mobile code: object hijack: http://www.owasp.org/index.php/Mobile_code:_object_hijack" +In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,unclassified,"OWASP , Attack Category : Mobile code: object hijack: http://www.owasp.org/index.php/Mobile_code:_object_hijack" CWE-495,EN-Private Array-Typed Field Returned From A Public Method (Type: Variant),"The product has a method that is declared public, but returns a reference to a private array, which could then be modified in unexpected ways. -An attacker can execute malicious code by compromising the host server, performing DNS spoofing, or modifying the code in transit.",,unclassified, +An attacker can execute malicious code by compromising the host server, performing DNS spoofing, or modifying the code in transit.",,unclassified, CWE-496,EN-Public Data Assigned to Private Array-Typed Field (Type: Variant),"Assigning public data to a private array is equivalent to giving public access to the array. -An attacker can execute malicious code by compromising the host server, performing DNS spoofing, or modifying the code in transit.",,unclassified, +An attacker can execute malicious code by compromising the host server, performing DNS spoofing, or modifying the code in transit.",,unclassified, CWE-497,EN-Exposure of System Data to an Unauthorized Control Sphere (Type: Variant),"Exposing system data or debugging information helps an adversary learn about the system and form an attack plan. -An information exposure occurs when system data or debugging information leaves the program through an output stream or logging function that makes it accessible to unauthorized parties. An attacker can also cause errors to occur by submitting unusual requests to the web application. The response to these errors can reveal detailed system information, deny service, cause security mechanisms to fail, and crash the server. An attacker can use error messages that reveal technologies, operating systems, and product versions to tune the attack against known vulnerabilities in these technologies. An application may use diagnostic methods that provide significant implementation details such as stack traces as part of its error handling mechanism.",,unclassified, +An information exposure occurs when system data or debugging information leaves the program through an output stream or logging function that makes it accessible to unauthorized parties. An attacker can also cause errors to occur by submitting unusual requests to the web application. The response to these errors can reveal detailed system information, deny service, cause security mechanisms to fail, and crash the server. An attacker can use error messages that reveal technologies, operating systems, and product versions to tune the attack against known vulnerabilities in these technologies. An application may use diagnostic methods that provide significant implementation details such as stack traces as part of its error handling mechanism.",,unclassified, CWE-5,EN-J2EE Misconfiguration: Data Transmission Without Encryption (Type: Variant),"Information sent over a network can be compromised while in transit. An attacker may be able to read/modify the contents if the data are sent in plaintext or are weakly encrypted. -Serializable classes are effectively open classes since data cannot be hidden in them. Classes that do not explicitly deny serialization can be serialized by any other class, which can then in turn use the data stored inside it.",,unclassified, +Serializable classes are effectively open classes since data cannot be hidden in them. Classes that do not explicitly deny serialization can be serialized by any other class, which can then in turn use the data stored inside it.",,unclassified, CWE-50,EN-Path Equivalence: //multiple/leading/slash (Type: Variant),"A software system that accepts path input in the form of multiple leading slash ('//multiple/leading/slash') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -Serializable classes are effectively open classes since data cannot be hidden in them. Classes that do not explicitly deny serialization can be serialized by any other class, which can then in turn use the data stored inside it.",,unclassified, +Serializable classes are effectively open classes since data cannot be hidden in them. Classes that do not explicitly deny serialization can be serialized by any other class, which can then in turn use the data stored inside it.",,unclassified, CWE-501,EN-Trust Boundary Violation (Type: Base),"The product mixes trusted and untrusted data in the same data structure or structured message. -By combining trusted and untrusted data in the same data structure, it becomes easier for programmers to mistakenly trust unvalidated data.",,unclassified, +By combining trusted and untrusted data in the same data structure, it becomes easier for programmers to mistakenly trust unvalidated data.",,unclassified, CWE-506,EN-Embedded Malicious Code (Type: Class),"The application contains code that appears to be malicious in nature. -Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified, +Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified, CWE-507,EN-Trojan Horse (Type: Base),"The software appears to contain benign or useful functionality, but it also contains code that is hidden from normal operation that violates the intended security policy of the user or the system administrator. -Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified,"Writing Secure Code: Chapter 7, ""Viruses, Trojans, and Worms In a Nutshell"" Page 208" +Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified,"Writing Secure Code: Chapter 7, ""Viruses, Trojans, and Worms In a Nutshell"" Page 208" CWE-508,EN-Non-Replicating Malicious Code (Type: Base),"Non-replicating malicious code only resides on the target system or software that is attacked; it does not attempt to spread to other systems. -Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified, +Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified, CWE-509,EN-Replicating Malicious Code (Virus or Worm) (Type: Base),"Replicating malicious code, including viruses and worms, will attempt to attack other systems once it has successfully compromised the target system or software. -Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified, +Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified, CWE-51,EN-Path Equivalence: /multiple//internal/slash (Type: Variant),"A software system that accepts path input in the form of multiple internal slash ('/multiple//internal/slash/') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified, +Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified, CWE-510,EN-Trapdoor (Type: Base),"A trapdoor is a hidden piece of code that responds to a special input, allowing its user access to resources without passing through the normal security enforcement mechanism. -Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified, +Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified, CWE-511,EN-Logic/Time Bomb (Type: Base),"The software contains code that is designed to disrupt the legitimate operation of the software (or its environment) when a certain time passes, or when a certain logical condition is met. -When the time bomb or logic bomb is detonated, it may perform a denial of service such as crashing the system, deleting critical data, or degrading system response time. This bomb might be placed within either a replicating or non-replicating Trojan horse.",,unclassified,Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/ +When the time bomb or logic bomb is detonated, it may perform a denial of service such as crashing the system, deleting critical data, or degrading system response time. This bomb might be placed within either a replicating or non-replicating Trojan horse.",,unclassified,Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/ CWE-512,EN-Spyware (Type: Base),"The software collects personally identifiable information about a human user or the user's activities, but the software accesses this information using other resources besides itself, and it does not require that user's explicit approval or direct input into the software. -""Spyware"" is a commonly used term with many definitions and interpretations. In general, it is meant to software that collects information or installs functionality that human users might not allow if they were fully aware of the actions being taken by the software. For example, a user might expect that tax software would collect a social security number and include it when filing a tax return, but that same user would not expect gaming software to obtain the social security number from that tax software's data.",,unclassified, +""Spyware"" is a commonly used term with many definitions and interpretations. In general, it is meant to software that collects information or installs functionality that human users might not allow if they were fully aware of the actions being taken by the software. For example, a user might expect that tax software would collect a social security number and include it when filing a tax return, but that same user would not expect gaming software to obtain the social security number from that tax software's data.",,unclassified, CWE-514,EN-Covert Channel (Type: Class),"A covert channel is a path that can be used to transfer information in a way not intended by the system's designers. -Typically the system has not given authorization for the transmission and has no knowledge of its occurrence.",,unclassified, +Typically the system has not given authorization for the transmission and has no knowledge of its occurrence.",,unclassified, CWE-516,EN-DEPRECATED (Duplicate): Covert Timing Channel (Type: Base),"This weakness can be found at CWE-385. -Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,unclassified, +Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,unclassified, CWE-52,EN-Path Equivalence: /multiple/trailing/slash// (Type: Variant),"A software system that accepts path input in the form of multiple trailing slash ('/multiple/trailing/slash//') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,unclassified, +Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,unclassified, CWE-520,EN-.NET Misconfiguration: Use of Impersonation (Type: Variant),"Allowing a .NET application to run at potentially escalated levels of access to the underlying operating and file systems can be dangerous and result in various forms of attacks. -Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,unclassified, +Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,unclassified, CWE-521,EN-Weak Password Requirements (Type: Base),"The product does not require that users should have strong passwords, which makes it easier for attackers to compromise user accounts. -An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" +An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" CWE-522,EN-Insufficiently Protected Credentials (Type: Base),"This weakness occurs when the application transmits or stores authentication credentials and uses an insecure method that is susceptible to unauthorized interception and/or retrieval. -An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" +An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" CWE-523,EN-Unprotected Transport of Credentials (Type: Variant),"Login pages not using adequate measures to protect the user name and password while they are in transit from the client to the server. -An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified, +An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified, CWE-524,EN-Information Exposure Through Caching (Type: Variant),"The application uses a cache to maintain a pool of objects, threads, connections, pages, or passwords to minimize the time it takes to access them or the resources to which they connect. If implemented improperly, these caches can allow access to unauthorized information or cause a denial of service vulnerability. -An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified, +An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified, CWE-525,EN-Information Exposure Through Browser Caching (Type: Variant),"For each web page, the application should have an appropriate caching policy specifying the extent to which the page and its form fields should be cached. -An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified, +An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified, CWE-526,EN-Information Exposure Through Environmental Variables (Type: Variant),"Environmental variables may contain sensitive information about a remote server. -An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified, +An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified, CWE-527,EN-Exposure of CVS Repository to an Unauthorized Control Sphere (Type: Variant),"The product stores a CVS repository in a directory or other container that is accessible to actors outside of the intended control sphere. -Information contained within a CVS subdirectory on a web server or other server could be recovered by an attacker and used for malicious purposes. This information may include usernames, filenames, path root, and IP addresses.",,unclassified, +Information contained within a CVS subdirectory on a web server or other server could be recovered by an attacker and used for malicious purposes. This information may include usernames, filenames, path root, and IP addresses.",,unclassified, CWE-528,EN-Exposure of Core Dump File to an Unauthorized Control Sphere (Type: Variant),"The product generates a core dump file in a directory that is accessible to actors outside of the intended control sphere. -Information contained within a CVS subdirectory on a web server or other server could be recovered by an attacker and used for malicious purposes. This information may include usernames, filenames, path root, and IP addresses.",,unclassified, +Information contained within a CVS subdirectory on a web server or other server could be recovered by an attacker and used for malicious purposes. This information may include usernames, filenames, path root, and IP addresses.",,unclassified, CWE-529,EN-Exposure of Access Control List Files to an Unauthorized Control Sphere (Type: Variant),"The product stores access control list files in a directory or other container that is accessible to actors outside of the intended control sphere. -Exposure of these access control list files may give the attacker information about the configuration of the site or system. This information may then be used to bypass the intended security policy or identify trusted systems from which an attack can be launched.",,unclassified, +Exposure of these access control list files may give the attacker information about the configuration of the site or system. This information may then be used to bypass the intended security policy or identify trusted systems from which an attack can be launched.",,unclassified, CWE-53,EN-Path Equivalence: \multiple\\internal\backslash (Type: Variant),"A software system that accepts path input in the form of multiple internal backslash ('\multiple\trailing\\slash') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -Exposure of these access control list files may give the attacker information about the configuration of the site or system. This information may then be used to bypass the intended security policy or identify trusted systems from which an attack can be launched.",,unclassified, +Exposure of these access control list files may give the attacker information about the configuration of the site or system. This information may then be used to bypass the intended security policy or identify trusted systems from which an attack can be launched.",,unclassified, CWE-530,EN-Exposure of Backup File to an Unauthorized Control Sphere (Type: Variant),"A backup file is stored in a directory that is accessible to actors outside of the intended control sphere. -Often, old files are renamed with an extension such as .~bk to distinguish them from production files. The source code for old files that have been renamed in this manner and left in the webroot can often be retrieved. This renaming may have been performed automatically by the web server, or manually by the administrator.",,unclassified, +Often, old files are renamed with an extension such as .~bk to distinguish them from production files. The source code for old files that have been renamed in this manner and left in the webroot can often be retrieved. This renaming may have been performed automatically by the web server, or manually by the administrator.",,unclassified, CWE-531,EN-Information Exposure Through Test Code (Type: Variant),"Accessible test applications can pose a variety of security risks. Since developers or administrators rarely consider that someone besides themselves would even know about the existence of these applications, it is common for them to contain sensitive information or functions. -Often, old files are renamed with an extension such as .~bk to distinguish them from production files. The source code for old files that have been renamed in this manner and left in the webroot can often be retrieved. This renaming may have been performed automatically by the web server, or manually by the administrator.",,unclassified, +Often, old files are renamed with an extension such as .~bk to distinguish them from production files. The source code for old files that have been renamed in this manner and left in the webroot can often be retrieved. This renaming may have been performed automatically by the web server, or manually by the administrator.",,unclassified, CWE-533,EN-Information Exposure Through Server Log Files (Type: Variant),"A server.log file was found. This can give information on whatever application left the file. Usually this can give full path names and system information, and sometimes usernames and passwords. -While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified, +While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified, CWE-534,EN-Information Exposure Through Debug Log Files (Type: Variant),"The application does not sufficiently restrict access to a log file that is used for debugging. -While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified, +While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified, CWE-535,EN-Information Exposure Through Shell Error Message (Type: Variant),"A command shell error message indicates that there exists an unhandled exception in the web application code. In many cases, an attacker can leverage the conditions that cause these errors in order to gain unauthorized access to the system. -While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified, +While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified, CWE-536,EN-Information Exposure Through Servlet Runtime Error Message (Type: Variant),"A servlet error message indicates that there exists an unhandled exception in your web application code and may provide useful information to an attacker. -While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified, +While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified, CWE-537,EN-Information Exposure Through Java Runtime Error Message (Type: Variant),"In many cases, an attacker can leverage the conditions that cause unhandled exception errors in order to gain unauthorized access to the system. -While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified, +While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified, CWE-538,EN-File and Directory Information Exposure (Type: Base),"The product stores sensitive information in files or directories that are accessible to actors outside of the intended control sphere. -While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 12: Information Leakage."" Page 191" +While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 12: Information Leakage."" Page 191" CWE-539,EN-Information Exposure Through Persistent Cookies (Type: Variant),"Persistent cookies are cookies that are stored on the browser's hard drive. This can cause security and privacy issues depending on the information stored in the cookie and how it is accessed. -Cookies are small bits of data that are sent by the web application but stored locally in the browser. This lets the application use the cookie to pass information between pages and store variable information. The web application controls what information is stored in a cookie and how it is used. Typical types of information stored in cookies are session Identifiers, personalization and customization information, and in rare cases even usernames to enable automated logins. There are two different types of cookies: session cookies and persistent cookies. Session cookies just live in the browser's memory, and are not stored anywhere, but persistent cookies are stored on the browser's hard drive.",,unclassified, +Cookies are small bits of data that are sent by the web application but stored locally in the browser. This lets the application use the cookie to pass information between pages and store variable information. The web application controls what information is stored in a cookie and how it is used. Typical types of information stored in cookies are session Identifiers, personalization and customization information, and in rare cases even usernames to enable automated logins. There are two different types of cookies: session cookies and persistent cookies. Session cookies just live in the browser's memory, and are not stored anywhere, but persistent cookies are stored on the browser's hard drive.",,unclassified, CWE-54,EN-Path Equivalence: filedir\ (Trailing Backslash) (Type: Variant),"A software system that accepts path input in the form of trailing backslash ('filedir\') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -Cookies are small bits of data that are sent by the web application but stored locally in the browser. This lets the application use the cookie to pass information between pages and store variable information. The web application controls what information is stored in a cookie and how it is used. Typical types of information stored in cookies are session Identifiers, personalization and customization information, and in rare cases even usernames to enable automated logins. There are two different types of cookies: session cookies and persistent cookies. Session cookies just live in the browser's memory, and are not stored anywhere, but persistent cookies are stored on the browser's hard drive.",,unclassified, +Cookies are small bits of data that are sent by the web application but stored locally in the browser. This lets the application use the cookie to pass information between pages and store variable information. The web application controls what information is stored in a cookie and how it is used. Typical types of information stored in cookies are session Identifiers, personalization and customization information, and in rare cases even usernames to enable automated logins. There are two different types of cookies: session cookies and persistent cookies. Session cookies just live in the browser's memory, and are not stored anywhere, but persistent cookies are stored on the browser's hard drive.",,unclassified, CWE-540,EN-Information Exposure Through Source Code (Type: Variant),"Source code on a web server often contains sensitive information and should generally not be accessible to users. -There are situations where it is critical to remove source code from an area or server. For example, obtaining Perl source code on a system allows an attacker to understand the logic of the script and extract extremely useful information such as code bugs or logins and passwords.",,unclassified, +There are situations where it is critical to remove source code from an area or server. For example, obtaining Perl source code on a system allows an attacker to understand the logic of the script and extract extremely useful information such as code bugs or logins and passwords.",,unclassified, CWE-541,EN-Information Exposure Through Include Source Code (Type: Variant),"If an include file source is accessible, the file can contain usernames and passwords, as well as sensitive information pertaining to the application and system. -There are situations where it is critical to remove source code from an area or server. For example, obtaining Perl source code on a system allows an attacker to understand the logic of the script and extract extremely useful information such as code bugs or logins and passwords.",,unclassified, +There are situations where it is critical to remove source code from an area or server. For example, obtaining Perl source code on a system allows an attacker to understand the logic of the script and extract extremely useful information such as code bugs or logins and passwords.",,unclassified, CWE-542,EN-Information Exposure Through Cleanup Log Files (Type: Variant),"The application does not properly protect or delete a log file related to cleanup. -There are situations where it is critical to remove source code from an area or server. For example, obtaining Perl source code on a system allows an attacker to understand the logic of the script and extract extremely useful information such as code bugs or logins and passwords.",,unclassified, +There are situations where it is critical to remove source code from an area or server. For example, obtaining Perl source code on a system allows an attacker to understand the logic of the script and extract extremely useful information such as code bugs or logins and passwords.",,unclassified, CWE-543,EN-Use of Singleton Pattern Without Synchronization in a Multithreaded Context (Type: Variant),"The software uses the singleton pattern when creating a resource within a multithreaded environment. -The use of a singleton pattern may not be thread-safe.",,unclassified,Thread-Specifc Storage for C/C++: http://www.cs.wustl.edu/~schmidt/PDF/TSS-pattern.pdf +The use of a singleton pattern may not be thread-safe.",,unclassified,Thread-Specifc Storage for C/C++: http://www.cs.wustl.edu/~schmidt/PDF/TSS-pattern.pdf CWE-544,EN-Missing Standardized Error Handling Mechanism (Type: Base),"The software does not use a standardized method for handling errors throughout the code, which might introduce inconsistent error handling and resultant weaknesses. -If the application handles error messages individually, on a one-by-one basis, this is likely to result in inconsistent error handling. The causes of errors may be lost. Also, detailed information about the causes of an error may be unintentionally returned to the user.",,unclassified, +If the application handles error messages individually, on a one-by-one basis, this is likely to result in inconsistent error handling. The causes of errors may be lost. Also, detailed information about the causes of an error may be unintentionally returned to the user.",,unclassified, CWE-545,EN-Use of Dynamic Class Loading (Type: Variant),"Dynamically loaded code has the potential to be malicious. -If the application handles error messages individually, on a one-by-one basis, this is likely to result in inconsistent error handling. The causes of errors may be lost. Also, detailed information about the causes of an error may be unintentionally returned to the user.",,unclassified, +If the application handles error messages individually, on a one-by-one basis, this is likely to result in inconsistent error handling. The causes of errors may be lost. Also, detailed information about the causes of an error may be unintentionally returned to the user.",,unclassified, CWE-546,EN-Suspicious Comment (Type: Variant),"The code contains comments that suggest the presence of bugs, incomplete functionality, or weaknesses. -Many suspicious comments, such as BUG, HACK, FIXME, LATER, LATER2, TODO, in the code indicate missing security functionality and checking. Others indicate code problems that programmers should fix, such as hard-coded variables, error handling, not using stored procedures, and performance issues.",,unclassified, +Many suspicious comments, such as BUG, HACK, FIXME, LATER, LATER2, TODO, in the code indicate missing security functionality and checking. Others indicate code problems that programmers should fix, such as hard-coded variables, error handling, not using stored procedures, and performance issues.",,unclassified, CWE-547,"EN-Use of Hard-coded, Security-relevant Constants (Type: Variant)","The program uses hard-coded constants instead of symbolic names for security-critical values, which increases the likelihood of mistakes during code maintenance or security policy change. -If the developer does not find all occurrences of the hard-coded constants, an incorrect policy decision may be made if one of the constants is not changed. Making changes to these values will require code changes that may be difficult or impossible once the system is released to the field. In addition, these hard-coded values may become available to attackers if the code is ever disclosed.",,unclassified, +If the developer does not find all occurrences of the hard-coded constants, an incorrect policy decision may be made if one of the constants is not changed. Making changes to these values will require code changes that may be difficult or impossible once the system is released to the field. In addition, these hard-coded values may become available to attackers if the code is ever disclosed.",,unclassified, CWE-548,EN-Information Exposure Through Directory Listing (Type: Variant),"A directory listing is inappropriately exposed, yielding potentially sensitive information to attackers. -A directory listing provides an attacker with the complete index of all the resources located inside of the directory. The specific risks and consequences vary depending on which files are listed and accessible.",,unclassified, +A directory listing provides an attacker with the complete index of all the resources located inside of the directory. The specific risks and consequences vary depending on which files are listed and accessible.",,unclassified, CWE-549,EN-Missing Password Field Masking (Type: Variant),"The software does not mask passwords during entry, increasing the potential for attackers to observe and capture passwords. -A directory listing provides an attacker with the complete index of all the resources located inside of the directory. The specific risks and consequences vary depending on which files are listed and accessible.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" +A directory listing provides an attacker with the complete index of all the resources located inside of the directory. The specific risks and consequences vary depending on which files are listed and accessible.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" CWE-55,EN-Path Equivalence: /./ (Single Dot Directory) (Type: Variant),"A software system that accepts path input in the form of single dot directory exploit ('/./') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -A directory listing provides an attacker with the complete index of all the resources located inside of the directory. The specific risks and consequences vary depending on which files are listed and accessible.",,unclassified, +A directory listing provides an attacker with the complete index of all the resources located inside of the directory. The specific risks and consequences vary depending on which files are listed and accessible.",,unclassified, CWE-550,EN-Information Exposure Through Server Error Message (Type: Variant),"Certain conditions, such as network failure, will cause a server error message to be displayed. -While error messages in and of themselves are not dangerous, per se, it is what an attacker can glean from them that might cause eventual problems.",,unclassified, +While error messages in and of themselves are not dangerous, per se, it is what an attacker can glean from them that might cause eventual problems.",,unclassified, CWE-551,EN-Incorrect Behavior Order: Authorization Before Parsing and Canonicalization (Type: Base),"If a web server does not fully parse requested URLs before it examines them for authorization, it may be possible for an attacker to bypass authorization protection. -For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,unclassified, +For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,unclassified, CWE-552,EN-Files or Directories Accessible to External Parties (Type: Base),"Files or directories are accessible in the environment that should not be. -For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,unclassified, +For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,unclassified, CWE-553,EN-Command Shell in Externally Accessible Directory (Type: Variant),"A possible shell file exists in /cgi-bin/ or other accessible directories. This is extremely dangerous and can be used by an attacker to execute commands on the web server. -For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,unclassified, +For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,unclassified, CWE-554,EN-ASP.NET Misconfiguration: Not Using Input Validation Framework (Type: Variant),"The ASP.NET application does not use an input validation framework. -For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,unclassified, +For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,unclassified, CWE-555,EN-J2EE Misconfiguration: Plaintext Password in Configuration File (Type: Variant),"The J2EE application stores a plaintext password in a configuration file. -Storing a plaintext password in a configuration file allows anyone who can read the file to access the password-protected resource, making it an easy target for attackers.",,unclassified, +Storing a plaintext password in a configuration file allows anyone who can read the file to access the password-protected resource, making it an easy target for attackers.",,unclassified, CWE-556,EN-ASP.NET Misconfiguration: Use of Identity Impersonation (Type: Variant),"Configuring an ASP.NET application to run with impersonated credentials may give the application unnecessary privileges. -The use of impersonated credentials allows an ASP.NET application to run with either the privileges of the client on whose behalf it is executing or with arbitrary privileges granted in its configuration.",,unclassified, +The use of impersonated credentials allows an ASP.NET application to run with either the privileges of the client on whose behalf it is executing or with arbitrary privileges granted in its configuration.",,unclassified, CWE-558,EN-Use of getlogin() in Multithreaded Application (Type: Variant),"The application uses the getlogin() function in a multithreaded context, potentially causing it to return incorrect values. -The getlogin() function returns a pointer to a string that contains the name of the user associated with the calling process. The function is not reentrant, meaning that if it is called from another process, the contents are not locked out and the value of the string can be changed by another process. This makes it very risky to use because the username can be changed by other processes, so the results of the function cannot be trusted.",,unclassified, +The getlogin() function returns a pointer to a string that contains the name of the user associated with the calling process. The function is not reentrant, meaning that if it is called from another process, the contents are not locked out and the value of the string can be changed by another process. This makes it very risky to use because the username can be changed by other processes, so the results of the function cannot be trusted.",,unclassified, CWE-56,EN-Path Equivalence: filedir* (Wildcard) (Type: Variant),"A software system that accepts path input in the form of asterisk wildcard ('filedir*') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files. -The getlogin() function returns a pointer to a string that contains the name of the user associated with the calling process. The function is not reentrant, meaning that if it is called from another process, the contents are not locked out and the value of the string can be changed by another process. This makes it very risky to use because the username can be changed by other processes, so the results of the function cannot be trusted.",,unclassified, +The getlogin() function returns a pointer to a string that contains the name of the user associated with the calling process. The function is not reentrant, meaning that if it is called from another process, the contents are not locked out and the value of the string can be changed by another process. This makes it very risky to use because the username can be changed by other processes, so the results of the function cannot be trusted.",,unclassified, CWE-560,EN-Use of umask() with chmod-style Argument (Type: Variant),"The product calls umask() with an incorrect argument that is specified as if it is an argument to chmod(). -The getlogin() function returns a pointer to a string that contains the name of the user associated with the calling process. The function is not reentrant, meaning that if it is called from another process, the contents are not locked out and the value of the string can be changed by another process. This makes it very risky to use because the username can be changed by other processes, so the results of the function cannot be trusted.",,unclassified, +The getlogin() function returns a pointer to a string that contains the name of the user associated with the calling process. The function is not reentrant, meaning that if it is called from another process, the contents are not locked out and the value of the string can be changed by another process. This makes it very risky to use because the username can be changed by other processes, so the results of the function cannot be trusted.",,unclassified, CWE-561,EN-Dead Code (Type: Variant),"The software contains dead code, which can never be executed. -Dead code is source code that can never be executed in a running program. The surrounding code makes it impossible for a section of code to ever be executed.",,unclassified, +Dead code is source code that can never be executed in a running program. The surrounding code makes it impossible for a section of code to ever be executed.",,unclassified, CWE-562,EN-Return of Stack Variable Address (Type: Base),"A function returns the address of a stack variable, which will cause unintended program behavior, typically in the form of a crash. -Dead code is source code that can never be executed in a running program. The surrounding code makes it impossible for a section of code to ever be executed.",,unclassified, +Dead code is source code that can never be executed in a running program. The surrounding code makes it impossible for a section of code to ever be executed.",,unclassified, CWE-563,EN-Unused Variable (Type: Variant),"The variable's value is assigned but never used, making it a dead store. -It is likely that the variable is simply vestigial, but it is also possible that the unused variable points out a bug.",,unclassified, +It is likely that the variable is simply vestigial, but it is also possible that the unused variable points out a bug.",,unclassified, CWE-564,EN-SQL Injection: Hibernate (Type: Variant),"Using Hibernate to execute a dynamic SQL statement built with user-controlled input can allow an attacker to modify the statement's meaning or to execute arbitrary SQL commands. -It is likely that the variable is simply vestigial, but it is also possible that the unused variable points out a bug.",,unclassified, +It is likely that the variable is simply vestigial, but it is also possible that the unused variable points out a bug.",,unclassified, CWE-565,EN-Reliance on Cookies without Validation and Integrity Checking (Type: Base),"The application relies on the existence or values of cookies when performing security-critical operations, but it does not properly ensure that the setting is valid for the associated user. -Attackers can easily modify cookies, within the browser or by implementing the client-side code outside of the browser. Reliance on cookies without detailed validation and integrity checking can allow attackers to bypass authentication, conduct injection attacks such as SQL injection and cross-site scripting, or otherwise modify inputs in unexpected ways.",,unclassified, +Attackers can easily modify cookies, within the browser or by implementing the client-side code outside of the browser. Reliance on cookies without detailed validation and integrity checking can allow attackers to bypass authentication, conduct injection attacks such as SQL injection and cross-site scripting, or otherwise modify inputs in unexpected ways.",,unclassified, CWE-566,EN-Authorization Bypass Through User-Controlled SQL Primary Key (Type: Variant),"The software uses a database table that includes records that should not be accessible to an actor, but it executes a SQL statement with a primary key that can be controlled by that actor. When a user can set a primary key to any value, then the user can modify the key to point to unauthorized records. Database access control errors occur when: Data enters a program from an untrusted source. The data is used to specify the value of a primary key in a SQL query. -The untrusted source does not have the permissions to be able to access all rows in the associated table.",,unclassified, +The untrusted source does not have the permissions to be able to access all rows in the associated table.",,unclassified, CWE-567,EN-Unsynchronized Access to Shared Data in a Multithreaded Context (Type: Base),"The product does not properly synchronize shared data, such as static variables across threads, which can lead to undefined behavior and unpredictable data changes. Within servlets, shared static variables are not protected from concurrent access, but servlets are multithreaded. This is a typical programming mistake in J2EE applications, since the multithreading is handled by the framework. When a shared variable can be influenced by an attacker, one thread could wind up modifying the variable to contain data that is not valid for a different thread that is also using the data within the variable. -Note that this weakness is not unique to servlets.",,unclassified, +Note that this weakness is not unique to servlets.",,unclassified, CWE-568,EN-finalize() Method Without super.finalize() (Type: Variant),"The software contains a finalize() method that does not call super.finalize(). -The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,unclassified, +The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,unclassified, CWE-57,EN-Path Equivalence: fakedir/../realdir/filename (Type: Variant),"The software contains protection mechanisms to restrict access to 'realdir/filename', but it constructs pathnames using external input in the form of 'fakedir/../realdir/filename' that are not handled by those mechanisms. This allows attackers to perform unauthorized actions against the targeted file. -The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,unclassified, +The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,unclassified, CWE-570,EN-Expression is Always False (Type: Variant),"The software contains an expression that will always evaluate to false. -The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,unclassified, +The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,unclassified, CWE-571,EN-Expression is Always True (Type: Variant),"The software contains an expression that will always evaluate to true. -The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,unclassified, +The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,unclassified, CWE-572,EN-Call to Thread run() instead of start() (Type: Variant),"The program calls a thread's run() method instead of calling start(), which causes the code to run in the thread of the caller instead of the callee. -In most cases a direct call to a Thread object's run() method is a bug. The programmer intended to begin a new thread of control, but accidentally called run() instead of start(), so the run() method will execute in the caller's thread of control.",,unclassified, +In most cases a direct call to a Thread object's run() method is a bug. The programmer intended to begin a new thread of control, but accidentally called run() instead of start(), so the run() method will execute in the caller's thread of control.",,unclassified, CWE-573,EN-Improper Following of Specification by Caller (Type: Class),"The software does not follow or incorrectly follows the specifications as required by the implementation language, environment, framework, protocol, or platform. -When leveraging external functionality, such as an API, it is important that the caller does so in accordance with the requirements of the external functionality or else unintended behaviors may result, possibly leaving the system vulnerable to any number of exploits.",,unclassified, +When leveraging external functionality, such as an API, it is important that the caller does so in accordance with the requirements of the external functionality or else unintended behaviors may result, possibly leaving the system vulnerable to any number of exploits.",,unclassified, CWE-574,EN-EJB Bad Practices: Use of Synchronization Primitives (Type: Variant),"The program violates the Enterprise JavaBeans (EJB) specification by using thread synchronization primitives. -The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not use thread synchronization primitives to synchronize execution of multiple instances."" The specification justifies this requirement in the following way: ""This rule is required to ensure consistent runtime semantics because while some EJB containers may use a single JVM to execute all enterprise bean's instances, others may distribute the instances across multiple JVMs.""",,unclassified, +The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not use thread synchronization primitives to synchronize execution of multiple instances."" The specification justifies this requirement in the following way: ""This rule is required to ensure consistent runtime semantics because while some EJB containers may use a single JVM to execute all enterprise bean's instances, others may distribute the instances across multiple JVMs.""",,unclassified, CWE-575,EN-EJB Bad Practices: Use of AWT Swing (Type: Variant),"The program violates the Enterprise JavaBeans (EJB) specification by using AWT/Swing. -The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not use the AWT functionality to attempt to output information to a display, or to input information from a keyboard."" The specification justifies this requirement in the following way: ""Most servers do not allow direct interaction between an application program and a keyboard/display attached to the server system.""",,unclassified, +The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not use the AWT functionality to attempt to output information to a display, or to input information from a keyboard."" The specification justifies this requirement in the following way: ""Most servers do not allow direct interaction between an application program and a keyboard/display attached to the server system.""",,unclassified, CWE-576,EN-EJB Bad Practices: Use of Java I/O (Type: Variant),"The program violates the Enterprise JavaBeans (EJB) specification by using the java.io package. -The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not use the java.io package to attempt to access files and directories in the file system."" The specification justifies this requirement in the following way: ""The file system APIs are not well-suited for business components to access data. Business components should use a resource manager API, such as JDBC, to store data.""",,unclassified, +The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not use the java.io package to attempt to access files and directories in the file system."" The specification justifies this requirement in the following way: ""The file system APIs are not well-suited for business components to access data. Business components should use a resource manager API, such as JDBC, to store data.""",,unclassified, CWE-577,EN-EJB Bad Practices: Use of Sockets (Type: Variant),"The program violates the Enterprise JavaBeans (EJB) specification by using sockets. -The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not attempt to listen on a socket, accept connections on a socket, or use a socket for multicast."" The specification justifies this requirement in the following way: ""The EJB architecture allows an enterprise bean instance to be a network socket client, but it does not allow it to be a network server. Allowing the instance to become a network server would conflict with the basic function of the enterprise bean-- to serve the EJB clients.""",,unclassified, +The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not attempt to listen on a socket, accept connections on a socket, or use a socket for multicast."" The specification justifies this requirement in the following way: ""The EJB architecture allows an enterprise bean instance to be a network socket client, but it does not allow it to be a network server. Allowing the instance to become a network server would conflict with the basic function of the enterprise bean-- to serve the EJB clients.""",,unclassified, CWE-578,EN-EJB Bad Practices: Use of Class Loader (Type: Variant),"The program violates the Enterprise JavaBeans (EJB) specification by using the class loader. -The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""The enterprise bean must not attempt to create a class loader; obtain the current class loader; set the context class loader; set security manager; create a new security manager; stop the JVM; or change the input, output, and error streams."" The specification justifies this requirement in the following way: ""These functions are reserved for the EJB container. Allowing the enterprise bean to use these functions could compromise security and decrease the container's ability to properly manage the runtime environment.""",,unclassified, +The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""The enterprise bean must not attempt to create a class loader; obtain the current class loader; set the context class loader; set security manager; create a new security manager; stop the JVM; or change the input, output, and error streams."" The specification justifies this requirement in the following way: ""These functions are reserved for the EJB container. Allowing the enterprise bean to use these functions could compromise security and decrease the container's ability to properly manage the runtime environment.""",,unclassified, CWE-579,EN-J2EE Bad Practices: Non-serializable Object Stored in Session (Type: Variant),"The application stores a non-serializable object as an HttpSession attribute, which can hurt reliability. -The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""The enterprise bean must not attempt to create a class loader; obtain the current class loader; set the context class loader; set security manager; create a new security manager; stop the JVM; or change the input, output, and error streams."" The specification justifies this requirement in the following way: ""These functions are reserved for the EJB container. Allowing the enterprise bean to use these functions could compromise security and decrease the container's ability to properly manage the runtime environment.""",,unclassified, +The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""The enterprise bean must not attempt to create a class loader; obtain the current class loader; set the context class loader; set security manager; create a new security manager; stop the JVM; or change the input, output, and error streams."" The specification justifies this requirement in the following way: ""These functions are reserved for the EJB container. Allowing the enterprise bean to use these functions could compromise security and decrease the container's ability to properly manage the runtime environment.""",,unclassified, CWE-58,EN-Path Equivalence: Windows 8.3 Filename (Type: Variant),"The software contains a protection mechanism that restricts access to a long filename on a Windows operating system, but the software does not properly restrict access to the equivalent short ""8.3"" filename. On later Windows operating systems, a file can have a ""long name"" and a short name that is compatible with older Windows file systems, with up to 8 characters in the filename and 3 characters for the extension. These ""8.3"" filenames, therefore, act as an alternate name for files with long names, so they are useful pathname equivalence manipulations.",,unclassified,"Writing Secure Code -The Art of Software Security Assessment: Chapter 11, ""DOS 8.3 Filenames"", Page 673." +The Art of Software Security Assessment: Chapter 11, ""DOS 8.3 Filenames"", Page 673." CWE-580,EN-clone() Method Without super.clone() (Type: Variant),"The software contains a clone() method that does not call super.clone() to obtain the new object. -All implementations of clone() should obtain the new object by calling super.clone(). If a class does not follow this convention, a subclass's clone() method will return an object of the wrong type.",,unclassified, +All implementations of clone() should obtain the new object by calling super.clone(). If a class does not follow this convention, a subclass's clone() method will return an object of the wrong type.",,unclassified, CWE-581,EN-Object Model Violation: Just One of Equals and Hashcode Defined (Type: Base),"The software does not maintain equal hashcodes for equal objects. -Java objects are expected to obey a number of invariants related to equality. One of these invariants is that equal objects must have equal hashcodes. In other words, if a.equals(b) == true then a.hashCode() == b.hashCode().",,unclassified, +Java objects are expected to obey a number of invariants related to equality. One of these invariants is that equal objects must have equal hashcodes. In other words, if a.equals(b) == true then a.hashCode() == b.hashCode().",,unclassified, CWE-582,"EN-Array Declared Public, Final, and Static (Type: Variant)","The program declares an array public, final, and static, which is not sufficient to prevent the array's contents from being modified. -Because arrays are mutable objects, the final constraint requires that the array object itself be assigned only once, but makes no guarantees about the values of the array elements. Since the array is public, a malicious program can change the values stored in the array. As such, in most cases an array declared public, final and static is a bug.",,unclassified, +Because arrays are mutable objects, the final constraint requires that the array object itself be assigned only once, but makes no guarantees about the values of the array elements. Since the array is public, a malicious program can change the values stored in the array. As such, in most cases an array declared public, final and static is a bug.",,unclassified, CWE-583,EN-finalize() Method Declared Public (Type: Variant),"The program violates secure coding principles for mobile code by declaring a finalize() method public. -A program should never call finalize explicitly, except to call super.finalize() inside an implementation of finalize(). In mobile code situations, the otherwise error prone practice of manual garbage collection can become a security threat if an attacker can maliciously invoke one of your finalize() methods because it is declared with public access.",,unclassified, +A program should never call finalize explicitly, except to call super.finalize() inside an implementation of finalize(). In mobile code situations, the otherwise error prone practice of manual garbage collection can become a security threat if an attacker can maliciously invoke one of your finalize() methods because it is declared with public access.",,unclassified, CWE-584,EN-Return Inside Finally Block (Type: Base),"The code has a return statement inside a finally block, which will cause any thrown exception in the try block to be discarded. -A program should never call finalize explicitly, except to call super.finalize() inside an implementation of finalize(). In mobile code situations, the otherwise error prone practice of manual garbage collection can become a security threat if an attacker can maliciously invoke one of your finalize() methods because it is declared with public access.",,unclassified, +A program should never call finalize explicitly, except to call super.finalize() inside an implementation of finalize(). In mobile code situations, the otherwise error prone practice of manual garbage collection can become a security threat if an attacker can maliciously invoke one of your finalize() methods because it is declared with public access.",,unclassified, CWE-585,EN-Empty Synchronized Block (Type: Variant),"The software contains an empty synchronized block. -An empty synchronized block does not actually accomplish any synchronization and may indicate a troubled section of code. An empty synchronized block can occur because code no longer needed within the synchronized block is commented out without removing the synchronized block.",,unclassified,Intrinsic Locks and Synchronization (in Java): http://java.sun.com/docs/books/tutorial/essential/concurrency/locksync.html +An empty synchronized block does not actually accomplish any synchronization and may indicate a troubled section of code. An empty synchronized block can occur because code no longer needed within the synchronized block is commented out without removing the synchronized block.",,unclassified,Intrinsic Locks and Synchronization (in Java): http://java.sun.com/docs/books/tutorial/essential/concurrency/locksync.html CWE-586,EN-Explicit Call to Finalize() (Type: Variant),"The software makes an explicit call to the finalize() method from outside the finalizer. -While the Java Language Specification allows an object's finalize() method to be called from outside the finalizer, doing so is usually a bad idea. For example, calling finalize() explicitly means that finalize() will be called more than once: the first time will be the explicit call and the last time will be the call that is made after the object is garbage collected.",,unclassified, +While the Java Language Specification allows an object's finalize() method to be called from outside the finalizer, doing so is usually a bad idea. For example, calling finalize() explicitly means that finalize() will be called more than once: the first time will be the explicit call and the last time will be the call that is made after the object is garbage collected.",,unclassified, CWE-587,EN-Assignment of a Fixed Address to a Pointer (Type: Base),"The software sets a pointer to a specific address other than NULL or 0. -Using a fixed address is not portable because that address will probably not be valid in all environments or platforms.",,unclassified, +Using a fixed address is not portable because that address will probably not be valid in all environments or platforms.",,unclassified, CWE-588,EN-Attempt to Access Child of a Non-structure Pointer (Type: Variant),"Casting a non-structure type to a structure type and accessing a field can lead to memory access errors or data corruption. -Using a fixed address is not portable because that address will probably not be valid in all environments or platforms.",,unclassified, +Using a fixed address is not portable because that address will probably not be valid in all environments or platforms.",,unclassified, CWE-589,EN-Call to Non-ubiquitous API (Type: Variant),"The software uses an API function that does not exist on all versions of the target platform. This could cause portability problems or inconsistencies that allow denial of service or other consequences. -Some functions that offer security features supported by the OS are not available on all versions of the OS in common use. Likewise, functions are often deprecated or made obsolete for security reasons and should not be used.",,unclassified, +Some functions that offer security features supported by the OS are not available on all versions of the OS in common use. Likewise, functions are often deprecated or made obsolete for security reasons and should not be used.",,unclassified, CWE-590,EN-Free of Memory not on the Heap (Type: Variant),"The application calls free() on a pointer to memory that was not allocated using associated heap allocation functions such as malloc(), calloc(), or realloc(). -When free() is called on an invalid pointer, the program's memory management data structures may become corrupted. This corruption can cause the program to crash or, in some circumstances, an attacker may be able to cause free() to operate on controllable memory locations to modify critical program variables or execute code.",,unclassified,Valgrind: http://valgrind.org/ +When free() is called on an invalid pointer, the program's memory management data structures may become corrupted. This corruption can cause the program to crash or, in some circumstances, an attacker may be able to cause free() to operate on controllable memory locations to modify critical program variables or execute code.",,unclassified,Valgrind: http://valgrind.org/ CWE-591,EN-Sensitive Data Storage in Improperly Locked Memory (Type: Variant),"The application stores sensitive data in memory that is not locked, or that has been incorrectly locked, which might cause the memory to be written to swap files on disk by the virtual memory manager. This can make the data more accessible to external actors. -On Windows systems the VirtualLock function can lock a page of memory to ensure that it will remain present in memory and not be swapped to disk. However, on older versions of Windows, such as 95, 98, or Me, the VirtualLock() function is only a stub and provides no protection. On POSIX systems the mlock() call ensures that a page will stay resident in memory but does not guarantee that the page will not appear in the swap. Therefore, it is unsuitable for use as a protection mechanism for sensitive data. Some platforms, in particular Linux, do make the guarantee that the page will not be swapped, but this is non-standard and is not portable. Calls to mlock() also require supervisor privilege. Return values for both of these calls must be checked to ensure that the lock operation was actually successful.",,unclassified, +On Windows systems the VirtualLock function can lock a page of memory to ensure that it will remain present in memory and not be swapped to disk. However, on older versions of Windows, such as 95, 98, or Me, the VirtualLock() function is only a stub and provides no protection. On POSIX systems the mlock() call ensures that a page will stay resident in memory but does not guarantee that the page will not appear in the swap. Therefore, it is unsuitable for use as a protection mechanism for sensitive data. Some platforms, in particular Linux, do make the guarantee that the page will not be swapped, but this is non-standard and is not portable. Calls to mlock() also require supervisor privilege. Return values for both of these calls must be checked to ensure that the lock operation was actually successful.",,unclassified, CWE-592,EN-Authentication Bypass Issues (Type: Class),"The software does not properly perform authentication, allowing it to be bypassed through various methods. -On Windows systems the VirtualLock function can lock a page of memory to ensure that it will remain present in memory and not be swapped to disk. However, on older versions of Windows, such as 95, 98, or Me, the VirtualLock() function is only a stub and provides no protection. On POSIX systems the mlock() call ensures that a page will stay resident in memory but does not guarantee that the page will not appear in the swap. Therefore, it is unsuitable for use as a protection mechanism for sensitive data. Some platforms, in particular Linux, do make the guarantee that the page will not be swapped, but this is non-standard and is not portable. Calls to mlock() also require supervisor privilege. Return values for both of these calls must be checked to ensure that the lock operation was actually successful.",,unclassified,"The Art of Software Security Assessment: Chapter 2, ""Untrustworthy Credentials"", Page 37." +On Windows systems the VirtualLock function can lock a page of memory to ensure that it will remain present in memory and not be swapped to disk. However, on older versions of Windows, such as 95, 98, or Me, the VirtualLock() function is only a stub and provides no protection. On POSIX systems the mlock() call ensures that a page will stay resident in memory but does not guarantee that the page will not appear in the swap. Therefore, it is unsuitable for use as a protection mechanism for sensitive data. Some platforms, in particular Linux, do make the guarantee that the page will not be swapped, but this is non-standard and is not portable. Calls to mlock() also require supervisor privilege. Return values for both of these calls must be checked to ensure that the lock operation was actually successful.",,unclassified,"The Art of Software Security Assessment: Chapter 2, ""Untrustworthy Credentials"", Page 37." CWE-593,EN-Authentication Bypass: OpenSSL CTX Object Modified after SSL Objects are Created (Type: Variant),"The software modifies the SSL context after connection creation has begun. -If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,unclassified, +If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,unclassified, CWE-594,EN-J2EE Framework: Saving Unserializable Objects to Disk (Type: Variant),"When the J2EE container attempts to write unserializable objects to disk there is no guarantee that the process will complete successfully. -If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,unclassified, +If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,unclassified, CWE-595,EN-Comparison of Object References Instead of Object Contents (Type: Base),"The program compares object references instead of the contents of the objects themselves, preventing it from detecting equivalent objects. -If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,unclassified, +If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,unclassified, CWE-596,EN-Incorrect Semantic Object Comparison (Type: Base),"The software does not correctly compare two objects based on their conceptual content. -If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,unclassified, +If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,unclassified, CWE-597,EN-Use of Wrong Operator in String Comparison (Type: Variant),"The product uses the wrong operator when comparing a string, such as using ""=="" when the equals() method should be used instead. -In Java, using == or != to compare two strings for equality actually compares two objects for equality, not their values. Chances are good that the two references will never be equal. While this weakness often only affects program correctness, if the equality is used for a security decision, it could be leveraged to affect program security.",,unclassified,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289." +In Java, using == or != to compare two strings for equality actually compares two objects for equality, not their values. Chances are good that the two references will never be equal. While this weakness often only affects program correctness, if the equality is used for a security decision, it could be leveraged to affect program security.",,unclassified,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289." CWE-598,EN-Information Exposure Through Query Strings in GET Request (Type: Variant),"The web application uses the GET method to process requests that contain sensitive information, which can expose that information through the browser's history, Referers, web logs, and other sources. -In Java, using == or != to compare two strings for equality actually compares two objects for equality, not their values. Chances are good that the two references will never be equal. While this weakness often only affects program correctness, if the equality is used for a security decision, it could be leveraged to affect program security.",,unclassified, +In Java, using == or != to compare two strings for equality actually compares two objects for equality, not their values. Chances are good that the two references will never be equal. While this weakness often only affects program correctness, if the equality is used for a security decision, it could be leveraged to affect program security.",,unclassified, CWE-599,EN-Missing Validation of OpenSSL Certificate (Type: Variant),"The software uses OpenSSL and trusts or uses a certificate without using the SSL_get_verify_result() function to ensure that the certificate satisfies all necessary security requirements. -This could allow an attacker to use an invalid certificate to claim to be a trusted host, use expired certificates, or conduct other attacks that could be detected if the certificate is properly validated.",,unclassified, +This could allow an attacker to use an invalid certificate to claim to be a trusted host, use expired certificates, or conduct other attacks that could be detected if the certificate is properly validated.",,unclassified, CWE-6,EN-J2EE Misconfiguration: Insufficient Session-ID Length (Type: Variant),"The J2EE application is configured to use an insufficient session ID length. -If an attacker can guess or steal a session ID, then he/she may be able to take over the user's session (called session hijacking). The number of possible session IDs increases with increased session ID length, making it more difficult to guess or steal a session ID.",,unclassified,No description: http://www.securiteam.com/securityreviews/5TP0F0UEVQ.html +If an attacker can guess or steal a session ID, then he/she may be able to take over the user's session (called session hijacking). The number of possible session IDs increases with increased session ID length, making it more difficult to guess or steal a session ID.",,unclassified,No description: http://www.securiteam.com/securityreviews/5TP0F0UEVQ.html CWE-600,EN-Uncaught Exception in Servlet (Type: Base),"The Servlet does not catch all exceptions, which may reveal sensitive debugging information. -When a Servlet throws an exception, the default error response the Servlet container sends back to the user typically includes debugging information. This information is of great value to an attacker. For example, a stack trace might show the attacker a malformed SQL query string, the type of database being used, and the version of the application container. This information enables the attacker to target known vulnerabilities in these components.",,unclassified, +When a Servlet throws an exception, the default error response the Servlet container sends back to the user typically includes debugging information. This information is of great value to an attacker. For example, a stack trace might show the attacker a malformed SQL query string, the type of database being used, and the version of the application container. This information enables the attacker to target known vulnerabilities in these components.",,unclassified, CWE-603,EN-Use of Client-Side Authentication (Type: Base),"A client/server product performs authentication within client code but not in server code, allowing server-side authentication to be bypassed via a modified client that omits the authentication check. -Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified,"The Art of Software Security Assessment: Chapter 2, ""Untrustworthy Credentials"", Page 37." +Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified,"The Art of Software Security Assessment: Chapter 2, ""Untrustworthy Credentials"", Page 37." CWE-605,EN-Multiple Binds to the Same Port (Type: Base),"When multiple sockets are allowed to bind to the same port, other services on that port may be stolen or spoofed. -Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified, +Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified, CWE-606,EN-Unchecked Input for Loop Condition (Type: Base),"The product does not properly check inputs that are used for loop conditions, potentially leading to a denial of service because of excessive looping. -Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified,"The Art of Software Security Assessment: Chapter 7, ""Looping Constructs"", Page 327." +Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified,"The Art of Software Security Assessment: Chapter 7, ""Looping Constructs"", Page 327." CWE-607,EN-Public Static Final Field References Mutable Object (Type: Variant),"A public or protected static final field references a mutable object, which allows the object to be changed by malicious code, or accidentally from another package. -Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified, +Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified, CWE-608,EN-Struts: Non-private Field in ActionForm Class (Type: Variant),"An ActionForm class contains a field that has not been declared private, which can be accessed without using a setter or getter. -Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified, +Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified, CWE-609,EN-Double-Checked Locking (Type: Base),"The program uses double-checked locking to access a resource without the overhead of explicit synchronization, but the locking is insufficient. Double-checked locking refers to the situation where a programmer checks to see if a resource has been initialized, grabs a lock, checks again to see if the resource has been initialized, and then performs the initialization if it has not occurred yet. This should not be done, as is not guaranteed to work in all languages and on all architectures. In summary, other threads may not be operating inside the synchronous block and are not guaranteed to see the operations execute in the same order as they would appear inside the synchronous block.",,unclassified,"The ""Double-Checked Locking is Broken"" Declaration: http://www.cs.umd.edu/~pugh/java/memoryModel/DoubleCheckedLocking.html JSR 133 (Java Memory Model) FAQ: http://www.cs.umd.edu/~pugh/java/memoryModel/jsr-133-faq.html#dcl -The Art of Software Security Assessment: Chapter 13, ""Threading Vulnerabilities"", Page 815." +The Art of Software Security Assessment: Chapter 13, ""Threading Vulnerabilities"", Page 815." CWE-610,EN-Externally Controlled Reference to a Resource in Another Sphere (Type: Class),"The product uses an externally controlled name or reference that resolves to a resource that is outside of the intended control sphere. -",,unclassified, +",,unclassified, CWE-611,EN-Improper Restriction of XML External Entity Reference (XXE) (Type: Variant),"The software processes an XML document that can contain XML entities with URIs that resolve to documents outside of the intended sphere of control, causing the product to embed incorrect documents into its output. XML documents optionally contain a Document Type Definition (DTD), which, among other features, enables the definition of XML entities. It is possible to define an entity by providing a substitution string in the form of a URI. The XML parser can access the contents of this URI and embed these contents back into the XML document for further processing. By submitting an XML file that defines an external entity with a file:// URI, an attacker can cause the processing application to read the contents of a local file. For example, a URI such as ""file:///c:/winnt/win.ini"" designates (in Windows) the file C:\Winnt\win.ini, or file:///etc/passwd designates the password file in Unix-based systems. Using URIs with other schemes such as http://, the attacker can force the application to make outgoing requests to servers that the attacker cannot reach directly, which can be used to bypass firewall restrictions or hide the source of attacks such as port scanning. @@ -1667,199 +1667,199 @@ XML External Entity Attacks (XXE): https://www.owasp.org/images/5/5d/XML_Exteral XXE (Xml eXternal Entity) Attack: http://www.securiteam.com/securitynews/6D0100A5PU.html XML External Entities (XXE) Attack: http://projects.webappsec.org/w/page/13247003/XML%20External%20Entities XML Denial of Service Attacks and Defenses: http://msdn.microsoft.com/en-us/magazine/ee335713.aspx -Preventing XXE in PHP: http://websec.io/2012/08/27/Preventing-XEE-in-PHP.html" +Preventing XXE in PHP: http://websec.io/2012/08/27/Preventing-XEE-in-PHP.html" CWE-612,EN-Information Exposure Through Indexing of Private Data (Type: Variant),"The product performs an indexing routine against private documents, but does not sufficiently verify that the actors who can access the index also have the privileges to access the private documents. -When an indexing routine is applied against a group of private documents, and that index's results are available to outsiders who do not have access to those documents, then outsiders might be able to obtain sensitive information by conducting targeted searches. The risk is especially dangerous if search results include surrounding text that was not part of the search query. This issue can appear in search engines that are not configured (or implemented) to ignore critical files that should remain hidden; even without permissions to download these files directly, the remote user could read them.",,unclassified, +When an indexing routine is applied against a group of private documents, and that index's results are available to outsiders who do not have access to those documents, then outsiders might be able to obtain sensitive information by conducting targeted searches. The risk is especially dangerous if search results include surrounding text that was not part of the search query. This issue can appear in search engines that are not configured (or implemented) to ignore critical files that should remain hidden; even without permissions to download these files directly, the remote user could read them.",,unclassified, CWE-613,EN-Insufficient Session Expiration (Type: Base),"According to WASC, ""Insufficient Session Expiration is when a web site permits an attacker to reuse old session credentials or session IDs for authorization."" -When an indexing routine is applied against a group of private documents, and that index's results are available to outsiders who do not have access to those documents, then outsiders might be able to obtain sensitive information by conducting targeted searches. The risk is especially dangerous if search results include surrounding text that was not part of the search query. This issue can appear in search engines that are not configured (or implemented) to ignore critical files that should remain hidden; even without permissions to download these files directly, the remote user could read them.",,unclassified, +When an indexing routine is applied against a group of private documents, and that index's results are available to outsiders who do not have access to those documents, then outsiders might be able to obtain sensitive information by conducting targeted searches. The risk is especially dangerous if search results include surrounding text that was not part of the search query. This issue can appear in search engines that are not configured (or implemented) to ignore critical files that should remain hidden; even without permissions to download these files directly, the remote user could read them.",,unclassified, CWE-614,EN-Sensitive Cookie in HTTPS Session Without Secure Attribute (Type: Variant),"The Secure attribute for sensitive cookies in HTTPS sessions is not set, which could cause the user agent to send those cookies in plaintext over an HTTP session. -When an indexing routine is applied against a group of private documents, and that index's results are available to outsiders who do not have access to those documents, then outsiders might be able to obtain sensitive information by conducting targeted searches. The risk is especially dangerous if search results include surrounding text that was not part of the search query. This issue can appear in search engines that are not configured (or implemented) to ignore critical files that should remain hidden; even without permissions to download these files directly, the remote user could read them.",,unclassified, +When an indexing routine is applied against a group of private documents, and that index's results are available to outsiders who do not have access to those documents, then outsiders might be able to obtain sensitive information by conducting targeted searches. The risk is especially dangerous if search results include surrounding text that was not part of the search query. This issue can appear in search engines that are not configured (or implemented) to ignore critical files that should remain hidden; even without permissions to download these files directly, the remote user could read them.",,unclassified, CWE-615,EN-Information Exposure Through Comments (Type: Variant),"While adding general comments is very useful, some programmers tend to leave important data, such as: filenames related to the web application, old links or links which were not meant to be browsed by users, old code fragments, etc. -An attacker who finds these comments can map the application's structure and files, expose hidden parts of the site, and study the fragments of code to reverse engineer the application, which may help develop further attacks against the site.",,unclassified, +An attacker who finds these comments can map the application's structure and files, expose hidden parts of the site, and study the fragments of code to reverse engineer the application, which may help develop further attacks against the site.",,unclassified, CWE-616,EN-Incomplete Identification of Uploaded File Variables (PHP) (Type: Variant),"The PHP application uses an old method for processing uploaded files by referencing the four global variables that are set for each file (e.g. $varname, $varname_size, $varname_name, $varname_type). These variables could be overwritten by attackers, causing the application to process unauthorized files. -These global variables could be overwritten by POST requests, cookies, or other methods of populating or overwriting these variables. This could be used to read or process arbitrary files by providing values such as ""/etc/passwd"".",,unclassified,"A Study in Scarlet - section 5, ""File Upload""" +These global variables could be overwritten by POST requests, cookies, or other methods of populating or overwriting these variables. This could be used to read or process arbitrary files by providing values such as ""/etc/passwd"".",,unclassified,"A Study in Scarlet - section 5, ""File Upload""" CWE-617,EN-Reachable Assertion (Type: Variant),"The product contains an assert() or similar statement that can be triggered by an attacker, which leads to an application exit or other behavior that is more severe than necessary. -For example, if a server handles multiple simultaneous connections, and an assert() occurs in one single connection that causes all other connections to be dropped, this is a reachable assertion that leads to a denial of service.",,unclassified, +For example, if a server handles multiple simultaneous connections, and an assert() occurs in one single connection that causes all other connections to be dropped, this is a reachable assertion that leads to a denial of service.",,unclassified, CWE-618,EN-Exposed Unsafe ActiveX Method (Type: Base),"An ActiveX control is intended for use in a web browser, but it exposes dangerous methods that perform actions that are outside of the browser's security model (e.g. the zone or domain). ActiveX controls can exercise far greater control over the operating system than typical Java or javascript. Exposed methods can be subject to various vulnerabilities, depending on the implemented behaviors of those methods, and whether input validation is performed on the provided arguments. If there is no integrity checking or origin validation, this method could be invoked by attackers.",,unclassified,"No description: http://msdn.microsoft.com/workshop/components/activex/safety.asp No description: http://msdn.microsoft.com/workshop/components/activex/security.asp -The Art of Software Security Assessment: Chapter 12, ""ActiveX Security"", Page 749." +The Art of Software Security Assessment: Chapter 12, ""ActiveX Security"", Page 749." CWE-619,EN-Dangling Database Cursor (Cursor Injection) (Type: Base),"If a database cursor is not closed properly, then it could become accessible to other users while retaining the same privileges that were originally assigned, leaving the cursor ""dangling."" For example, an improper dangling cursor could arise from unhandled exceptions. The impact of the issue depends on the cursor's role, but SQL injection attacks are commonly possible.",,unclassified,"The Oracle Hacker's Handbook -Cursor Injection: http://www.databasesecurity.com/dbsec/cursor-injection.pdf" +Cursor Injection: http://www.databasesecurity.com/dbsec/cursor-injection.pdf" CWE-62,EN-UNIX Hard Link (Type: Variant),"The software, when opening a file or directory, does not sufficiently account for when the name is associated with a hard link to a target that is outside of the intended control sphere. This could allow an attacker to cause the software to operate on unauthorized files. -Failure for a system to check for hard links can result in vulnerability to different types of attacks. For example, an attacker can escalate their privileges if a file used by a privileged program is replaced with a hard link to a sensitive file (e.g. /etc/passwd). When the process opens the file, the attacker can assume the privileges of that process.",,unclassified,"The Art of Software Security Assessment: Chapter 9, ""Hard Links"", Page 518." +Failure for a system to check for hard links can result in vulnerability to different types of attacks. For example, an attacker can escalate their privileges if a file used by a privileged program is replaced with a hard link to a sensitive file (e.g. /etc/passwd). When the process opens the file, the attacker can assume the privileges of that process.",,unclassified,"The Art of Software Security Assessment: Chapter 9, ""Hard Links"", Page 518." CWE-620,EN-Unverified Password Change (Type: Variant),"When setting a new password for a user, the product does not require knowledge of the original password, or using another form of authentication. -This could be used by an attacker to change passwords for another user, thus gaining the privileges associated with that user.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" +This could be used by an attacker to change passwords for another user, thus gaining the privileges associated with that user.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279" CWE-621,EN-Variable Extraction Error (Type: Base),"The product uses external input to determine the names of variables into which information is extracted, without verifying that the names of the specified variables are valid. This could cause the program to overwrite unintended variables. -For example, in PHP, calling extract() or import_request_variables() without the proper arguments could allow arbitrary global variables to be overwritten, including superglobals. Similar functionality might be possible in other interpreted languages, including custom languages.",,unclassified, +For example, in PHP, calling extract() or import_request_variables() without the proper arguments could allow arbitrary global variables to be overwritten, including superglobals. Similar functionality might be possible in other interpreted languages, including custom languages.",,unclassified, CWE-622,EN-Improper Validation of Function Hook Arguments (Type: Variant),"A product adds hooks to user-accessible API functions, but does not properly validate the arguments. This could lead to resultant vulnerabilities. -Such hooks can be used in defensive software that runs with privileges, such as anti-virus or firewall, which hooks kernel calls. When the arguments are not validated, they could be used to bypass the protection scheme or attack the product itself.",,unclassified, +Such hooks can be used in defensive software that runs with privileges, such as anti-virus or firewall, which hooks kernel calls. When the arguments are not validated, they could be used to bypass the protection scheme or attack the product itself.",,unclassified, CWE-623,EN-Unsafe ActiveX Control Marked Safe For Scripting (Type: Variant),"An ActiveX control is intended for restricted use, but it has been marked as safe-for-scripting. This might allow attackers to use dangerous functionality via a web page that accesses the control, which can lead to different resultant vulnerabilities, depending on the control's behavior.",,unclassified,"No description: http://msdn.microsoft.com/workshop/components/activex/safety.asp No description: http://msdn.microsoft.com/workshop/components/activex/security.asp No description: http://support.microsoft.com/kb/240797 Writing Secure Code: Chapter 16, ""What ActiveX Components Are Safe for Initialization and Safe for Scripting?"" Page 510 -The Art of Software Security Assessment: Chapter 12, ""ActiveX Security"", Page 749." +The Art of Software Security Assessment: Chapter 12, ""ActiveX Security"", Page 749." CWE-624,EN-Executable Regular Expression Error (Type: Base),"The product uses a regular expression that either (1) contains an executable component with user-controlled inputs, or (2) allows a user to enable execution by inserting pattern modifiers. -Case (2) is possible in the PHP preg_replace() function, and possibly in other languages when a user-controlled input is inserted into a string that is later parsed as a regular expression.",,unclassified, +Case (2) is possible in the PHP preg_replace() function, and possibly in other languages when a user-controlled input is inserted into a string that is later parsed as a regular expression.",,unclassified, CWE-625,EN-Permissive Regular Expression (Type: Base),"The product uses a regular expression that does not sufficiently restrict the set of allowed values. This effectively causes the regexp to accept substrings that match the pattern, which produces a partial comparison to the target. In some cases, this can lead to other weaknesses. Common errors include: not identifying the beginning and end of the target string using wildcards instead of acceptable character ranges -others",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Character Stripping Vulnerabilities"", Page 437." +others",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Character Stripping Vulnerabilities"", Page 437." CWE-626,EN-Null Byte Interaction Error (Poison Null Byte) (Type: Variant),"The product does not properly handle null bytes or NUL characters when passing data between different representations or components. A null byte (NUL character) can have different meanings across representations or languages. For example, it is a string terminator in standard C libraries, but Perl and PHP strings do not treat it as a terminator. When two representations are crossed - such as when Perl or PHP invokes underlying C functionality - this can produce an interaction error with unexpected results. Similar issues have been reported for ASP. Other interpreters written in C might also be affected.",,unclassified,"Poison NULL byte: http://insecure.org/news/P55-07.txt 0x00 vs ASP file upload scripts: http://www.security-assessment.com/Whitepapers/0x00_vs_ASP_File_Uploads.pdf -ShAnKaR: multiple PHP application poison NULL byte vulnerability: http://seclists.org/fulldisclosure/2006/Sep/0185.html" +ShAnKaR: multiple PHP application poison NULL byte vulnerability: http://seclists.org/fulldisclosure/2006/Sep/0185.html" CWE-627,EN-Dynamic Variable Evaluation (Type: Base),"In a language where the user can influence the name of a variable at runtime, if the variable names are not controlled, an attacker can read or write to arbitrary variables, or access arbitrary functions. The resultant vulnerabilities depend on the behavior of the application, both at the crossover point and in any control/data flow that is reachable by the related variables or functions.",,unclassified,"Dynamic Evaluation Vulnerabilities in PHP applications: http://seclists.org/fulldisclosure/2006/May/0035.html -A Study In Scarlet: Exploiting Common Vulnerabilities in PHP Applications: http://www.securereality.com.au/studyinscarlet.txt" +A Study In Scarlet: Exploiting Common Vulnerabilities in PHP Applications: http://www.securereality.com.au/studyinscarlet.txt" CWE-628,EN-Function Call with Incorrectly Specified Arguments (Type: Base),"The product calls a function, procedure, or routine with arguments that are not correctly specified, leading to always-incorrect behavior and resultant weaknesses. There are multiple ways in which this weakness can be introduced, including: the wrong variable or reference; an incorrect number of arguments; incorrect order of arguments; wrong type of arguments; or -wrong value.",,unclassified, +wrong value.",,unclassified, CWE-636,EN-Not Failing Securely (Failing Open) (Type: Class),"When the product encounters an error condition or failure, its design requires it to fall back to a state that is less secure than other options that are available, such as selecting the weakest encryption algorithm or using the most permissive access control restrictions. By entering a less secure state, the product inherits the weaknesses associated with that state, making it easier to compromise. At the least, it causes administrators to have a false sense of security. This weakness typically occurs as a result of wanting to ""fail functional"" to minimize administration and support costs, instead of ""failing safe.""",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ -Failing Securely: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/349.html" +Failing Securely: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/349.html" CWE-637,EN-Unnecessary Complexity in Protection Mechanism (Not Using Economy of Mechanism) (Type: Class),"The software uses a more complex mechanism than necessary, which could lead to resultant weaknesses when the mechanism is not correctly understood, modeled, configured, implemented, or used. Security mechanisms should be as simple as possible. Complex security mechanisms may engender partial implementations and compatibility problems, with resulting mismatches in assumptions and implemented security. A corollary of this principle is that data specifications should be as simple as possible, because complex data specifications result in complex validation code. Complex tasks and systems may also need to be guarded by complex security checks, so simple systems should be preferred.",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ -Economy of Mechanism: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/348.html" +Economy of Mechanism: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/348.html" CWE-638,EN-Not Using Complete Mediation (Type: Class),"The software does not perform access checks on a resource every time the resource is accessed by an entity, which can create resultant weaknesses if that entity's rights or privileges change over time. ",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ -Complete Mediation: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/346.html" +Complete Mediation: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/346.html" CWE-65,EN-Windows Hard Link (Type: Variant),"The software, when opening a file or directory, does not sufficiently handle when the name is associated with a hard link to a target that is outside of the intended control sphere. This could allow an attacker to cause the software to operate on unauthorized files. -Failure for a system to check for hard links can result in vulnerability to different types of attacks. For example, an attacker can escalate their privileges if a file used by a privileged program is replaced with a hard link to a sensitive file (e.g. AUTOEXEC.BAT). When the process opens the file, the attacker can assume the privileges of that process, or prevent the program from accurately processing data.",,unclassified,"The Art of Software Security Assessment: Chapter 11, ""Links"", Page 676." +Failure for a system to check for hard links can result in vulnerability to different types of attacks. For example, an attacker can escalate their privileges if a file used by a privileged program is replaced with a hard link to a sensitive file (e.g. AUTOEXEC.BAT). When the process opens the file, the attacker can assume the privileges of that process, or prevent the program from accurately processing data.",,unclassified,"The Art of Software Security Assessment: Chapter 11, ""Links"", Page 676." CWE-651,EN-Information Exposure Through WSDL File (Type: Variant),"The Web services architecture may require exposing a WSDL file that contains information on the publicly accessible services and how callers of these services should interact with them (e.g. what parameters they expect and what types they return). An information exposure may occur if any of the following apply: The WSDL file is accessible to a wider audience than intended. The WSDL file contains information on the methods/services that should not be publicly accessible or information about deprecated methods. This problem is made more likely due to the WSDL often being automatically generated from the code. -Information in the WSDL file helps guess names/locations of methods/resources that should not be publicly accessible.",,unclassified, +Information in the WSDL file helps guess names/locations of methods/resources that should not be publicly accessible.",,unclassified, CWE-653,EN-Insufficient Compartmentalization (Type: Base),"The product does not sufficiently compartmentalize functionality or processes that require different privilege levels, rights, or permissions. When a weakness occurs in functionality that is accessible by lower-privileged users, then without strong boundaries, an attack might extend the scope of the damage to higher-privileged users.",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ -Separation of Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/357.html" +Separation of Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/357.html" CWE-654,EN-Reliance on a Single Factor in a Security Decision (Type: Base),"A protection mechanism relies exclusively, or to a large extent, on the evaluation of a single condition or the integrity of a single object or entity in order to make a decision about granting access to restricted resources or functionality. When a weakness occurs in functionality that is accessible by lower-privileged users, then without strong boundaries, an attack might extend the scope of the damage to higher-privileged users.",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ -Separation of Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/357.html" +Separation of Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/357.html" CWE-655,EN-Insufficient Psychological Acceptability (Type: Base),"The software has a protection mechanism that is too difficult or inconvenient to use, encouraging non-malicious users to disable or bypass the mechanism, whether by accident or on purpose. When a weakness occurs in functionality that is accessible by lower-privileged users, then without strong boundaries, an attack might extend the scope of the damage to higher-privileged users.",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ Psychological Acceptability: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/354.html Usability of Security: A Case Study: http://reports-archive.adm.cs.cmu.edu/anon/1998/CMU-CS-98-155.pdf -24 Deadly Sins of Software Security: ""Sin 14: Poor Usability."" Page 217" +24 Deadly Sins of Software Security: ""Sin 14: Poor Usability."" Page 217" CWE-656,EN-Reliance on Security Through Obscurity (Type: Base),"The software uses a protection mechanism whose strength depends heavily on its obscurity, such that knowledge of its algorithms or key data is sufficient to defeat the mechanism. This reliance on ""security through obscurity"" can produce resultant weaknesses if an attacker is able to reverse engineer the inner workings of the mechanism. Note that obscurity can be one small part of defense in depth, since it can create more work for an attacker; however, it is a significant risk if used as the primary means of protection.",,unclassified,"RFC: 793, TRANSMISSION CONTROL PROTOCOL: http://www.ietf.org/rfc/rfc0793.txt The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ -Never Assuming that Your Secrets Are Safe: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/352.html" +Never Assuming that Your Secrets Are Safe: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/352.html" CWE-657,EN-Violation of Secure Design Principles (Type: Class),"The product violates well-established principles for secure design. This can introduce resultant weaknesses or make it easier for developers to introduce related weaknesses during implementation. Because code is centered around design, it can be resource-intensive to fix design problems.",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/ -Design Principles: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/358.html" +Design Principles: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/358.html" CWE-66,EN-Improper Handling of File Names that Identify Virtual Resources (Type: Base),"The product does not handle or incorrectly handles a file name that identifies a ""virtual"" resource that is not directly specified within the directory that is associated with the file name, causing the product to perform file-based operations on a resource that is not a file. -Virtual file names are represented like normal file names, but they are effectively aliases for other resources that do not behave like normal files. Depending on their functionality, they could be alternate entities. They are not necessarily listed in directories.",,unclassified, +Virtual file names are represented like normal file names, but they are effectively aliases for other resources that do not behave like normal files. Depending on their functionality, they could be alternate entities. They are not necessarily listed in directories.",,unclassified, CWE-662,EN-Improper Synchronization (Type: Base),"The software attempts to use a shared resource in an exclusive manner, but does not prevent or incorrectly prevents use of the resource by another thread or process. -Virtual file names are represented like normal file names, but they are effectively aliases for other resources that do not behave like normal files. Depending on their functionality, they could be alternate entities. They are not necessarily listed in directories.",,unclassified, +Virtual file names are represented like normal file names, but they are effectively aliases for other resources that do not behave like normal files. Depending on their functionality, they could be alternate entities. They are not necessarily listed in directories.",,unclassified, CWE-663,EN-Use of a Non-reentrant Function in a Concurrent Context (Type: Base),"The software calls a non-reentrant function in a concurrent context in which a competing code sequence (e.g. thread or signal handler) may have an opportunity to call the same function or otherwise influence its state. Virtual file names are represented like normal file names, but they are effectively aliases for other resources that do not behave like normal files. Depending on their functionality, they could be alternate entities. They are not necessarily listed in directories.",,unclassified,"Java Concurrency API: http://java.sun.com/j2se/1.5.0/docs/api/java/util/concurrent/locks/ReentrantLock.html -Use reentrant functions for safer signal handling: http://www.ibm.com/developerworks/linux/library/l-reent.html" +Use reentrant functions for safer signal handling: http://www.ibm.com/developerworks/linux/library/l-reent.html" CWE-664,EN-Improper Control of a Resource Through its Lifetime (Type: Class),"The software does not maintain or incorrectly maintains control over a resource throughout its lifetime of creation, use, and release. Resources often have explicit instructions on how to be created, used and destroyed. When software does not follow these instructions, it can lead to unexpected behaviors and potentially exploitable states. -Even without explicit instructions, various principles are expected to be adhered to, such as ""Do not use an object until after its creation is complete,"" or ""do not use an object after it has been slated for destruction.""",,unclassified, +Even without explicit instructions, various principles are expected to be adhered to, such as ""Do not use an object until after its creation is complete,"" or ""do not use an object after it has been slated for destruction.""",,unclassified, CWE-666,EN-Operation on Resource in Wrong Phase of Lifetime (Type: Base),"The software performs an operation on a resource at the wrong phase of the resource's lifecycle, which can lead to unexpected behaviors. -When a developer wants to initialize, use or release a resource, it is important to follow the specifications outlined for how to operate on that resource and to ensure that the resource is in the expected state. In this case, the software wants to perform a normally valid operation, initialization, use or release, on a resource when it is in the incorrect phase of its lifetime.",,unclassified, +When a developer wants to initialize, use or release a resource, it is important to follow the specifications outlined for how to operate on that resource and to ensure that the resource is in the expected state. In this case, the software wants to perform a normally valid operation, initialization, use or release, on a resource when it is in the incorrect phase of its lifetime.",,unclassified, CWE-667,EN-Improper Locking (Type: Base),"The software does not properly acquire a lock on a resource, or it does not properly release a lock on a resource, leading to unexpected resource state changes and behaviors. -When a developer wants to initialize, use or release a resource, it is important to follow the specifications outlined for how to operate on that resource and to ensure that the resource is in the expected state. In this case, the software wants to perform a normally valid operation, initialization, use or release, on a resource when it is in the incorrect phase of its lifetime.",,unclassified, +When a developer wants to initialize, use or release a resource, it is important to follow the specifications outlined for how to operate on that resource and to ensure that the resource is in the expected state. In this case, the software wants to perform a normally valid operation, initialization, use or release, on a resource when it is in the incorrect phase of its lifetime.",,unclassified, CWE-668,EN-Exposure of Resource to Wrong Sphere (Type: Class),"The product exposes a resource to the wrong control sphere, providing unintended actors with inappropriate access to the resource. Resources such as files and directories may be inadvertently exposed through mechanisms such as insecure permissions, or when a program accidentally operates on the wrong object. For example, a program may intend that private files can only be provided to a specific user. This effectively defines a control sphere that is intended to prevent attackers from accessing these private files. If the file permissions are insecure, then parties other than the user will be able to access those files. A separate control sphere might effectively require that the user can only access the private files, but not any other files on the system. If the program does not ensure that the user is only requesting private files, then the user might be able to access other files on the system. -In either case, the end result is that a resource has been exposed to the wrong party.",,unclassified, +In either case, the end result is that a resource has been exposed to the wrong party.",,unclassified, CWE-669,EN-Incorrect Resource Transfer Between Spheres (Type: Class),"The product does not properly transfer a resource/behavior to another sphere, or improperly imports a resource/behavior from another sphere, in a manner that provides unintended control over that resource. Resources such as files and directories may be inadvertently exposed through mechanisms such as insecure permissions, or when a program accidentally operates on the wrong object. For example, a program may intend that private files can only be provided to a specific user. This effectively defines a control sphere that is intended to prevent attackers from accessing these private files. If the file permissions are insecure, then parties other than the user will be able to access those files. A separate control sphere might effectively require that the user can only access the private files, but not any other files on the system. If the program does not ensure that the user is only requesting private files, then the user might be able to access other files on the system. -In either case, the end result is that a resource has been exposed to the wrong party.",,unclassified, +In either case, the end result is that a resource has been exposed to the wrong party.",,unclassified, CWE-670,EN-Always-Incorrect Control Flow Implementation (Type: Class),"The code contains a control flow path that does not reflect the algorithm that the path is intended to implement, leading to incorrect behavior any time this path is navigated. -This weakness captures cases in which a particular code segment is always incorrect with respect to the algorithm that it is implementing. For example, if a C programmer intends to include multiple statements in a single block but does not include the enclosing braces (CWE-483), then the logic is always incorrect. This issue is in contrast to most weaknesses in which the code usually behaves correctly, except when it is externally manipulated in malicious ways.",,unclassified, +This weakness captures cases in which a particular code segment is always incorrect with respect to the algorithm that it is implementing. For example, if a C programmer intends to include multiple statements in a single block but does not include the enclosing braces (CWE-483), then the logic is always incorrect. This issue is in contrast to most weaknesses in which the code usually behaves correctly, except when it is externally manipulated in malicious ways.",,unclassified, CWE-671,EN-Lack of Administrator Control over Security (Type: Class),"The product uses security features in a way that prevents the product's administrator from tailoring security settings to reflect the environment in which the product is being used. This introduces resultant weaknesses or prevents it from operating at a level of security that is desired by the administrator. -If the product's administrator does not have the ability to manage security-related decisions at all times, then protecting the product from outside threats - including the product's developer - can become impossible. For example, a hard-coded account name and password cannot be changed by the administrator, thus exposing that product to attacks that the administrator can not prevent.",,unclassified, +If the product's administrator does not have the ability to manage security-related decisions at all times, then protecting the product from outside threats - including the product's developer - can become impossible. For example, a hard-coded account name and password cannot be changed by the administrator, thus exposing that product to attacks that the administrator can not prevent.",,unclassified, CWE-672,EN-Operation on a Resource after Expiration or Release (Type: Base),"The software uses, accesses, or otherwise operates on a resource after that resource has been expired, released, or revoked. -If the product's administrator does not have the ability to manage security-related decisions at all times, then protecting the product from outside threats - including the product's developer - can become impossible. For example, a hard-coded account name and password cannot be changed by the administrator, thus exposing that product to attacks that the administrator can not prevent.",,unclassified, +If the product's administrator does not have the ability to manage security-related decisions at all times, then protecting the product from outside threats - including the product's developer - can become impossible. For example, a hard-coded account name and password cannot be changed by the administrator, thus exposing that product to attacks that the administrator can not prevent.",,unclassified, CWE-673,EN-External Influence of Sphere Definition (Type: Class),"The product does not prevent the definition of control spheres from external actors. -Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,unclassified, +Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,unclassified, CWE-674,EN-Uncontrolled Recursion (Type: Base),"The product does not properly control the amount of recursion that takes place, which consumes excessive resources, such as allocated memory or the program stack. -Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,unclassified, +Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,unclassified, CWE-675,EN-Duplicate Operations on Resource (Type: Class),"The product performs the same operation on a resource two or more times, when the operation should only be applied once. -Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,unclassified, +Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,unclassified, CWE-683,EN-Function Call With Incorrect Order of Arguments (Type: Variant),"The software calls a function, procedure, or routine, but the caller specifies the arguments in an incorrect order, leading to resultant weaknesses. -While this weakness might be caught by the compiler in some languages, it can occur more frequently in cases in which the called function accepts variable numbers or types of arguments, such as format strings in C. It also can occur in languages or environments that do not enforce strong typing.",,unclassified, +While this weakness might be caught by the compiler in some languages, it can occur more frequently in cases in which the called function accepts variable numbers or types of arguments, such as format strings in C. It also can occur in languages or environments that do not enforce strong typing.",,unclassified, CWE-684,EN-Incorrect Provision of Specified Functionality (Type: Base),"The code does not function according to its published specifications, potentially leading to incorrect usage. -When providing functionality to an external party, it is important that the software behaves in accordance with the details specified. When requirements of nuances are not documented, the functionality may produce unintended behaviors for the caller, possibly leading to an exploitable state.",,unclassified, +When providing functionality to an external party, it is important that the software behaves in accordance with the details specified. When requirements of nuances are not documented, the functionality may produce unintended behaviors for the caller, possibly leading to an exploitable state.",,unclassified, CWE-685,EN-Function Call With Incorrect Number of Arguments (Type: Variant),"The software calls a function, procedure, or routine, but the caller specifies too many arguments, or too few arguments, which may lead to undefined behavior and resultant weaknesses. -When providing functionality to an external party, it is important that the software behaves in accordance with the details specified. When requirements of nuances are not documented, the functionality may produce unintended behaviors for the caller, possibly leading to an exploitable state.",,unclassified, +When providing functionality to an external party, it is important that the software behaves in accordance with the details specified. When requirements of nuances are not documented, the functionality may produce unintended behaviors for the caller, possibly leading to an exploitable state.",,unclassified, CWE-686,EN-Function Call With Incorrect Argument Type (Type: Variant),"The software calls a function, procedure, or routine, but the caller specifies an argument that is the wrong data type, which may lead to resultant weaknesses. -This weakness is most likely to occur in loosely typed languages, or in strongly typed languages in which the types of variable arguments cannot be enforced at compilation time, or where there is implicit casting.",,unclassified, +This weakness is most likely to occur in loosely typed languages, or in strongly typed languages in which the types of variable arguments cannot be enforced at compilation time, or where there is implicit casting.",,unclassified, CWE-687,EN-Function Call With Incorrectly Specified Argument Value (Type: Variant),"The software calls a function, procedure, or routine, but the caller specifies an argument that contains the wrong value, which may lead to resultant weaknesses. -This weakness is most likely to occur in loosely typed languages, or in strongly typed languages in which the types of variable arguments cannot be enforced at compilation time, or where there is implicit casting.",,unclassified, +This weakness is most likely to occur in loosely typed languages, or in strongly typed languages in which the types of variable arguments cannot be enforced at compilation time, or where there is implicit casting.",,unclassified, CWE-688,EN-Function Call With Incorrect Variable or Reference as Argument (Type: Variant),"The software calls a function, procedure, or routine, but the caller specifies the wrong variable or reference as one of the arguments, which may lead to undefined behavior and resultant weaknesses. -This weakness is most likely to occur in loosely typed languages, or in strongly typed languages in which the types of variable arguments cannot be enforced at compilation time, or where there is implicit casting.",,unclassified, +This weakness is most likely to occur in loosely typed languages, or in strongly typed languages in which the types of variable arguments cannot be enforced at compilation time, or where there is implicit casting.",,unclassified, CWE-69,EN-Improper Handling of Windows ::DATA Alternate Data Stream (Type: Variant),"The software does not properly prevent access to, or detect usage of, alternate data streams (ADS). An attacker can use an ADS to hide information about a file (e.g. size, the name of the process) from a system or file browser tools such as Windows Explorer and 'dir' at the command line utility. Alternately, the attacker might be able to bypass intended access restrictions for the associated data fork.",,unclassified,"Windows NTFS Alternate Data Streams: http://www.securityfocus.com/infocus/1822 -Writing Secure Code" +Writing Secure Code" CWE-691,EN-Insufficient Control Flow Management (Type: Class),"The code does not sufficiently manage its control flow during execution, creating conditions in which the control flow can be modified in unexpected ways. -An attacker can use an ADS to hide information about a file (e.g. size, the name of the process) from a system or file browser tools such as Windows Explorer and 'dir' at the command line utility. Alternately, the attacker might be able to bypass intended access restrictions for the associated data fork.",,unclassified, +An attacker can use an ADS to hide information about a file (e.g. size, the name of the process) from a system or file browser tools such as Windows Explorer and 'dir' at the command line utility. Alternately, the attacker might be able to bypass intended access restrictions for the associated data fork.",,unclassified, CWE-693,EN-Protection Mechanism Failure (Type: Class),"The product does not use or incorrectly uses a protection mechanism that provides sufficient defense against directed attacks against the product. -This weakness covers three distinct situations. A ""missing"" protection mechanism occurs when the application does not define any mechanism against a certain class of attack. An ""insufficient"" protection mechanism might provide some defenses - for example, against the most common attacks - but it does not protect against everything that is intended. Finally, an ""ignored"" mechanism occurs when a mechanism is available and in active use within the product, but the developer has not applied it in some code path.",,unclassified, +This weakness covers three distinct situations. A ""missing"" protection mechanism occurs when the application does not define any mechanism against a certain class of attack. An ""insufficient"" protection mechanism might provide some defenses - for example, against the most common attacks - but it does not protect against everything that is intended. Finally, an ""ignored"" mechanism occurs when a mechanism is available and in active use within the product, but the developer has not applied it in some code path.",,unclassified, CWE-694,EN-Use of Multiple Resources with Duplicate Identifier (Type: Base),"The software uses multiple resources that can have the same identifier, in a context in which unique identifiers are required. -If the software assumes that each resource has a unique identifier, the software could operate on the wrong resource if attackers can cause multiple resources to be associated with the same identifier.",,unclassified, +If the software assumes that each resource has a unique identifier, the software could operate on the wrong resource if attackers can cause multiple resources to be associated with the same identifier.",,unclassified, CWE-695,EN-Use of Low-Level Functionality (Type: Base),"The software uses low-level functionality that is explicitly prohibited by the framework or specification under which the software is supposed to operate. -The use of low-level functionality can violate the specification in unexpected ways that effectively disable built-in protection mechanisms, introduce exploitable inconsistencies, or otherwise expose the functionality to attack.",,unclassified, +The use of low-level functionality can violate the specification in unexpected ways that effectively disable built-in protection mechanisms, introduce exploitable inconsistencies, or otherwise expose the functionality to attack.",,unclassified, CWE-696,EN-Incorrect Behavior Order (Type: Class),"The software performs multiple related behaviors, but the behaviors are performed in the wrong order in ways which may produce resultant weaknesses. -The use of low-level functionality can violate the specification in unexpected ways that effectively disable built-in protection mechanisms, introduce exploitable inconsistencies, or otherwise expose the functionality to attack.",,unclassified, +The use of low-level functionality can violate the specification in unexpected ways that effectively disable built-in protection mechanisms, introduce exploitable inconsistencies, or otherwise expose the functionality to attack.",,unclassified, CWE-697,EN-Insufficient Comparison (Type: Class),"The software compares two entities in a security-relevant context, but the comparison is insufficient, which may lead to resultant weaknesses. This weakness class covers several possibilities: the comparison checks one factor incorrectly; -the comparison should consider multiple factors, but it does not check some of those factors at all.",,unclassified, +the comparison should consider multiple factors, but it does not check some of those factors at all.",,unclassified, CWE-698,EN-Execution After Redirect (EAR) (Type: Base),"The web application sends a redirect to another location, but instead of exiting, it executes additional code. This weakness class covers several possibilities: the comparison checks one factor incorrectly; -the comparison should consider multiple factors, but it does not check some of those factors at all.",,unclassified,Fear the EAR: Discovering and Mitigating Execution After Redirect Vulnerabilities: http://cs.ucsb.edu/~bboe/public/pubs/fear-the-ear-ccs2011.pdf +the comparison should consider multiple factors, but it does not check some of those factors at all.",,unclassified,Fear the EAR: Discovering and Mitigating Execution After Redirect Vulnerabilities: http://cs.ucsb.edu/~bboe/public/pubs/fear-the-ear-ccs2011.pdf CWE-7,EN-J2EE Misconfiguration: Missing Custom Error Page (Type: Variant),"The default error page of a web application should not display sensitive information about the software system. -A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified,19 Deadly Sins of Software Security +A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified,19 Deadly Sins of Software Security CWE-703,EN-Improper Check or Handling of Exceptional Conditions (Type: Class),"The software does not properly anticipate or handle exceptional conditions that rarely occur during normal operation of the software. A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified,"A Taxonomy of Security Faults in the UNIX Operating System: http://ftp.cerias.purdue.edu/pub/papers/taimur-aslam/aslam-taxonomy-msthesis.pdf Use of A Taxonomy of Security Faults: http://csrc.nist.gov/nissc/1996/papers/NISSC96/paper057/PAPER.PDF -24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143" +24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143" CWE-704,EN-Incorrect Type Conversion or Cast (Type: Class),"The software does not correctly convert an object, resource or structure from one type to a different type. -A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified, +A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified, CWE-705,EN-Incorrect Control Flow Scoping (Type: Class),"The software does not properly return control flow to the proper location after it has completed a task or detected an unusual condition. -A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified, +A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified, CWE-706,EN-Use of Incorrectly-Resolved Name or Reference (Type: Class),"The software uses a name or reference to access a resource, but the name/reference resolves to a resource that is outside of the intended control sphere. -A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified, +A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified, CWE-707,EN-Improper Enforcement of Message or Data Structure (Type: Class),"The software does not enforce or incorrectly enforces that structured messages or data are well-formed before being read from an upstream component or sent to a downstream component. If a message is malformed it may cause the message to be incorrectly interpreted. -This weakness typically applies in cases where the product prepares a control message that another process must act on, such as a command or query, and malicious input that was intended as data, can enter the control plane instead. However, this weakness also applies to more general cases where there are not always control implications.",,unclassified, +This weakness typically applies in cases where the product prepares a control message that another process must act on, such as a command or query, and malicious input that was intended as data, can enter the control plane instead. However, this weakness also applies to more general cases where there are not always control implications.",,unclassified, CWE-708,EN-Incorrect Ownership Assignment (Type: Base),"The software assigns an owner to a resource, but the owner is outside of the intended control sphere. -This may allow the resource to be manipulated by actors outside of the intended control sphere.",,unclassified, +This may allow the resource to be manipulated by actors outside of the intended control sphere.",,unclassified, CWE-71,EN-Apple .DS_Store (Type: Variant),"Software operating in a MAC OS environment, where .DS_Store is in effect, must carefully manage hard links, otherwise an attacker may be able to leverage a hard link from .DS_Store to overwrite arbitrary files and gain privileges. -This may allow the resource to be manipulated by actors outside of the intended control sphere.",,unclassified, +This may allow the resource to be manipulated by actors outside of the intended control sphere.",,unclassified, CWE-710,EN-Coding Standards Violation (Type: Class),"The software does not follow certain coding rules for development, which can lead to resultant weaknesses or increase the severity of the associated vulnerabilities. -This may allow the resource to be manipulated by actors outside of the intended control sphere.",,unclassified, +This may allow the resource to be manipulated by actors outside of the intended control sphere.",,unclassified, CWE-72,EN-Improper Handling of Apple HFS+ Alternate Data Stream Path (Type: Variant),"The software does not properly handle special paths that may identify the data or resource fork of a file on the HFS+ file system. -If the software chooses actions to take based on the file name, then if an attacker provides the data or resource fork, the software may take unexpected actions. Further, if the software intends to restrict access to a file, then an attacker might still be able to bypass intended access restrictions by requesting the data or resource fork for that file.",,unclassified,No description: http://docs.info.apple.com/article.html?artnum=300422 +If the software chooses actions to take based on the file name, then if an attacker provides the data or resource fork, the software may take unexpected actions. Further, if the software intends to restrict access to a file, then an attacker might still be able to bypass intended access restrictions by requesting the data or resource fork for that file.",,unclassified,No description: http://docs.info.apple.com/article.html?artnum=300422 CWE-733,EN-Compiler Optimization Removal or Modification of Security-critical Code (Type: Base),"The developer builds a security-critical protection mechanism into the software but the compiler optimizes the program such that the mechanism is removed or modified. -When a resource is given a permissions setting that provides access to a wider range of actors than required, it could lead to the exposure of sensitive information, or the modification of that resource by unintended parties. This is especially dangerous when the resource is related to program configuration, execution or sensitive user data.",,unclassified,"Writing Secure Code: Chapter 9, ""A Compiler Optimization Caveat"" Page 322" +When a resource is given a permissions setting that provides access to a wider range of actors than required, it could lead to the exposure of sensitive information, or the modification of that resource by unintended parties. This is especially dangerous when the resource is related to program configuration, execution or sensitive user data.",,unclassified,"Writing Secure Code: Chapter 9, ""A Compiler Optimization Caveat"" Page 322" CWE-75,EN-Failure to Sanitize Special Elements into a Different Plane (Special Element Injection) (Type: Class),"The software does not adequately filter user-controlled input for special elements with control implications. This weakness can lead to a wide variety of resultant weaknesses, depending on the behavior of the exposed method. It can apply to any number of technologies and approaches, such as ActiveX controls, Java functions, IOCTLs, and so on. The exposure can occur in a few different ways: 1) The function/method was never intended to be exposed to outside actors. -2) The function/method was only intended to be accessible to a limited set of actors, such as Internet-based access from a single web site.",,unclassified, +2) The function/method was only intended to be accessible to a limited set of actors, such as Internet-based access from a single web site.",,unclassified, CWE-756,EN-Missing Custom Error Page (Type: Class),"The software does not return custom error pages to the user, possibly exposing sensitive information. The programmer may assume that certain events or conditions will never occur or do not need to be worried about, such as low memory conditions, lack of access to resources due to restrictive permissions, or misbehaving clients or components. However, attackers may intentionally trigger these unusual conditions, thus violating the programmer's assumptions, possibly introducing instability, incorrect behavior, or a vulnerability. -Note that this entry is not exclusively about the use of exceptions and exception handling, which are mechanisms for both checking and handling unusual or unexpected conditions.",,unclassified, +Note that this entry is not exclusively about the use of exceptions and exception handling, which are mechanisms for both checking and handling unusual or unexpected conditions.",,unclassified, CWE-757,EN-Selection of Less-Secure Algorithm During Negotiation (Algorithm Downgrade) (Type: Class),"A protocol or its implementation supports interaction between multiple actors and allows those actors to negotiate which algorithm should be used as a protection mechanism such as encryption or authentication, but it does not select the strongest algorithm that is available to both parties. -When a security mechanism can be forced to downgrade to use a less secure algorithm, this can make it easier for attackers to compromise the software by exploiting weaker algorithm. The victim might not be aware that the less secure algorithm is being used. For example, if an attacker can force a communications channel to use cleartext instead of strongly-encrypted data, then the attacker could read the channel by sniffing, instead of going through extra effort of trying to decrypt the data using brute force techniques.",,unclassified, +When a security mechanism can be forced to downgrade to use a less secure algorithm, this can make it easier for attackers to compromise the software by exploiting weaker algorithm. The victim might not be aware that the less secure algorithm is being used. For example, if an attacker can force a communications channel to use cleartext instead of strongly-encrypted data, then the attacker could read the channel by sniffing, instead of going through extra effort of trying to decrypt the data using brute force techniques.",,unclassified, CWE-758,"EN-Reliance on Undefined, Unspecified, or Implementation-Defined Behavior (Type: Class)","The software uses an API function, data structure, or other entity in a way that relies on properties that are not always guaranteed to hold for that entity. -This can lead to resultant weaknesses when the required properties change, such as when the software is ported to a different platform or if an interaction error (CWE-435) occurs.",,unclassified, +This can lead to resultant weaknesses when the required properties change, such as when the software is ported to a different platform or if an interaction error (CWE-435) occurs.",,unclassified, CWE-759,EN-Use of a One-Way Hash without a Salt (Type: Base),"The software uses a one-way cryptographic hash against an input that should not be reversible, such as a password, but the software does not also use a salt as part of the input. This makes it easier for attackers to pre-compute the hash value using dictionary attack techniques such as rainbow tables. It should be noted that, despite common perceptions, the use of a good salt with a hash does not sufficiently increase the effort for an attacker who is targeting an individual password, or who has a large amount of computing resources available, such as with cloud-based services or specialized, inexpensive hardware. Offline password cracking can still be effective if the hash function is not expensive to compute; many cryptographic functions are designed to be efficient and can be vulnerable to attacks using massive computing resources, even if the hash is cryptographically strong. The use of a salt only slightly increases the computing requirements for an attacker compared to other strategies such as adaptive hash functions. See CWE-916 for more details.",,unclassified,"bcrypt: http://bcrypt.sourceforge.net/ @@ -1878,7 +1878,7 @@ Writing Secure Code: Chapter 9, ""Creating a Salted Hash"" Page 302 The Art of Software Security Assessment: Chapter 2, ""Salt Values"", Page 46. How To Safely Store A Password: http://codahale.com/how-to-safely-store-a-password/ Our password hashing has no clothes: http://www.troyhunt.com/2012/06/our-password-hashing-has-no-clothes.html -Should we really use bcrypt/scrypt?: http://www.analyticalengine.net/2012/06/should-we-really-use-bcryptscrypt/" +Should we really use bcrypt/scrypt?: http://www.analyticalengine.net/2012/06/should-we-really-use-bcryptscrypt/" CWE-760,EN-Use of a One-Way Hash with a Predictable Salt (Type: Base),"The software uses a one-way cryptographic hash against an input that should not be reversible, such as a password, but the software uses a predictable salt as part of the input. This makes it easier for attackers to pre-compute the hash value using dictionary attack techniques such as rainbow tables, effectively disabling the protection that an unpredictable salt would provide. It should be noted that, despite common perceptions, the use of a good salt with a hash does not sufficiently increase the effort for an attacker who is targeting an individual password, or who has a large amount of computing resources available, such as with cloud-based services or specialized, inexpensive hardware. Offline password cracking can still be effective if the hash function is not expensive to compute; many cryptographic functions are designed to be efficient and can be vulnerable to attacks using massive computing resources, even if the hash is cryptographically strong. The use of a salt only slightly increases the computing requirements for an attacker compared to other strategies such as adaptive hash functions. See CWE-916 for more details.",,unclassified,"bcrypt: http://bcrypt.sourceforge.net/ @@ -1897,28 +1897,28 @@ Writing Secure Code: Chapter 9, ""Creating a Salted Hash"" Page 302 The Art of Software Security Assessment: Chapter 2, ""Salt Values"", Page 46. How To Safely Store A Password: http://codahale.com/how-to-safely-store-a-password/ Our password hashing has no clothes: http://www.troyhunt.com/2012/06/our-password-hashing-has-no-clothes.html -Should we really use bcrypt/scrypt?: http://www.analyticalengine.net/2012/06/should-we-really-use-bcryptscrypt/" +Should we really use bcrypt/scrypt?: http://www.analyticalengine.net/2012/06/should-we-really-use-bcryptscrypt/" CWE-761,EN-Free of Pointer not at Start of Buffer (Type: Variant),"The application calls free() on a pointer to a memory resource that was allocated on the heap, but the pointer is not at the start of the buffer. This can cause the application to crash, or in some cases, modify critical program variables or execute code. This weakness often occurs when the memory is allocated explicitly on the heap with one of the malloc() family functions and free() is called, but pointer arithmetic has caused the pointer to be in the interior or end of the buffer.",,unclassified,"boost C++ Library Smart Pointers: http://www.boost.org/doc/libs/1_38_0/libs/smart_ptr/smart_ptr.htm -Valgrind: http://valgrind.org/" +Valgrind: http://valgrind.org/" CWE-763,EN-Release of Invalid Pointer or Reference (Type: Base),"The application attempts to return a memory resource to the system, but calls the wrong release function or calls the appropriate release function incorrectly. This weakness can take several forms, such as: The memory was allocated, explicitly or implicitly, via one memory management method and deallocated using a different, non-compatible function (CWE-762). The function calls or memory management routines chosen are appropriate, however they are used incorrectly, such as in CWE-761.",,unclassified,"boost C++ Library Smart Pointers: http://www.boost.org/doc/libs/1_38_0/libs/smart_ptr/smart_ptr.htm -Valgrind: http://valgrind.org/" +Valgrind: http://valgrind.org/" CWE-764,EN-Multiple Locks of a Critical Resource (Type: Variant),"The software locks a critical resource more times than intended, leading to an unexpected state in the system. -When software is operating in a concurrent environment and repeatedly locks a critical resource, the consequences will vary based on the type of lock, the lock's implementation, and the resource being protected. In some situations such as with semaphores, the resources are pooled and extra locking calls will reduce the size of the total available pool, possibly leading to degraded performance or a denial of service. If this can be triggered by an attacker, it will be similar to an unrestricted lock (CWE-412). In the context of a binary lock, it is likely that any duplicate locking attempts will never succeed since the lock is already held and progress may not be possible.",,unclassified, +When software is operating in a concurrent environment and repeatedly locks a critical resource, the consequences will vary based on the type of lock, the lock's implementation, and the resource being protected. In some situations such as with semaphores, the resources are pooled and extra locking calls will reduce the size of the total available pool, possibly leading to degraded performance or a denial of service. If this can be triggered by an attacker, it will be similar to an unrestricted lock (CWE-412). In the context of a binary lock, it is likely that any duplicate locking attempts will never succeed since the lock is already held and progress may not be possible.",,unclassified, CWE-765,EN-Multiple Unlocks of a Critical Resource (Type: Variant),"The software unlocks a critical resource more times than intended, leading to an unexpected state in the system. -When software is operating in a concurrent environment and repeatedly unlocks a critical resource, the consequences will vary based on the type of lock, the lock's implementation, and the resource being protected. In some situations such as with semaphores, the resources are pooled and extra calls to unlock will increase the count for the number of available resources, likely resulting in a crash or unpredictable behavior when the system nears capacity.",,unclassified, +When software is operating in a concurrent environment and repeatedly unlocks a critical resource, the consequences will vary based on the type of lock, the lock's implementation, and the resource being protected. In some situations such as with semaphores, the resources are pooled and extra calls to unlock will increase the count for the number of available resources, likely resulting in a crash or unpredictable behavior when the system nears capacity.",,unclassified, CWE-785,EN-Use of Path Manipulation Function without Maximum-sized Buffer (Type: Variant),"The software invokes a function for normalizing paths or file names, but it provides an output buffer that is smaller than the maximum possible size, such as PATH_MAX. -Passing an inadequately-sized output buffer to a path manipulation function can result in a buffer overflow. Such functions include realpath(), readlink(), PathAppend(), and others.",,unclassified, +Passing an inadequately-sized output buffer to a path manipulation function can result in a buffer overflow. Such functions include realpath(), readlink(), PathAppend(), and others.",,unclassified, CWE-786,EN-Access of Memory Location Before Start of Buffer (Type: Base),"The software reads or writes to a buffer using an index or pointer that references a memory location prior to the beginning of the buffer. -This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used.",,unclassified, +This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used.",,unclassified, CWE-787,EN-Out-of-bounds Write (Type: Base),"The software writes data past the end, or before the beginning, of the intended buffer. -This typically occurs when the pointer or its index is incremented or decremented to a position beyond the bounds of the buffer or when pointer arithmetic results in a position outside of the valid memory location to name a few. This may result in corruption of sensitive information, a crash, or code execution among other things.",,unclassified, +This typically occurs when the pointer or its index is incremented or decremented to a position beyond the bounds of the buffer or when pointer arithmetic results in a position outside of the valid memory location to name a few. This may result in corruption of sensitive information, a crash, or code execution among other things.",,unclassified, CWE-788,EN-Access of Memory Location After End of Buffer (Type: Base),"The software reads or writes to a buffer using an index or pointer that references a memory location after the end of the buffer. -This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. These problems may be resultant from missing sentinel values (CWE-463) or trusting a user-influenced input length variable.",,unclassified, +This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. These problems may be resultant from missing sentinel values (CWE-463) or trusting a user-influenced input length variable.",,unclassified, CWE-790,EN-Improper Filtering of Special Elements (Type: Class),"The software receives data from an upstream component, but does not filter or incorrectly filters special elements before sending it to a downstream component. Cross-site scripting (XSS) vulnerabilities occur when: 1. Untrusted data enters a web application, typically from a web request. @@ -1932,7 +1932,7 @@ The server reads data directly from the HTTP request and reflects it back in the The application stores dangerous data in a database, message forum, visitor log, or other trusted data store. At a later time, the dangerous data is subsequently read back into the application and included in dynamic content. From an attacker's perspective, the optimal place to inject malicious content is in an area that is displayed to either many users or particularly interesting users. Interesting users typically have elevated privileges in the application or interact with sensitive data that is valuable to the attacker. If one of these users executes malicious content, the attacker may be able to perform privileged operations on behalf of the user or gain access to sensitive data belonging to the user. For example, the attacker might inject XSS into a log message, which might not be handled properly when an administrator views the logs. In DOM-based XSS, the client performs the injection of XSS into the page; in the other types, the server performs the injection. DOM-based XSS generally involves server-controlled, trusted script that is sent to the client, such as Javascript that performs sanity checks on a form before the user submits it. If the server-supplied script processes user-supplied data and then injects it back into the web page (such as with dynamic HTML), then DOM-based XSS is possible. Once the malicious script is injected, the attacker can perform a variety of malicious activities. The attacker could transfer private information, such as cookies that may include session information, from the victim's machine to the attacker. The attacker could send malicious requests to a web site on behalf of the victim, which could be especially dangerous to the site if the victim has administrator privileges to manage that site. Phishing attacks could be used to emulate trusted web sites and trick the victim into entering a password, allowing the attacker to compromise the victim's account on that web site. Finally, the script could exploit a vulnerability in the web browser itself possibly taking over the victim's machine, sometimes referred to as ""drive-by hacking."" -In many cases, the attack can be launched without the victim even being aware of it. Even with careful users, attackers frequently use a variety of methods to encode the malicious portion of the attack, such as URL encoding or Unicode, so the request looks less suspicious.",,unclassified, +In many cases, the attack can be launched without the victim even being aware of it. Even with careful users, attackers frequently use a variety of methods to encode the malicious portion of the attack, such as URL encoding or Unicode, so the request looks less suspicious.",,unclassified, CWE-791,EN-Incomplete Filtering of Special Elements (Type: Base),"The software receives data from an upstream component, but does not completely filter special elements before sending it to a downstream component. Cross-site scripting (XSS) vulnerabilities occur when: 1. Untrusted data enters a web application, typically from a web request. @@ -1946,66 +1946,66 @@ The server reads data directly from the HTTP request and reflects it back in the The application stores dangerous data in a database, message forum, visitor log, or other trusted data store. At a later time, the dangerous data is subsequently read back into the application and included in dynamic content. From an attacker's perspective, the optimal place to inject malicious content is in an area that is displayed to either many users or particularly interesting users. Interesting users typically have elevated privileges in the application or interact with sensitive data that is valuable to the attacker. If one of these users executes malicious content, the attacker may be able to perform privileged operations on behalf of the user or gain access to sensitive data belonging to the user. For example, the attacker might inject XSS into a log message, which might not be handled properly when an administrator views the logs. In DOM-based XSS, the client performs the injection of XSS into the page; in the other types, the server performs the injection. DOM-based XSS generally involves server-controlled, trusted script that is sent to the client, such as Javascript that performs sanity checks on a form before the user submits it. If the server-supplied script processes user-supplied data and then injects it back into the web page (such as with dynamic HTML), then DOM-based XSS is possible. Once the malicious script is injected, the attacker can perform a variety of malicious activities. The attacker could transfer private information, such as cookies that may include session information, from the victim's machine to the attacker. The attacker could send malicious requests to a web site on behalf of the victim, which could be especially dangerous to the site if the victim has administrator privileges to manage that site. Phishing attacks could be used to emulate trusted web sites and trick the victim into entering a password, allowing the attacker to compromise the victim's account on that web site. Finally, the script could exploit a vulnerability in the web browser itself possibly taking over the victim's machine, sometimes referred to as ""drive-by hacking."" -In many cases, the attack can be launched without the victim even being aware of it. Even with careful users, attackers frequently use a variety of methods to encode the malicious portion of the attack, such as URL encoding or Unicode, so the request looks less suspicious.",,unclassified, +In many cases, the attack can be launched without the victim even being aware of it. Even with careful users, attackers frequently use a variety of methods to encode the malicious portion of the attack, such as URL encoding or Unicode, so the request looks less suspicious.",,unclassified, CWE-792,EN-Incomplete Filtering of One or More Instances of Special Elements (Type: Variant),"The software receives data from an upstream component, but does not completely filter one or more instances of special elements before sending it to a downstream component. Incomplete filtering of this nature involves either only filtering a single instance of a special element when more exist, or -not filtering all instances or all elements where multiple special elements exist.",,unclassified, +not filtering all instances or all elements where multiple special elements exist.",,unclassified, CWE-793,EN-Only Filtering One Instance of a Special Element (Type: Variant),"The software receives data from an upstream component, but only filters a single instance of a special element before sending it to a downstream component. -Incomplete filtering of this nature may be location-dependent, as in only the first or last element is filtered.",,unclassified, +Incomplete filtering of this nature may be location-dependent, as in only the first or last element is filtered.",,unclassified, CWE-794,EN-Incomplete Filtering of Multiple Instances of Special Elements (Type: Variant),"The software receives data from an upstream component, but does not filter all instances of a special element before sending it to a downstream component. Incomplete filtering of this nature may be applied to sequential elements (special elements that appear next to each other) or -non-sequential elements (special elements that appear multiple times in different locations).",,unclassified, +non-sequential elements (special elements that appear multiple times in different locations).",,unclassified, CWE-795,EN-Only Filtering Special Elements at a Specified Location (Type: Base),"The software receives data from an upstream component, but only accounts for special elements at a specified location, thereby missing remaining special elements that may exist before sending it to a downstream component. A filter might only account for instances of special elements when they occur: relative to a marker (e.g. ""at the beginning/end of string; the second argument""), or at an absolute position (e.g. ""byte number 10""). -This may leave special elements in the data that did not match the filter position, but still may be dangerous.",,unclassified, +This may leave special elements in the data that did not match the filter position, but still may be dangerous.",,unclassified, CWE-796,EN-Only Filtering Special Elements Relative to a Marker (Type: Variant),"The software receives data from an upstream component, but only accounts for special elements positioned relative to a marker (e.g. ""at the beginning/end of a string; the second argument""), thereby missing remaining special elements that may exist before sending it to a downstream component. A filter might only account for instances of special elements when they occur: relative to a marker (e.g. ""at the beginning/end of string; the second argument""), or at an absolute position (e.g. ""byte number 10""). -This may leave special elements in the data that did not match the filter position, but still may be dangerous.",,unclassified, +This may leave special elements in the data that did not match the filter position, but still may be dangerous.",,unclassified, CWE-797,EN-Only Filtering Special Elements at an Absolute Position (Type: Variant),"The software receives data from an upstream component, but only accounts for special elements at an absolute position (e.g. ""byte number 10""), thereby missing remaining special elements that may exist before sending it to a downstream component. A filter might only account for instances of special elements when they occur: relative to a marker (e.g. ""at the beginning/end of string; the second argument""), or at an absolute position (e.g. ""byte number 10""). -This may leave special elements in the data that did not match the filter position, but still may be dangerous.",,unclassified, +This may leave special elements in the data that did not match the filter position, but still may be dangerous.",,unclassified, CWE-799,EN-Improper Control of Interaction Frequency (Type: Class),"The software does not properly limit the number or frequency of interactions that it has with an actor, such as the number of incoming requests. -This can allow the actor to perform actions more frequently than expected. The actor could be a human or an automated process such as a virus or bot. This could be used to cause a denial of service, compromise program logic (such as limiting humans to a single vote), or other consequences. For example, an authentication routine might not limit the number of times an attacker can guess a password. Or, a web site might conduct a poll but only expect humans to vote a maximum of once a day.",,unclassified,Insufficient Anti-automation: http://projects.webappsec.org/Insufficient+Anti-automation +This can allow the actor to perform actions more frequently than expected. The actor could be a human or an automated process such as a virus or bot. This could be used to cause a denial of service, compromise program logic (such as limiting humans to a single vote), or other consequences. For example, an authentication routine might not limit the number of times an attacker can guess a password. Or, a web site might conduct a poll but only expect humans to vote a maximum of once a day.",,unclassified,Insufficient Anti-automation: http://projects.webappsec.org/Insufficient+Anti-automation CWE-8,EN-J2EE Misconfiguration: Entity Bean Declared Remote (Type: Variant),"When an application exposes a remote interface for an entity bean, it might also expose methods that get or set the bean's data. These methods could be leveraged to read sensitive information, or to change data in ways that violate the application's expectations, potentially leading to other vulnerabilities. -This can allow the actor to perform actions more frequently than expected. The actor could be a human or an automated process such as a virus or bot. This could be used to cause a denial of service, compromise program logic (such as limiting humans to a single vote), or other consequences. For example, an authentication routine might not limit the number of times an attacker can guess a password. Or, a web site might conduct a poll but only expect humans to vote a maximum of once a day.",,unclassified, +This can allow the actor to perform actions more frequently than expected. The actor could be a human or an automated process such as a virus or bot. This could be used to cause a denial of service, compromise program logic (such as limiting humans to a single vote), or other consequences. For example, an authentication routine might not limit the number of times an attacker can guess a password. Or, a web site might conduct a poll but only expect humans to vote a maximum of once a day.",,unclassified, CWE-81,EN-Improper Neutralization of Script in an Error Message Web Page (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special characters that could be interpreted as web-scripting elements when they are sent to an error page. Error pages may include customized 403 Forbidden or 404 Not Found pages. -When an attacker can trigger an error that contains unneutralized input, then cross-site scripting attacks may be possible.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183" +When an attacker can trigger an error that contains unneutralized input, then cross-site scripting attacks may be possible.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183" CWE-82,EN-Improper Neutralization of Script in Attributes of IMG Tags in a Web Page (Type: Variant),"The web application does not neutralize or incorrectly neutralizes scripting elements within attributes of HTML IMG tags, such as the src attribute. -Attackers can embed XSS exploits into the values for IMG attributes (e.g. SRC) that is streamed and then executed in a victim's browser. Note that when the page is loaded into a user's browsers, the exploit will automatically execute.",,unclassified, +Attackers can embed XSS exploits into the values for IMG attributes (e.g. SRC) that is streamed and then executed in a victim's browser. Note that when the page is loaded into a user's browsers, the exploit will automatically execute.",,unclassified, CWE-820,EN-Missing Synchronization (Type: Base),"The software utilizes a shared resource in a concurrent manner but does not attempt to synchronize access to the resource. -If access to a shared resource is not synchronized, then the resource may not be in a state that is expected by the software. This might lead to unexpected or insecure behaviors, especially if an attacker can influence the shared resource.",,unclassified, +If access to a shared resource is not synchronized, then the resource may not be in a state that is expected by the software. This might lead to unexpected or insecure behaviors, especially if an attacker can influence the shared resource.",,unclassified, CWE-821,EN-Incorrect Synchronization (Type: Base),"The software utilizes a shared resource in a concurrent manner but it does not correctly synchronize access to the resource. -If access to a shared resource is not correctly synchronized, then the resource may not be in a state that is expected by the software. This might lead to unexpected or insecure behaviors, especially if an attacker can influence the shared resource.",,unclassified, +If access to a shared resource is not correctly synchronized, then the resource may not be in a state that is expected by the software. This might lead to unexpected or insecure behaviors, especially if an attacker can influence the shared resource.",,unclassified, CWE-822,EN-Untrusted Pointer Dereference (Type: Base),"The program obtains a value from an untrusted source, converts this value to a pointer, and dereferences the resulting pointer. An attacker can supply a pointer for memory locations that the program is not expecting. If the pointer is dereferenced for a write operation, the attack might allow modification of critical program state variables, cause a crash, or execute code. If the dereferencing operation is for a read, then the attack might allow reading of sensitive data, cause a crash, or set a program variable to an unexpected value (since the value will be read from an unexpected memory location). There are several variants of this weakness, including but not necessarily limited to: The untrusted value is directly invoked as a function call. In OS kernels or drivers where there is a boundary between ""userland"" and privileged memory spaces, an untrusted pointer might enter through an API or system call (see CWE-781 for one such example). -Inadvertently accepting the value from an untrusted control sphere when it did not have to be accepted as input at all. This might occur when the code was originally developed to be run by a single user in a non-networked environment, and the code is then ported to or otherwise exposed to a networked environment.",,unclassified, +Inadvertently accepting the value from an untrusted control sphere when it did not have to be accepted as input at all. This might occur when the code was originally developed to be run by a single user in a non-networked environment, and the code is then ported to or otherwise exposed to a networked environment.",,unclassified, CWE-823,EN-Use of Out-of-range Pointer Offset (Type: Base),"The program performs pointer arithmetic on a valid pointer, but it uses an offset that can point outside of the intended range of valid memory locations for the resulting pointer. While a pointer can contain a reference to any arbitrary memory location, a program typically only intends to use the pointer to access limited portions of memory, such as contiguous memory used to access an individual array. Programs may use offsets in order to access fields or sub-elements stored within structured data. The offset might be out-of-range if it comes from an untrusted source, is the result of an incorrect calculation, or occurs because of another error. -If an attacker can control or influence the offset so that it points outside of the intended boundaries of the structure, then the attacker may be able to read or write to memory locations that are used elsewhere in the program. As a result, the attack might change the state of the software as accessed through program variables, cause a crash or instable behavior, and possibly lead to code execution.",,unclassified,"The Art of Software Security Assessment: Chapter 6, ""Pointer Arithmetic"", Page 277." +If an attacker can control or influence the offset so that it points outside of the intended boundaries of the structure, then the attacker may be able to read or write to memory locations that are used elsewhere in the program. As a result, the attack might change the state of the software as accessed through program variables, cause a crash or instable behavior, and possibly lead to code execution.",,unclassified,"The Art of Software Security Assessment: Chapter 6, ""Pointer Arithmetic"", Page 277." CWE-824,EN-Access of Uninitialized Pointer (Type: Base),"The program accesses or uses a pointer that has not been initialized. If the pointer contains an uninitialized value, then the value might not point to a valid memory location. This could cause the program to read from or write to unexpected memory locations, leading to a denial of service. If the uninitialized pointer is used as a function call, then arbitrary functions could be invoked. If an attacker can influence the portion of uninitialized memory that is contained in the pointer, this weakness could be leveraged to execute code or perform other attacks. -Depending on memory layout, associated memory management behaviors, and program operation, the attacker might be able to influence the contents of the uninitialized pointer, thus gaining more fine-grained control of the memory location to be accessed.",,unclassified,"The Art of Software Security Assessment: Chapter 7, ""Variable Initialization"", Page 312." +Depending on memory layout, associated memory management behaviors, and program operation, the attacker might be able to influence the contents of the uninitialized pointer, thus gaining more fine-grained control of the memory location to be accessed.",,unclassified,"The Art of Software Security Assessment: Chapter 7, ""Variable Initialization"", Page 312." CWE-825,EN-Expired Pointer Dereference (Type: Base),"The program dereferences a pointer that contains a location for memory that was previously valid, but is no longer valid. -When a program releases memory, but it maintains a pointer to that memory, then the memory might be re-allocated at a later time. If the original pointer is accessed to read or write data, then this could cause the program to read or modify data that is in use by a different function or process. Depending on how the newly-allocated memory is used, this could lead to a denial of service, information exposure, or code execution.",,unclassified, +When a program releases memory, but it maintains a pointer to that memory, then the memory might be re-allocated at a later time. If the original pointer is accessed to read or write data, then this could cause the program to read or modify data that is in use by a different function or process. Depending on how the newly-allocated memory is used, this could lead to a denial of service, information exposure, or code execution.",,unclassified, CWE-826,EN-Premature Release of Resource During Expected Lifetime (Type: Base),"The program releases a resource that is still intended to be used by the program itself or another actor. This weakness focuses on errors in which the program should not release a resource, but performs the release anyway. This is different than a weakness in which the program releases a resource at the appropriate time, but it maintains a reference to the resource, which it later accesses. For this weaknesses, the resource should still be valid upon the subsequent access. -When a program releases a resource that is still being used, it is possible that operations will still be taken on this resource, which may have been repurposed in the meantime, leading to issues similar to CWE-825. Consequences may include denial of service, information exposure, or code execution.",,unclassified, +When a program releases a resource that is still being used, it is possible that operations will still be taken on this resource, which may have been repurposed in the meantime, leading to issues similar to CWE-825. Consequences may include denial of service, information exposure, or code execution.",,unclassified, CWE-827,EN-Improper Control of Document Type Definition (Type: Base),"The software does not restrict a reference to a Document Type Definition (DTD) to the intended control sphere. This might allow attackers to reference arbitrary DTDs, possibly causing the software to expose files, consume excessive system resources, or execute arbitrary http requests on behalf of the attacker. As DTDs are processed, they might try to read or include files on the machine performing the parsing. If an attacker is able to control the DTD, then the attacker might be able to specify sensitive resources or requests or provide malicious content. -For example, the SOAP specification prohibits SOAP messages from containing DTDs.",,unclassified,Apache CXF Security Advisory (CVE-2010-2076): http://svn.apache.org/repos/asf/cxf/trunk/security/CVE-2010-2076.pdf +For example, the SOAP specification prohibits SOAP messages from containing DTDs.",,unclassified,Apache CXF Security Advisory (CVE-2010-2076): http://svn.apache.org/repos/asf/cxf/trunk/security/CVE-2010-2076.pdf CWE-828,EN-Signal Handler with Functionality that is not Asynchronous-Safe (Type: Base),"The software defines a signal handler that contains code sequences that are not asynchronous-safe, i.e., the functionality is not reentrant, or it can be interrupted. This can lead to an unexpected system state with a variety of potential consequences depending on context, including denial of service and code execution. Signal handlers are typically intended to interrupt normal functionality of a program, or even other signals, in order to notify the process of an event. When a signal handler uses global or static variables, or invokes functions that ultimately depend on such state or its associated metadata, then it could corrupt system state that is being used by normal functionality. This could subject the program to race conditions or other weaknesses that allow an attacker to cause the program state to be corrupted. While denial of service is frequently the consequence, in some cases this weakness could be leveraged for code execution. @@ -2015,39 +2015,39 @@ Code sequences (not necessarily function calls) contain non-atomic use of global The signal handler function is intended to run at most one time, but instead it can be invoked multiple times. This could happen by repeated delivery of the same signal, or by delivery of different signals that have the same handler function (CWE-831). Note that in some environments or contexts, it might be possible for the signal handler to be interrupted itself. If both a signal handler and the normal behavior of the software have to operate on the same set of state variables, and a signal is received in the middle of the normal execution's modifications of those variables, the variables may be in an incorrect or corrupt state during signal handler execution, and possibly still incorrect or corrupt upon return.",,unclassified,"Delivering Signals for Fun and Profit: http://lcamtuf.coredump.cx/signals.txt -Race Condition: Signal Handling: http://www.fortify.com/vulncat/en/vulncat/cpp/race_condition_signal_handling.html" +Race Condition: Signal Handling: http://www.fortify.com/vulncat/en/vulncat/cpp/race_condition_signal_handling.html" CWE-829,EN-Inclusion of Functionality from Untrusted Control Sphere (Type: Class),"The software imports, requires, or includes executable functionality (such as a library) from a source that is outside of the intended control sphere. When including third-party functionality, such as a web widget, library, or other source of functionality, the software must effectively trust that functionality. Without sufficient protection mechanisms, the functionality could be malicious in nature (either by coming from an untrusted source, being spoofed, or being modified in transit from a trusted source). The functionality might also contain its own weaknesses, or grant access to additional functionality and state information that should be kept private to the base system, such as system state information, sensitive application data, or the DOM of a web application. This might lead to many different consequences depending on the included functionality, but some examples include injection of malware, information exposure by granting excessive privileges or permissions to the untrusted functionality, DOM-based XSS vulnerabilities, stealing user's cookies, or open redirect to malware (CWE-601).",,unclassified,"OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI -Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html" +Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html" CWE-83,EN-Improper Neutralization of Script in Attributes in a Web Page (Type: Variant),"The software does not neutralize or incorrectly neutralizes ""javascript:"" or other URIs from dangerous attributes within tags, such as onmouseover, onload, onerror, or style. When including third-party functionality, such as a web widget, library, or other source of functionality, the software must effectively trust that functionality. Without sufficient protection mechanisms, the functionality could be malicious in nature (either by coming from an untrusted source, being spoofed, or being modified in transit from a trusted source). The functionality might also contain its own weaknesses, or grant access to additional functionality and state information that should be kept private to the base system, such as system state information, sensitive application data, or the DOM of a web application. -This might lead to many different consequences depending on the included functionality, but some examples include injection of malware, information exposure by granting excessive privileges or permissions to the untrusted functionality, DOM-based XSS vulnerabilities, stealing user's cookies, or open redirect to malware (CWE-601).",,unclassified, +This might lead to many different consequences depending on the included functionality, but some examples include injection of malware, information exposure by granting excessive privileges or permissions to the untrusted functionality, DOM-based XSS vulnerabilities, stealing user's cookies, or open redirect to malware (CWE-601).",,unclassified, CWE-830,EN-Inclusion of Web Functionality from an Untrusted Source (Type: Base),"The software includes web functionality (such as a web widget) from another domain, which causes it to operate within the domain of the software, potentially granting total access and control of the software to the untrusted source. Including third party functionality in a web-based environment is risky, especially if the source of the functionality is untrusted. Even if the third party is a trusted source, the software may still be exposed to attacks and malicious behavior if that trusted source is compromised, or if the code is modified in transmission from the third party to the software. This weakness is common in ""mashup"" development on the web, which may include source functionality from other domains. For example, Javascript-based web widgets may be inserted by using ' + diff --git a/server/www/scripts/commons/directives/fileModel.js b/server/www/scripts/commons/directives/fileModel.js new file mode 100644 index 00000000000..51cb9f0ad27 --- /dev/null +++ b/server/www/scripts/commons/directives/fileModel.js @@ -0,0 +1,19 @@ +// Faraday Penetration Test IDE +// Copyright (C) 2018 Infobyte LLC (http://www.infobytesec.com/) +// See the file 'doc/LICENSE' for the license information + +angular.module('faradayApp').directive('fileModel', ['$parse', function ($parse) { + return { + restrict: 'A', + link: function(scope, element, attrs) { + var model = $parse(attrs.fileModel); + var modelSetter = model.assign; + + element.bind('change', function(){ + scope.$apply(function(){ + modelSetter(scope, element[0].files[0]); + }); + }); + } + }; +}]); diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 62cf0149b0a..f60b7b7a577 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -8,11 +8,11 @@ angular.module("faradayApp") "$location", "$uibModal", "$cookies", "$q", "$window", "BASEURL", "SEVERITIES", "EASEOFRESOLUTION", "STATUSES", "hostsManager", "commonsFact", "vulnsManager", "workspacesFact", "csvService", "uiGridConstants", "vulnModelsManager", - "referenceFact", "ServerAPI", + "referenceFact", "ServerAPI", '$http', function($scope, $filter, $routeParams, $location, $uibModal, $cookies, $q, $window, BASEURL, SEVERITIES, EASEOFRESOLUTION, STATUSES, hostsManager, commonsFact, - vulnsManager, workspacesFact, csvService, uiGridConstants, vulnModelsManager, referenceFact, ServerAPI) { + vulnsManager, workspacesFact, csvService, uiGridConstants, vulnModelsManager, referenceFact, ServerAPI, $http) { $scope.baseurl; $scope.columns; $scope.columnsWidths; @@ -1091,6 +1091,47 @@ angular.module("faradayApp") return $scope.encodeUrl(srvName); }; + function toggleFileUpload() { + if($scope.fileUploadEnabled === false) { + $scope.fileUploadEnabled = true; + } else { + $scope.fileUploadEnabled = false; + $scope.fileToUpload = undefined; + } + } + + $scope.enableFileUpload = function() { + if($scope.fileUploadEnabled === undefined) { + $http.get('/_api/session').then( + function(d) { + $scope.csrf_token = d.data.csrf_token; + $scope.fileUploadEnabled = true; + } + ); + } else { + toggleFileUpload(); + } + }; + + $scope.uploadFile = function() { + var fd = new FormData(); + fd.append('csrf_token', $scope.csrf_token); + fd.append('file', $scope.fileToUpload); + $http.post('http://localhost:5985/_api/v2/ws/' + $scope.workspace + '/upload_report', fd, { + transformRequest: angular.identity, + withCredentials: false, + headers: {'Content-Type': undefined}, + responseType: "arraybuffer", + params: { + fd + } + }).then( + function(d) { + $window.location.reload(); + } + ); + }; + $scope.concatForTooltip = function (items, isArray, useDoubleLinebreak) { var elements = []; for (var property in items) { diff --git a/server/www/scripts/statusReport/partials/statusReport.html b/server/www/scripts/statusReport/partials/statusReport.html index 46cf1b23dff..6f8b98f1f75 100644 --- a/server/www/scripts/statusReport/partials/statusReport.html +++ b/server/www/scripts/statusReport/partials/statusReport.html @@ -14,7 +14,7 @@
    -
    +
    + +
    +
    + + + + + + +

    {{fileToUpload.name}}

    +     +
    From d3ea9984b445ab96c8f7f75553793b5ed1560e6f Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Mon, 11 Jun 2018 12:41:01 -0300 Subject: [PATCH 1309/1506] [ADD] .gitignore to migrations/versions/ to add folder in pulls --- migrations/versions/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 migrations/versions/.gitignore diff --git a/migrations/versions/.gitignore b/migrations/versions/.gitignore new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/migrations/versions/.gitignore @@ -0,0 +1 @@ + From 57ecb1c8edd768bc0bb6beedfc7f73cb602650b8 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 11 Jun 2018 13:07:05 -0300 Subject: [PATCH 1310/1506] [FIX] Change jumbotron background color --- server/www/estilos-v3.css | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/www/estilos-v3.css b/server/www/estilos-v3.css index f69651edd42..c7eda195dd5 100644 --- a/server/www/estilos-v3.css +++ b/server/www/estilos-v3.css @@ -214,3 +214,6 @@ .file-selected { z-index: -999; } +.jumbotron { + background-color: #f4f3f4; +} From 79ba0c479d2db2c943797a51b039351c8c4c67e9 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 11 Jun 2018 15:43:14 -0300 Subject: [PATCH 1311/1506] [FIX] section width --- server/www/estilos.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/estilos.css b/server/www/estilos.css index 9e505591a53..a59957986c3 100644 --- a/server/www/estilos.css +++ b/server/www/estilos.css @@ -178,7 +178,7 @@ header.head { } .seccion { padding: 0px; - width: 101%; + width: 100%; float: left; } .seccion.treemap{ From 7b346bc2724d3fdfab8e7996a8c97babd87effde Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 11 Jun 2018 18:30:38 -0300 Subject: [PATCH 1312/1506] [MOD] Improvements on initdb workflow --- manage.py | 8 ++++++-- server/commands/initdb.py | 10 ++++++++-- server/importer.py | 3 ++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/manage.py b/manage.py index c3373ecc2b0..3e4989235c1 100755 --- a/manage.py +++ b/manage.py @@ -69,7 +69,11 @@ def faraday_schema_display(): def initdb(): with app.app_context(): InitDB().run() - ImportCouchDB().run() + couchdb_config_present = server.config.couchdb + if couchdb_config_present and couchdb_config_present.user and couchdb_config_present.password: + print('Importing data from CouchDB, please wait...') + ImportCouchDB().run() + print('All users from CouchDB were imported. You can login with your old username/password to faraday now.') @click.command() def import_from_couchdb(): @@ -123,7 +127,7 @@ def createsuperuser(username, email, password): email=email, password=password, role='admin', - is_ldap=False) + is_ldap=False) db.session.commit() click.echo(click.style( 'User {} created successfully!'.format(username), diff --git a/server/commands/initdb.py b/server/commands/initdb.py index 5d9cadb6653..7b986a4c825 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -34,6 +34,7 @@ from colorama import Fore from sqlalchemy.exc import OperationalError +import server.config from config.globals import CONST_FARADAY_HOME_PATH from server.config import LOCAL_CONFIG_FILE init() @@ -95,7 +96,11 @@ def run(self): current_psql_output.close() conn_string = self._save_config(config, username, password, database_name, hostname) self._create_tables(conn_string) - self._create_admin_user(conn_string) + couchdb_config_present = server.config.couchdb + if not (couchdb_config_present and couchdb_config_present.user and couchdb_config_present.password): + self._create_admin_user(conn_string) + else: + print('Skipping new admin creation since couchdb configuration was found.') except KeyboardInterrupt: current_psql_output.close() print('User cancelled.') @@ -156,7 +161,7 @@ def _configure_new_postgres_user(self, psql_log_file): we return username and password and those values will be saved in the config file. """ print('This script will {blue} create a new postgres user {white} and {blue} save faraday-server settings {white}(server.ini). '.format(blue=Fore.BLUE, white=Fore.WHITE)) - username = 'faraday' + username = 'faraday_postgresql' postgres_command = ['sudo', '-u', 'postgres'] if sys.platform == 'darwin': postgres_command = [] @@ -172,6 +177,7 @@ def _configure_new_postgres_user(self, psql_log_file): print("{yellow}WARNING{white}: Role {username} already exists, skipping creation ".format(yellow=Fore.YELLOW, white=Fore.WHITE, username=username)) try: + password = server.config.database.connection_string.split(':')[2].split('@')[0] connection = psycopg2.connect(dbname='postgres', user=username, password=password) diff --git a/server/importer.py b/server/importer.py index 330dc82c76c..024a8193883 100644 --- a/server/importer.py +++ b/server/importer.py @@ -81,7 +81,7 @@ server.config.CONSTANTS.CONST_FARADAY_LOGS_PATH, 'couchdb-importer.log')) importer_file_handler = logging.FileHandler(importer_logfile) formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + '%(asctime)s - %(name)s:%(lineno)d - %(levelname)s - %(message)s') importer_file_handler.setFormatter(formatter) importer_file_handler.setLevel(logging.DEBUG) logger.addHandler(importer_file_handler) @@ -1327,6 +1327,7 @@ def _open_couchdb_conn(self): logger.error(u"CouchDB is not running at {}. Check faraday-server's"\ " configuration and make sure CouchDB is running".format( server.couchdb.get_couchdb_url())) + logger.error(u'Please start CouchDB and re-execute the importer with: \n\n --> python manage.py import_from_couchdb <--') sys.exit(1) except Unauthorized: From 92f022d5813f0ff5bf2628194b73ac669da85d0b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 11 Jun 2018 18:47:44 -0300 Subject: [PATCH 1313/1506] [FIX] allow user.xml to be backwards compatible, avoid to delete couch config --- config/configuration.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/config/configuration.py b/config/configuration.py index 3e686fb391d..4ea6d4f5ff7 100644 --- a/config/configuration.py +++ b/config/configuration.py @@ -42,6 +42,9 @@ CONST_API_URL = "api_url" CONST_API_USERNAME = "api_username" CONST_API_PASSWORD = "api_password" +CONST_COUCH_URI = "couch_uri" +CONST_COUCH_REPLICS = "couch_replics" +CONST_COUCH_ISREPLICATED = "couch_is_replicated" CONST_REPO_URL = "repo_url" CONST_REPO_USER = "repo_user" CONST_REPORT_PATH = "report_path" @@ -149,6 +152,9 @@ def _getConfig(self): self._api_url = self._getValue(tree, CONST_API_URL) self._api_username = self._getValue(tree, CONST_API_USERNAME) self._api_password = self._getValue(tree, CONST_API_PASSWORD) + self._couch_uri = self._getValue(tree, CONST_COUCH_URI, default = "") + self._couch_replics = self._getValue(tree, CONST_COUCH_REPLICS, default = "") + self._couch_is_replicated = bool(self._getValue(tree, CONST_COUCH_ISREPLICATED, default = False)) self._repo_url = self._getValue(tree, CONST_REPO_URL) self._repo_user = self._getValue(tree, CONST_REPO_USER) self._report_path = self._getValue(tree, CONST_REPORT_PATH) @@ -305,6 +311,18 @@ def getAPIUsername(self): def getAPIPassword(self): return self._api_password + def getCouchURI(self): + if self._couch_uri and self._couch_uri.endswith('/'): + return self._couch_uri[:-1] + else: + return self._couch_uri + + def getCouchReplics(self): + return self._couch_replics + + def getCouchIsReplicated(self): + return self._couch_is_replicated + def setLastWorkspace(self, workspaceName): self._last_workspace = workspaceName @@ -414,6 +432,15 @@ def setAPIUsername(self, username): def setAPIPassword(self, password): self._api_password = password + def setCouchUri(self, uri): + self._couch_uri = uri + + def setCouchIsReplicated(self, is_it): + self._couch_is_replicated = is_it + + def setCouchReplics(self, urls): + self._couch_replics = urls + def setPluginSettings(self, settings): self._plugin_settings = settings @@ -570,6 +597,18 @@ def saveConfig(self, xml_file="~/.faraday/config/user.xml"): SERVER_PASSWORD.text = self.getAPIPassword() ROOT.append(SERVER_PASSWORD) + COUCH_URI = Element(CONST_COUCH_URI) + COUCH_URI.text = self.getCouchURI() + ROOT.append(COUCH_URI) + + COUCH_IS_REPLICATED = Element(CONST_COUCH_ISREPLICATED) + COUCH_IS_REPLICATED.text = str(self.getCouchIsReplicated()) + ROOT.append(COUCH_IS_REPLICATED) + + COUCH_REPLICS = Element(CONST_COUCH_REPLICS) + COUCH_REPLICS.text = self.getCouchReplics() + ROOT.append(COUCH_REPLICS) + VERSION = Element(CONST_VERSION) VERSION.text = self.getVersion() ROOT.append(VERSION) From 53d3c108965b0867463920f355f3b2d63f791ab0 Mon Sep 17 00:00:00 2001 From: WinnaZ Date: Mon, 11 Jun 2018 20:12:21 -0300 Subject: [PATCH 1314/1506] renamed old fplugins --- bin/{delAllHost.py => del_all_hosts.py} | 0 bin/{delAllServiceClosed.py => del_all_service_closed.py} | 0 bin/{delAllVulnsWith.py => del_all_vulns_with.py} | 0 bin/{getAllIps.py => get_all_ips.py} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename bin/{delAllHost.py => del_all_hosts.py} (100%) rename bin/{delAllServiceClosed.py => del_all_service_closed.py} (100%) rename bin/{delAllVulnsWith.py => del_all_vulns_with.py} (100%) rename bin/{getAllIps.py => get_all_ips.py} (100%) diff --git a/bin/delAllHost.py b/bin/del_all_hosts.py similarity index 100% rename from bin/delAllHost.py rename to bin/del_all_hosts.py diff --git a/bin/delAllServiceClosed.py b/bin/del_all_service_closed.py similarity index 100% rename from bin/delAllServiceClosed.py rename to bin/del_all_service_closed.py diff --git a/bin/delAllVulnsWith.py b/bin/del_all_vulns_with.py similarity index 100% rename from bin/delAllVulnsWith.py rename to bin/del_all_vulns_with.py diff --git a/bin/getAllIps.py b/bin/get_all_ips.py similarity index 100% rename from bin/getAllIps.py rename to bin/get_all_ips.py From fd20c2595b75a24cf25a5d1a623af18e7194ffef Mon Sep 17 00:00:00 2001 From: WinnaZ Date: Mon, 11 Jun 2018 20:23:45 -0300 Subject: [PATCH 1315/1506] one more rename --- bin/{del_all_service_closed.py => del_all_services_closed.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename bin/{del_all_service_closed.py => del_all_services_closed.py} (100%) diff --git a/bin/del_all_service_closed.py b/bin/del_all_services_closed.py similarity index 100% rename from bin/del_all_service_closed.py rename to bin/del_all_services_closed.py From 5d18c161ed40553c3dd9956c43747cc87590bcce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 12 Jun 2018 13:17:25 -0300 Subject: [PATCH 1316/1506] Add more views to the list of views with no workspace viewer Some views don't depend on a workspace but they were showing the workspace switcher, and the edit button --- server/www/scripts/commons/controllers/headerCtrl.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/www/scripts/commons/controllers/headerCtrl.js b/server/www/scripts/commons/controllers/headerCtrl.js index 58efe365355..244e799b6cf 100644 --- a/server/www/scripts/commons/controllers/headerCtrl.js +++ b/server/www/scripts/commons/controllers/headerCtrl.js @@ -9,7 +9,11 @@ angular.module('faradayApp') $scope.confirmed = ($cookies.get('confirmed') == undefined) ? false : JSON.parse($cookies.get('confirmed')); $scope.showSwitcher = function() { - var noSwitcher = ["", "home", "login", "index", "workspaces", "users", "licenses"]; + var noSwitcher = [ + "", "home", "login", "index", "workspaces", "users", "licenses", + "taskgroup", "executive", // Only for white versions!! + "vulndb", "comparison", "webshell" + ]; return noSwitcher.indexOf($scope.component) < 0; }; From 083c0731fa63d6e01e703ca0c9325e611699cc14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 12 Jun 2018 13:19:00 -0300 Subject: [PATCH 1317/1506] Only show edit workspace button in workspaced views --- server/www/scripts/commons/partials/header.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/commons/partials/header.html b/server/www/scripts/commons/partials/header.html index afe275f6c8b..8b18ab5e490 100644 --- a/server/www/scripts/commons/partials/header.html +++ b/server/www/scripts/commons/partials/header.html @@ -51,7 +51,7 @@ - From 8c34793ae2c520f66163bb02ac36442b1101771f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 12 Jun 2018 13:24:25 -0300 Subject: [PATCH 1318/1506] Add header to commercial version views --- server/www/scripts/commons/partials/commercial.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/www/scripts/commons/partials/commercial.html b/server/www/scripts/commons/partials/commercial.html index 19510e7b0b0..9db98269e6b 100644 --- a/server/www/scripts/commons/partials/commercial.html +++ b/server/www/scripts/commons/partials/commercial.html @@ -4,7 +4,9 @@
    -
    +
    +
    +

    Welcome to the {{header}} panel!

    This feature belongs to our commercial versions

    From fa5f5ee1830ebee74b91541a9db1f060d86cee18 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 12 Jun 2018 13:19:41 -0300 Subject: [PATCH 1319/1506] [FIX] remove all third party imports from begin of the file. If a dep is missing other commands will work. --- manage.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/manage.py b/manage.py index c3373ecc2b0..d2b634efee3 100755 --- a/manage.py +++ b/manage.py @@ -11,9 +11,6 @@ import click import requests import sys -from requests import ConnectionError -from sqlalchemy.exc import OperationalError -from pgcli.main import PGCli import server.config from persistence.server.server import _conf, FARADAY_UP, SERVER_URL @@ -43,6 +40,16 @@ def check_faraday_server(url): @click.option('--workspace', default=None) @click.option('--polling/--no-polling', default=True) def process_reports(debug, workspace, polling): + try: + from requests import ConnectionError + except ImportError: + print('Python requests was not found. Please install it with: pip install requests') + sys.exit(1) + try: + from sqlalchemy.exc import OperationalError + except ImportError: + print('SQLAlchemy was not found please install it with: pip install sqlalchemy') + sys.exit(1) setUpLogger(debug) configuration = _conf() url = '{0}/_api/v2/info'.format(configuration.getServerURI() if FARADAY_UP else SERVER_URL) @@ -82,6 +89,11 @@ def database_schema(): @click.command() def sql_shell(): + try: + from pgcli.main import PGCli + except ImportError: + print('PGCli was not found, please install it with: pip install pgcli') + sys.exit(1) conn_string = server.config.database.connection_string.strip("'") pgcli = PGCli() From 11633847f906d67573cd892112d84e595a4f535d Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 12 Jun 2018 13:29:16 -0300 Subject: [PATCH 1320/1506] [FIX] return log file to the beginning for check errros --- server/commands/initdb.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index 5d9cadb6653..efd7e1e5d15 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -133,6 +133,7 @@ def _configure_existing_postgres_user(self): return username, password def _check_psql_output(self, current_psql_output_file, process_status): + current_psql_output_file.seek(0) psql_output = current_psql_output_file.read() if 'unknown user: postgres' in psql_output: print('ERROR: Postgres user not found. Did you install package {blue}postgresql{white}?'.format(blue=Fore.BLUE, white=Fore.WHITE)) From 1c84700a9eb6d2b8df1f40d5160e8ee9481be853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 12 Jun 2018 16:42:11 -0300 Subject: [PATCH 1321/1506] Add header to help page --- server/www/scripts/commons/controllers/headerCtrl.js | 2 +- server/www/scripts/help/partials/help.html | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/commons/controllers/headerCtrl.js b/server/www/scripts/commons/controllers/headerCtrl.js index 244e799b6cf..da73641c1ad 100644 --- a/server/www/scripts/commons/controllers/headerCtrl.js +++ b/server/www/scripts/commons/controllers/headerCtrl.js @@ -12,7 +12,7 @@ angular.module('faradayApp') var noSwitcher = [ "", "home", "login", "index", "workspaces", "users", "licenses", "taskgroup", "executive", // Only for white versions!! - "vulndb", "comparison", "webshell" + "vulndb", "comparison", "webshell", "help" ]; return noSwitcher.indexOf($scope.component) < 0; }; diff --git a/server/www/scripts/help/partials/help.html b/server/www/scripts/help/partials/help.html index 27b6f63cc84..2ac0b7a59be 100644 --- a/server/www/scripts/help/partials/help.html +++ b/server/www/scripts/help/partials/help.html @@ -5,7 +5,9 @@
    -
    +
    +
    +

    Relax! Help is coming your way!

    From f0a9841ef0c6ed5934d572acf8b915803802740d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 12 Jun 2018 16:42:22 -0300 Subject: [PATCH 1322/1506] Fix typo in upload report button in status report --- server/www/scripts/statusReport/partials/statusReport.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/statusReport/partials/statusReport.html b/server/www/scripts/statusReport/partials/statusReport.html index 6f8b98f1f75..41b62657e77 100644 --- a/server/www/scripts/statusReport/partials/statusReport.html +++ b/server/www/scripts/statusReport/partials/statusReport.html @@ -84,7 +84,7 @@
    -
    From c07d377d9e53adeafee5e1a49a169b34e47d4a9b Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Tue, 12 Jun 2018 16:48:21 -0300 Subject: [PATCH 1323/1506] Add RELEASE.md changes --- RELEASE.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index 02ed4de3ea3..fce8929411a 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -36,6 +36,15 @@ TBA: * Add fix for net sparker regular and cloud fix on severity * Removed Chat feature (data is kept inside notes) * Plugin reports now can be imported in the server, from the Web UI +* Add CVSS score to reference field in Nessus plugin. +* Add HP Webinspect plugin. +* Fix unicode characters bug in Netsparker plugin. +* Fix qualys plugin. +* Fix bugs with MACOS and GTK. + +April 10, 2018: +--- +* Fix bug with tornado version 5.0 and GTK client. November 17, 2017: --- From 5469f56513829473895f9f41f8b74c6580e2ff47 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 12 Jun 2018 16:50:59 -0300 Subject: [PATCH 1324/1506] [FIX] use psql_output instead of the file --- server/commands/initdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index 70be0233f05..6ecf35968e3 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -146,7 +146,7 @@ def _check_psql_output(self, current_psql_output_file, process_status): print('ERROR: {red}PostgreSQL service{white} is not running. Please verify that it is running in port 5432 before executing setup script.'.format(red=Fore.RED, white=Fore.WHITE)) elif process_status > 0: current_psql_output_file.seek(0) - print('ERROR: ' + current_psql_output_file.read()) + print('ERROR: ' + psql_output) if process_status is not 0: current_psql_output_file.close() # delete temp file From 35ff8833989c5042d06877e450013aefe20b9bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 12 Jun 2018 16:54:17 -0300 Subject: [PATCH 1325/1506] Redirect to dashboard after importing a report --- server/www/scripts/statusReport/controllers/statusReport.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index f60b7b7a577..afa3655d6e5 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -1127,7 +1127,7 @@ angular.module("faradayApp") } }).then( function(d) { - $window.location.reload(); + $location.path("/dashboard/ws/" + $routeParams.wsId); } ); }; From aec7acda0c4e30706581458d82a6482c9b7064f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 12 Jun 2018 16:59:13 -0300 Subject: [PATCH 1326/1506] Show error when report upload failed --- server/www/scripts/statusReport/controllers/statusReport.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index afa3655d6e5..0d6c50891db 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -1128,6 +1128,9 @@ angular.module("faradayApp") }).then( function(d) { $location.path("/dashboard/ws/" + $routeParams.wsId); + }, + function(d){ + commonsFact.showMessage("Error uploading report"); } ); }; From a8210016309d3b317a20eaafc690a020e1715a9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 12 Jun 2018 17:55:11 -0300 Subject: [PATCH 1327/1506] Add header for host create, edit and detail views --- .../www/scripts/commons/partials/header.html | 1 + server/www/scripts/hosts/partials/new.html | 20 ++++------------- .../www/scripts/services/partials/list.html | 22 ++++--------------- 3 files changed, 9 insertions(+), 34 deletions(-) diff --git a/server/www/scripts/commons/partials/header.html b/server/www/scripts/commons/partials/header.html index 8b18ab5e490..67bd227ab0d 100644 --- a/server/www/scripts/commons/partials/header.html +++ b/server/www/scripts/commons/partials/header.html @@ -7,6 +7,7 @@ Dashboard Status Report Hosts + Hosts Credentials Executive Report Tasks diff --git a/server/www/scripts/hosts/partials/new.html b/server/www/scripts/hosts/partials/new.html index 4b2cd3468cf..421121c9d97 100644 --- a/server/www/scripts/hosts/partials/new.html +++ b/server/www/scripts/hosts/partials/new.html @@ -5,25 +5,13 @@
    -
    -

    - Creating host -
    - - - -
    -

    +
    +
    +

    - Host details + Create a host diff --git a/server/www/scripts/services/partials/list.html b/server/www/scripts/services/partials/list.html index 906f02e7116..f03e09f3f23 100644 --- a/server/www/scripts/services/partials/list.html +++ b/server/www/scripts/services/partials/list.html @@ -4,25 +4,11 @@
    -
    -

    - Editing {{hostName}} in {{workspace}} - Creating host - Viewing {{hostName}} in {{workspace}} - -

    +
    +
    +
    -

    Host services

    +

    Services for host {{hostName}}

    diff --git a/server/www/scripts/commons/controllers/headerCtrl.js b/server/www/scripts/commons/controllers/headerCtrl.js index da73641c1ad..3a93773e953 100644 --- a/server/www/scripts/commons/controllers/headerCtrl.js +++ b/server/www/scripts/commons/controllers/headerCtrl.js @@ -12,7 +12,7 @@ angular.module('faradayApp') var noSwitcher = [ "", "home", "login", "index", "workspaces", "users", "licenses", "taskgroup", "executive", // Only for white versions!! - "vulndb", "comparison", "webshell", "help" + "vulndb", "comparison", "webshell", "help", "forbidden" ]; return noSwitcher.indexOf($scope.component) < 0; }; From b37c4ffe64768b04201e515de217dc4508c72474 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 12 Jun 2018 19:20:57 -0300 Subject: [PATCH 1330/1506] Improve workspace progress widget when no dates set --- .../controllers/workspaceProgressCtrl.js | 4 ++-- .../partials/workspace-progress.html | 20 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/server/www/scripts/dashboard/controllers/workspaceProgressCtrl.js b/server/www/scripts/dashboard/controllers/workspaceProgressCtrl.js index 3d2c65260cc..bec0a16973f 100644 --- a/server/www/scripts/dashboard/controllers/workspaceProgressCtrl.js +++ b/server/www/scripts/dashboard/controllers/workspaceProgressCtrl.js @@ -33,7 +33,7 @@ angular.module('faradayApp') today = new Date(), total = 0; - if(duration.start == "" || duration.end == "") { + if(!duration.start || !duration.end) { progress = null; } else { today = today.getTime(); @@ -53,4 +53,4 @@ angular.module('faradayApp') dashboardSrv.registerCallback(init); init(); - }]); \ No newline at end of file + }]); diff --git a/server/www/scripts/dashboard/partials/workspace-progress.html b/server/www/scripts/dashboard/partials/workspace-progress.html index 6bbc9501544..ae1b3577a84 100644 --- a/server/www/scripts/dashboard/partials/workspace-progress.html +++ b/server/www/scripts/dashboard/partials/workspace-progress.html @@ -6,23 +6,25 @@

    +
    + +

    Start date and end date are required

    +
    {{wsProgress}}%
    - Start date: {{wsStart | amUtc | amDateFormat:'MM/DD/YYYY'}} + Start date: {{wsStart | amUtc | amDateFormat:'MM/DD/YYYY'}} + The workspace doesn't have a start date
    - End date: {{wsEnd | amUtc | amDateFormat:'MM/DD/YYYY'}} + End date: {{wsEnd | amUtc | amDateFormat:'MM/DD/YYYY'}} + The workspace doesn't have an end date
    -
    - -

    Start date and end date are required

    -
    From 79190f8184b94449d01fb6679b39a78c8b65c240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 12 Jun 2018 19:28:11 -0300 Subject: [PATCH 1331/1506] Make scope work when globally editing a workspace This is really ugly, but appearently works. Ticket #4864 --- server/www/scripts/commons/controllers/headerCtrl.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/www/scripts/commons/controllers/headerCtrl.js b/server/www/scripts/commons/controllers/headerCtrl.js index 3a93773e953..66a49b198a1 100644 --- a/server/www/scripts/commons/controllers/headerCtrl.js +++ b/server/www/scripts/commons/controllers/headerCtrl.js @@ -81,6 +81,13 @@ angular.module('faradayApp') } }); + // copy pasted from server/www/scripts/workspaces/controllers/workspaces.js + // it makes scope work properly (i think) + workspace.scope = workspace.scope.map(function(scope){ + return {key: scope} + }); + if (workspace.scope.length == 0) workspace.scope.push({key: ''}); + var oldName = $scope.workspace; var modal = $uibModal.open({ templateUrl: 'scripts/workspaces/partials/modalEdit.html', From 0b0998465ae741bcfcbbfc43327692ad555d318e Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Tue, 12 Jun 2018 20:16:48 -0300 Subject: [PATCH 1332/1506] Fix burp plugin --- plugins/repo/burp/faraday-burp.rb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/plugins/repo/burp/faraday-burp.rb b/plugins/repo/burp/faraday-burp.rb index 3b6ee4b6627..de62215ba5b 100644 --- a/plugins/repo/burp/faraday-burp.rb +++ b/plugins/repo/burp/faraday-burp.rb @@ -7,7 +7,7 @@ #__author__ = "Francisco Amato" #__copyright__ = "Copyright (c) 2014, Infobyte LLC" #__credits__ = ["Francisco Amato", "Micaela Ranea Sanchez"] -#__version__ = "1.3.0" +#__version__ = "1.4.0" #__maintainer__ = "Francisco Amato" #__email__ = "famato@infobytesec.com" #__status__ = "Development" @@ -17,7 +17,7 @@ require "pp" -PLUGINVERSION="Faraday v1.3 Ruby" +PLUGINVERSION="Faraday v1.4 Ruby" #Tested: Burp Professional v1.6.09 XMLRPC::Config.module_eval do @@ -250,10 +250,6 @@ def newScanIssue(issue, ctx=nil, import=nil) s_id = @server.call("createAndAddServiceToInterface",h_id, i_id, issue.getProtocol(),"tcp",[port],"open") - #Save website - n_id = @server.call("createAndAddNoteToService",h_id,s_id,"website","") - n2_id = @server.call("createAndAddNoteToNote",h_id,s_id,n_id,host,"") - path = "" response = "" request = "" From 33b24c09de848466470bd8d708450bd7c4d1abae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 13 Jun 2018 12:28:27 -0300 Subject: [PATCH 1333/1506] Show at most 10 commands in the dashboard command history --- server/www/scripts/commons/providers/server.js | 4 ++-- server/www/scripts/dashboard/providers/dashboard.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index afccaa17d12..b0f5df62c7a 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -258,8 +258,8 @@ angular.module("faradayApp") return get(getUrl, data); } - ServerAPI.getCommands = function(wsName, data) { - var getUrl = createGetUrl(wsName, 'commands'); + ServerAPI.getCommands = function(wsName, data, onlyLastCommands) { + var getUrl = createGetUrl(wsName, 'commands') + (onlyLastCommands ? '?page_size=10&page=1' : ''); return get(getUrl, data); } diff --git a/server/www/scripts/dashboard/providers/dashboard.js b/server/www/scripts/dashboard/providers/dashboard.js index 24615e15a59..d8e9c86d8a4 100644 --- a/server/www/scripts/dashboard/providers/dashboard.js +++ b/server/www/scripts/dashboard/providers/dashboard.js @@ -256,7 +256,7 @@ angular.module('faradayApp') dashboardSrv.getCommands = function(ws) { var deferred = $q.defer(); - ServerAPI.getCommands(ws) + ServerAPI.getCommands(ws, undefined, true) .then(function(res) { var tmp = []; res.data.commands.forEach(function(cmd) { From a909b277fc21509767af39b273706d7bd2c2eeef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 13 Jun 2018 12:29:34 -0300 Subject: [PATCH 1334/1506] Add pagination and default sorting for commands api --- server/api/modules/commandsrun.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/api/modules/commandsrun.py b/server/api/modules/commandsrun.py index e48861b130e..cd6f5bdfa38 100644 --- a/server/api/modules/commandsrun.py +++ b/server/api/modules/commandsrun.py @@ -9,7 +9,7 @@ from flask_classful import route from marshmallow import fields, post_load -from server.api.base import AutoSchema, ReadWriteWorkspacedView +from server.api.base import AutoSchema, ReadWriteWorkspacedView, PaginatedMixin from server.utils.logger import get_logger from server.utils.web import ( gzipped, @@ -55,11 +55,12 @@ class Meta: 'params', 'user', 'workspace', 'tool', 'import_source') -class CommandView(ReadWriteWorkspacedView): +class CommandView(PaginatedMixin, ReadWriteWorkspacedView): route_base = 'commands' model_class = Command schema_class = CommandSchema get_joinedloads = [Command.workspace] + order_field = Command.start_date.desc() def _envelope_list(self, objects, pagination_metadata=None): commands = [] From 914d8a94ac01fddb0d361e202e37eb1171346aac Mon Sep 17 00:00:00 2001 From: montive Date: Wed, 13 Jun 2018 12:45:02 -0300 Subject: [PATCH 1335/1506] Add Wfuzz Plugin --- RELEASE.md | 1 + plugins/repo/wfuzz/__init__.py | 6 ++ plugins/repo/wfuzz/plugin.py | 103 +++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 plugins/repo/wfuzz/__init__.py create mode 100644 plugins/repo/wfuzz/plugin.py diff --git a/RELEASE.md b/RELEASE.md index d1925242e10..c7984c60134 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -32,6 +32,7 @@ TBA: * Add new screenshot fplugin which takes a screenshot of the ip:ports of a given protocol * Add fix for net sparker regular and cloud fix on severity * Removed Chat feature (data is kept inside notes) +* Add new plugin wfuzz November 17, 2017: --- diff --git a/plugins/repo/wfuzz/__init__.py b/plugins/repo/wfuzz/__init__.py new file mode 100644 index 00000000000..004c49be6c6 --- /dev/null +++ b/plugins/repo/wfuzz/__init__.py @@ -0,0 +1,6 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' diff --git a/plugins/repo/wfuzz/plugin.py b/plugins/repo/wfuzz/plugin.py new file mode 100644 index 00000000000..0d75c784dcd --- /dev/null +++ b/plugins/repo/wfuzz/plugin.py @@ -0,0 +1,103 @@ +from plugins import core +from urlparse import urljoin, urlparse + + +class WfuzzPlugin(core.PluginBase): + + def __init__(self): + core.PluginBase.__init__(self) + self.id = "Wfuzz" + self.name = "Wfuzz Plugin" + self.plugin_version = "0.0.1" + self.version = "2.2.11" + self.options = None + + self.host = None + self.port = None + self.protocol = None + self.fail = None + + def parseData(self, output): + + data = { + 'target' : '', + 'findings' : [ + + ] + } + for line in output: + + if(line.startswith('Target')): + data['target'] = line[8:].rstrip() + + if(line.startswith('0')): + aux = line.split(' ') + res = {} + for item in aux: + if 'C=' in item: + res['response'] = int(item.replace('C=', '')) + elif 'L' in item and ' ' in item: + res['lines'] = int(item.replace('L', '')) + elif 'W' in item and ' ' in item: + res['words'] = int(item.replace('W', '')) + elif 'Ch' in item and ' ' in item: + res['chars'] = int(item.replace('Ch', '')) + else: + res['request'] = item.rstrip().replace('"', '') + data['findings'].append(res) + + return data + + + def parseOutputString(self, output, debug=False): + output_list = output.split('\n') + + print output_list + info = self.parseData(output_list) + + target = info['target'] + target_url = urlparse(target) + port = 80 + if target_url.scheme == 'https': + port = 443 + custom_port = target_url.netloc.split(':') + if len(custom_port) > 1: + port = custom_port[1] + + host_id = self.createAndAddHost(target) + + service_id = self.createAndAddServiceToHost(host_id,name="http",protocol="tcp", ports=[port] ) + + for item in info['findings']: + path = item['request'] + status = item['response'] + url = urljoin(target, path) + lines = item['lines'] + chars = item['chars'] + words = item['words'] + name = "Wfuzz found: {path} with status {status} on url {url}".format(path=path, status=status, url=url) + desc = 'Wfuzz found a response with status {status}. Response contains: \n* {words} words \n* {lines} lines \n* {chars} chars'.format( + words=words, + url=url, + lines=lines, + chars=chars, + status=status + ) + vuln = self.createAndAddVulnWebToService(host_id, + service_id, + name, + desc, + severity="info", + website=target, + path=path + ) + + +def createPlugin(): + return WfuzzPlugin() + + +if __name__ == '__main__': + parser = WfuzzPlugin() + with open("/home/javier/salida", "r") as report: + parser.parseOutputString(report) From ef02e11711d3ff9513d7da71e2ed169404e6be2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 13 Jun 2018 14:24:47 -0300 Subject: [PATCH 1336/1506] Add message when no credentials found in a workspace --- server/www/scripts/credentials/partials/list.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/www/scripts/credentials/partials/list.html b/server/www/scripts/credentials/partials/list.html index deb6611ac6b..2cd8b461af0 100644 --- a/server/www/scripts/credentials/partials/list.html +++ b/server/www/scripts/credentials/partials/list.html @@ -74,6 +74,9 @@

    End Date Vulns HostsLast Modified Services
    {{objects[ws.name]['total_vulns']}} {{objects[ws.name]['hosts']}}
    +

    + No credentials found in this workspace +

    From 41b5c9a1b9594705612a2000bcaefb0d6361fc16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 13 Jun 2018 14:47:29 -0300 Subject: [PATCH 1337/1506] Wrap text columns in vuln template list --- server/www/scripts/vulndb/partials/vulndb.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/www/scripts/vulndb/partials/vulndb.html b/server/www/scripts/vulndb/partials/vulndb.html index 9fb37ee198d..ecaad043614 100644 --- a/server/www/scripts/vulndb/partials/vulndb.html +++ b/server/www/scripts/vulndb/partials/vulndb.html @@ -71,11 +71,11 @@ selection-model-mode="multiple-additive" selection-model-selected-class="multi-selected" selection-model-on-change="selectedModels()"> -
    		{{model.name}}{{model.description}}{{model.resolution}}{{model.exploitation}}{{model.name}}{{model.description}}{{model.resolution}}{{model.exploitation}}
    From d71c6b449e42152e769417e69eaa3ebdeebdb2c9 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 13 Jun 2018 15:00:45 -0300 Subject: [PATCH 1338/1506] Fix url in status report upload report --- server/www/scripts/statusReport/controllers/statusReport.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 0d6c50891db..e047e7a8114 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -46,6 +46,7 @@ angular.module("faradayApp") var init = function() { $scope.baseurl = BASEURL; + console.log($scope.baseurl); $scope.severities = SEVERITIES; $scope.easeofresolution = EASEOFRESOLUTION; $scope.propertyGroupBy = $routeParams.groupbyId; @@ -1112,12 +1113,11 @@ angular.module("faradayApp") toggleFileUpload(); } }; - $scope.uploadFile = function() { var fd = new FormData(); fd.append('csrf_token', $scope.csrf_token); fd.append('file', $scope.fileToUpload); - $http.post('http://localhost:5985/_api/v2/ws/' + $scope.workspace + '/upload_report', fd, { + $http.post($scope.baseurl + '_api/v2/ws/' + $scope.workspace + '/upload_report', fd, { transformRequest: angular.identity, withCredentials: false, headers: {'Content-Type': undefined}, From 8d44b06f44b205bc83f6c3e568c44a8557f67365 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 13 Jun 2018 15:10:02 -0300 Subject: [PATCH 1339/1506] Fix align status report --- server/www/scripts/dashboard/partials/services.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/dashboard/partials/services.html b/server/www/scripts/dashboard/partials/services.html index df27d16760b..1642ae72a15 100644 --- a/server/www/scripts/dashboard/partials/services.html +++ b/server/www/scripts/dashboard/partials/services.html @@ -15,7 +15,7 @@

    Services report
    -
    +
    {{srv.count}}
    From 30089e3dacadd0625f0dd73c48e16511f1ab1e39 Mon Sep 17 00:00:00 2001 From: montive Date: Wed, 13 Jun 2018 15:38:19 -0300 Subject: [PATCH 1340/1506] Se agrego self._command_regex pero sigue sin funcionar en la terminal del GTK --- plugins/repo/wfuzz/plugin.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/plugins/repo/wfuzz/plugin.py b/plugins/repo/wfuzz/plugin.py index 0d75c784dcd..c9558eedd01 100644 --- a/plugins/repo/wfuzz/plugin.py +++ b/plugins/repo/wfuzz/plugin.py @@ -1,3 +1,5 @@ +import sys +import re from plugins import core from urlparse import urljoin, urlparse @@ -16,6 +18,8 @@ def __init__(self): self.port = None self.protocol = None self.fail = None + self._command_regex = re.compile( + r'^(wfuzz).*?') def parseData(self, output): @@ -50,14 +54,14 @@ def parseData(self, output): def parseOutputString(self, output, debug=False): - output_list = output.split('\n') - print output_list + output_list = output.split('\n') info = self.parseData(output_list) target = info['target'] target_url = urlparse(target) port = 80 + if target_url.scheme == 'https': port = 443 custom_port = target_url.netloc.split(':') @@ -92,6 +96,7 @@ def parseOutputString(self, output, debug=False): path=path ) + def createPlugin(): return WfuzzPlugin() @@ -100,4 +105,4 @@ def createPlugin(): if __name__ == '__main__': parser = WfuzzPlugin() with open("/home/javier/salida", "r") as report: - parser.parseOutputString(report) + parser.parseOutputString(report.read()) From b3a65056652a1b909479a2ee5a2c523d4c74ea18 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 13 Jun 2018 15:44:34 -0300 Subject: [PATCH 1341/1506] Fix workspace switcher in commertial header data_analysis --- server/www/scripts/commons/controllers/headerCtrl.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/commons/controllers/headerCtrl.js b/server/www/scripts/commons/controllers/headerCtrl.js index 66a49b198a1..9212d80c6b3 100644 --- a/server/www/scripts/commons/controllers/headerCtrl.js +++ b/server/www/scripts/commons/controllers/headerCtrl.js @@ -12,7 +12,7 @@ angular.module('faradayApp') var noSwitcher = [ "", "home", "login", "index", "workspaces", "users", "licenses", "taskgroup", "executive", // Only for white versions!! - "vulndb", "comparison", "webshell", "help", "forbidden" + "vulndb", "comparison", "webshell", "help", "forbidden", "data_analysis" ]; return noSwitcher.indexOf($scope.component) < 0; }; From 21f842dcae335bef4a7fd30cf6d79a8b7666c84b Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 13 Jun 2018 15:58:13 -0300 Subject: [PATCH 1342/1506] [FIX] check for pid file first --- faraday-server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/faraday-server.py b/faraday-server.py index 20d7e5e1c3b..470199d7278 100755 --- a/faraday-server.py +++ b/faraday-server.py @@ -131,11 +131,11 @@ def main(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) result = sock.connect_ex((args.bind_address or server.config.faraday_server.bind_address, int(args.port or server.config.faraday_server.port))) - if result == 0: - logger.error("Faraday server port in use. Check your processes and run the server again...") + if is_server_running(): sys.exit(1) - if is_server_running(): + if result == 0: + logger.error("Faraday server port in use. Check your processes and run the server again...") sys.exit(1) # Overwrites config option if SSL is set by argument From 36fb7d54c14c1b4a2c16da5e570d89d38701fd0f Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 13 Jun 2018 17:16:47 -0300 Subject: [PATCH 1343/1506] [FIX] Add one unique handler for each signal --- server/web.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/server/web.py b/server/web.py index 8b96711d200..df409425dc9 100644 --- a/server/web.py +++ b/server/web.py @@ -133,12 +133,13 @@ def __build_websockets_resource(self): return factory def install_signal(self): - def signal_handler(*args): - logger.info("Stopping threads, please wait...") - # teardown() - self.raw_report_processor.stop() - reactor.stop() for sig in (SIGABRT, SIGILL, SIGINT, SIGSEGV, SIGTERM): + def signal_handler(*args): + logger.info("Stopping threads, please wait...") + # teardown() + self.raw_report_processor.stop() + reactor.stop() + signal(sig, signal_handler) def run(self): From 5c7d25bd24e251bb4defb880ef22ca3923967b72 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 13 Jun 2018 18:03:22 -0300 Subject: [PATCH 1344/1506] [FIX] Improve logging. Use proper logging for thread. Don't block upload report' --- server/api/modules/upload_reports.py | 14 ++++++-------- server/utils/daemonize.py | 2 +- server/utils/logger.py | 8 +++++++- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/server/api/modules/upload_reports.py b/server/api/modules/upload_reports.py index 1a4c1509df0..edd75cc080f 100644 --- a/server/api/modules/upload_reports.py +++ b/server/api/modules/upload_reports.py @@ -81,7 +81,7 @@ def run(self): while not self._stop: try: workspace, file_path, cookie = UPLOAD_REPORTS_QUEUE.get(False, timeout=0.1) - logger.info('Processing raw report {0}'.format(file_path)) + get_logger().info('Processing raw report {0}'.format(file_path)) # Cookie of user, used to create objects in server with the right owner. server.FARADAY_UPLOAD_REPORTS_WEB_COOKIE = cookie @@ -91,13 +91,15 @@ def run(self): if not command_id: continue self.end_event.wait() + get_logger().info('Report processing of report {0} finished'.format(file_path)) self.end_event.clear() except Empty: time.sleep(0.1) except KeyboardInterrupt as ex: - break + get_logger().info('Keyboard interrupt, stopping report processing thread') + self.stop() except Exception as ex: - logger.exception(ex) + get_logger().exception(ex) continue @@ -136,8 +138,4 @@ def file_upload(workspace=None): output.write(report_file.read()) UPLOAD_REPORTS_QUEUE.put((workspace, file_path, request.cookies)) - command_id = UPLOAD_REPORTS_CMD_QUEUE.get() - if command_id: - # return jsonify({"status": "processing", "command_id": command_id}) - return redirect('/#/dashboard/ws/' + workspace) - abort(make_response(jsonify(message="Invalid file."), 400)) + return redirect('/#/dashboard/ws/' + workspace) diff --git a/server/utils/daemonize.py b/server/utils/daemonize.py index 021dc6e9690..a9029306930 100644 --- a/server/utils/daemonize.py +++ b/server/utils/daemonize.py @@ -140,13 +140,13 @@ def start_server(): def stop_server(): """Stops Faraday Server if it isn't running""" logger = get_logger(__name__) - pid = is_server_running() if pid is None: logger.error('Faraday Server is not running') return False try: + logger.info('Sending SIGTERM to pid {0}'.format(pid)) os.kill(pid, signal.SIGTERM) except OSError, err: if err.errno == errno.EPERM: diff --git a/server/utils/logger.py b/server/utils/logger.py index bf02a22e013..711dfae0a50 100644 --- a/server/utils/logger.py +++ b/server/utils/logger.py @@ -24,11 +24,12 @@ def setup_logging(): logger.setLevel(logging.DEBUG) formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + '%(asctime)s - %(name)s - %(levelname)s {%(threadName)s} [%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s') setup_console_logging(formatter) setup_file_logging(formatter) + def setup_console_logging(formatter): console_handler = logging.StreamHandler() console_handler.setFormatter(formatter) @@ -36,6 +37,7 @@ def setup_console_logging(formatter): add_handler(console_handler) LVL_SETTABLE_HANDLERS.append(console_handler) + def setup_file_logging(formatter): create_logging_path() file_handler = logging.handlers.RotatingFileHandler( @@ -44,11 +46,13 @@ def setup_file_logging(formatter): file_handler.setLevel(logging.DEBUG) add_handler(file_handler) + def add_handler(handler): logger = logging.getLogger(ROOT_LOGGER) logger.addHandler(handler) LOGGING_HANDLERS.append(handler) + def get_logger(obj=None): """Creates a logger named by a string or an object's class name. Allowing logger to additionally accept strings as names @@ -64,11 +68,13 @@ def get_logger(obj=None): return logger + def set_logging_level(level): server.config.LOGGING_LEVEL = level for handler in LVL_SETTABLE_HANDLERS: handler.setLevel(level) + def create_logging_path(): try: os.makedirs(os.path.dirname(LOG_FILE)) From cf44305e90bd857290b1fb9d4dfc290b5b9a1d09 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 13 Jun 2018 18:07:48 -0300 Subject: [PATCH 1345/1506] Remove white spaces --- server/utils/logger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/utils/logger.py b/server/utils/logger.py index 711dfae0a50..a60f05b1766 100644 --- a/server/utils/logger.py +++ b/server/utils/logger.py @@ -18,13 +18,14 @@ LOGGING_HANDLERS = [] LVL_SETTABLE_HANDLERS = [] + def setup_logging(): logger = logging.getLogger(ROOT_LOGGER) logger.propagate = False logger.setLevel(logging.DEBUG) formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s {%(threadName)s} [%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s') + '%(asctime)s - %(name)s - %(levelname)s {%(threadName)s} [%(filename)s:%(lineno)s - %(funcName)s() ] %(message)s') setup_console_logging(formatter) setup_file_logging(formatter) From 5060f7260c74d81915236eaed062676917006022 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 14 Jun 2018 13:11:03 -0300 Subject: [PATCH 1346/1506] [ADD] OS detection to install.sh --- install.sh | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/install.sh b/install.sh index 6e44a1c343b..baead324478 100755 --- a/install.sh +++ b/install.sh @@ -10,14 +10,30 @@ if [ $EUID -ne 0 ]; then exit 1 fi -apt-get update +case `uname` in + Linux ) + LINUX=1 + type apt-get >/dev/null 2>&1 && { package_manager='apt-get'; } + type apt >/dev/null 2>&1 && { package_manager='apt'; } + ;; + Darwin ) + DARWIN=1 + type brew >/dev/null 2>&1 && { package_manager='brew'; } + ;; +esac -#Install community dependencies -for pkg in build-essential python-setuptools python-pip python-dev libpq-dev libffi-dev gir1.2-gtk-3.0 gir1.2-vte-2.91 python-gobject zsh curl python-psycopg2 ; do - apt-get install -y $pkg -done + +if [ "$package_manager" == "apt-get" ] +then + apt-get update + + #Install community dependencies + for pkg in build-essential python-setuptools python-pip python-dev libpq-dev libffi-dev gir1.2-gtk-3.0 gir1.2-vte-2.91 python-gobject zsh curl python-psycopg2 ; do + apt-get install -y $pkg + done +fi pip2 install -r requirements_server.txt pip2 install -r requirements.txt -echo "You can now run Faraday, enjoy!" \ No newline at end of file +echo "You can now run Faraday, enjoy!" From 8ee5b96312730f39002c731fa8c931b20bf93c99 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Thu, 14 Jun 2018 14:46:07 -0300 Subject: [PATCH 1347/1506] [FIX] w3af and zap expected xml parsing --- plugins/repo/w3af/plugin.py | 28 +++++++++++++++++++++------- plugins/repo/zap/plugin.py | 13 ++++++++++--- server/app.py | 2 +- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/plugins/repo/w3af/plugin.py b/plugins/repo/w3af/plugin.py index a45e537f2bf..30a4098a7bc 100644 --- a/plugins/repo/w3af/plugin.py +++ b/plugins/repo/w3af/plugin.py @@ -84,7 +84,11 @@ def get_items(self, tree): """ bugtype = "" - scaninfo = tree.findall('scan-info')[0] + if len(tree.findall('scan-info')) == 0: + scaninfo = tree.findall('scaninfo')[0] + else: + scaninfo = tree.findall('scan-info')[0] + self.target = scaninfo.get('target') host = re.search( "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))[\:]*([0-9]+)*([/]*($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+)).*?$", self.target) @@ -165,16 +169,26 @@ def __init__(self, item_node): self.req = self.resp = '' for tx in self.node.findall('http-transactions/http-transaction'): - self.req = tx.find('http-request/status').text - for h in tx.findall('http-request/headers/header'): + if tx.find('http-request'): + hreq = tx.find('http-request') + else: + hreq = tx.find('httprequest') + + if tx.find('http-response'): + hres = tx.find('http-response') + else: + hres = tx.find('httpresponse') + + self.req = hreq.find('status').text + for h in hreq.findall('headers/header'): self.req += "\n%s: %s" % (h.get('field'), h.get('content')) - self.resp = tx.find('http-response/status').text - for h in tx.findall('http-response/headers/header'): + self.resp = hres.find('status').text + for h in hres.findall('headers/header'): self.resp += "\n%s: %s" % (h.get('field'), h.get('content')) - if tx.find('http-response/body'): - self.resp += "\n%s" % tx.find('http-response/body').text + if hres.find('body'): + self.resp += "\n%s" % hres.find('body').text def do_clean(self, value): myreturn = "" diff --git a/plugins/repo/zap/plugin.py b/plugins/repo/zap/plugin.py index 4a243a34cdb..5b2cece0e60 100644 --- a/plugins/repo/zap/plugin.py +++ b/plugins/repo/zap/plugin.py @@ -199,10 +199,18 @@ def __init__(self, item_node): self.items = [] - for instance in item_node.find('instances'): + if item_node.find('instances'): + arr = item_node.find('instances') + else: + arr = [item_node] + + for elem in arr: + uri = elem.find('uri').text + self.parse_uri(uri) - uri = instance.find('uri').text + self.requests = "\n".join([i['uri'] for i in self.items]) + def parse_uri(self, uri): mregex = re.search( "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&" ";%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]" @@ -238,7 +246,6 @@ def __init__(self, item_node): } self.items.append(item) - self.requests = "\n".join([i['uri'] for i in self.items]) def get_text_from_subnode(self, subnode_xpath_expr): """ diff --git a/server/app.py b/server/app.py index 7f170ebee44..680995514d3 100644 --- a/server/app.py +++ b/server/app.py @@ -26,7 +26,7 @@ Security, SQLAlchemyUserDatastore, ) -from flask.ext.session import Session +from flask_session import Session from nplusone.ext.flask_sqlalchemy import NPlusOne from depot.manager import DepotManager From c05b726f53ca1ab994e10c1e0fecd66e830f2085 Mon Sep 17 00:00:00 2001 From: fedek Date: Thu, 14 Jun 2018 15:24:10 -0300 Subject: [PATCH 1348/1506] Change in the servicesbyhost modal from the dashboard to update the new UI elements Change in the workspace list to move the column DATE as the last item --- .../summarizedCtrlServicesModal.js | 1 + .../dashboard/partials/last-vulns.html | 7 +-- .../partials/modal-services-by-host.html | 59 ++++++++++++------- .../www/scripts/workspaces/partials/list.html | 5 +- 4 files changed, 44 insertions(+), 28 deletions(-) diff --git a/server/www/scripts/dashboard/controllers/summarizedCtrlServicesModal.js b/server/www/scripts/dashboard/controllers/summarizedCtrlServicesModal.js index 455b0595467..ca2314b5929 100644 --- a/server/www/scripts/dashboard/controllers/summarizedCtrlServicesModal.js +++ b/server/www/scripts/dashboard/controllers/summarizedCtrlServicesModal.js @@ -11,6 +11,7 @@ angular.module('faradayApp') $scope.sortField = 'port'; $scope.sortReverse = false; $scope.osint = osint; + $scope.workspace = workspace // toggles sort field and order $scope.toggleSort = function(field) { diff --git a/server/www/scripts/dashboard/partials/last-vulns.html b/server/www/scripts/dashboard/partials/last-vulns.html index 4e1684a7fa1..eab8e8185c9 100644 --- a/server/www/scripts/dashboard/partials/last-vulns.html +++ b/server/www/scripts/dashboard/partials/last-vulns.html @@ -14,7 +14,7 @@

    Last Vulnerabilities

    		Severity Target NameWebOwner Date
    {{vuln.name}} - - - {{vuln.owner}}
    +
    - - - - - - + + + + + + - - + - - + - - + + -
    -
    Operating System: {{host.os}}
    -
    Created
    +
    +
    + OS: {{host.os | limitTo: 40}} +
    + {{host.metadata.owner}} was created + +
    +
    NameDescriptionPortProtocolStatus
    + Name + + Version + + Port + + Protocol + + Status +
    +
    {{srv.name}} - {{srv.description}} + + {{srv.version}} + {{srv.ports.toString().replace("[]", "")}} {{srv.protocol}}{{srv.status}} + {{srv.protocol}} + + {{srv.status}} +
    -
    - + `
    diff --git a/server/www/scripts/workspaces/partials/list.html b/server/www/scripts/workspaces/partials/list.html index f59769886f4..e92e002e2bc 100644 --- a/server/www/scripts/workspaces/partials/list.html +++ b/server/www/scripts/workspaces/partials/list.html @@ -49,9 +49,8 @@ End Date Vulns Hosts - - Last Modified Services + Last Modified @@ -66,8 +65,8 @@ {{objects[ws.name]['total_vulns']}} {{objects[ws.name]['hosts']}} - + From b287907544e5905133766f79ad6b2767f77046a5 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Thu, 14 Jun 2018 15:35:34 -0300 Subject: [PATCH 1349/1506] Fix bad phrase in services by host WebUi --- .../scripts/dashboard/partials/modal-services-by-host.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/www/scripts/dashboard/partials/modal-services-by-host.html b/server/www/scripts/dashboard/partials/modal-services-by-host.html index 1a6037903bb..02c164543eb 100644 --- a/server/www/scripts/dashboard/partials/modal-services-by-host.html +++ b/server/www/scripts/dashboard/partials/modal-services-by-host.html @@ -49,10 +49,10 @@

    Services for
    - OS: {{host.os | limitTo: 40}} + OS: {{host.os | limitTo: 40}}
    - {{host.metadata.owner}} was created - + Created by {{host.metadata.owner}} +

    From 1b3bd7c65fd752ba2655ebe6b65a0a6ed6efb30b Mon Sep 17 00:00:00 2001 From: fedek Date: Thu, 14 Jun 2018 15:56:06 -0300 Subject: [PATCH 1350/1506] Changed ServiceByHosts modal to new UI --- .../controllers/summarizedCtrlHostsModal.js | 1 + .../partials/modal-hosts-by-service.html | 30 ++++++++++++------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/server/www/scripts/dashboard/controllers/summarizedCtrlHostsModal.js b/server/www/scripts/dashboard/controllers/summarizedCtrlHostsModal.js index f75d5214eca..b0bf10e6cb5 100644 --- a/server/www/scripts/dashboard/controllers/summarizedCtrlHostsModal.js +++ b/server/www/scripts/dashboard/controllers/summarizedCtrlHostsModal.js @@ -11,6 +11,7 @@ angular.module('faradayApp') $scope.sortField = 'name'; $scope.sortReverse = false; $scope.clipText = "Copy to Clipboard"; + $scope.workspace = workspace // toggles sort field and order $scope.toggleSort = function(field) { diff --git a/server/www/scripts/dashboard/partials/modal-hosts-by-service.html b/server/www/scripts/dashboard/partials/modal-hosts-by-service.html index 0845dfc2d7a..c1484809457 100644 --- a/server/www/scripts/dashboard/partials/modal-hosts-by-service.html +++ b/server/www/scripts/dashboard/partials/modal-hosts-by-service.html @@ -7,22 +7,32 @@

    Services for {{name}} ({{hosts.length}} total)

    - +
    - - - - + + + + - - - + + - +
    OwnedNameOS
    + Owned + + Name + + OS +
    - {{host.name}} +
    + + + {{host.name}} {{host.os}} + {{host.os}} +
    From 2408b3325c6b53b898d3b759b80ed16fdaea6191 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Thu, 14 Jun 2018 16:56:31 -0300 Subject: [PATCH 1351/1506] Fix size of description field in Status report new vulnerability and Edit modal --- server/www/estilos.css | 6 +++++- server/www/scripts/statusReport/partials/modalEdit.html | 3 +-- server/www/scripts/statusReport/partials/modalNew.html | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/server/www/estilos.css b/server/www/estilos.css index a59957986c3..e5c1d780b0b 100644 --- a/server/www/estilos.css +++ b/server/www/estilos.css @@ -374,7 +374,7 @@ article.untercio{ /* Home Menu grande > Corporate */ .home-list.corporate .item:nth-child(5), .home-list.corporate .item:nth-child(9), .home-list.corporate .item:nth-child(13) {clear:left;} - +/* Home Animate Icon Order FlipIn */ .item:nth-child(0) {-webkit-animation-delay: 0s;} .item:nth-child(1) {-webkit-animation-delay: 0.1s;} .item:nth-child(2) {-webkit-animation-delay: 0.2s;} @@ -385,6 +385,10 @@ article.untercio{ .item:nth-child(7) {-webkit-animation-delay: 0.7s;} .item:nth-child(8) {-webkit-animation-delay: 0.8s;} .item:nth-child(9) {-webkit-animation-delay: 0.9s;} +.item:nth-child(10) {-webkit-animation-delay: 1.0s;} +.item:nth-child(11) {-webkit-animation-delay: 1.1s;} +.item:nth-child(12) {-webkit-animation-delay: 1.2s;} + /* Icons on Home */ .icons-color-home{color: #B3B4B5;} .fa.host{font-size: 2.6em;} diff --git a/server/www/scripts/statusReport/partials/modalEdit.html b/server/www/scripts/statusReport/partials/modalEdit.html index 0ccadc42fca..b6724a36418 100644 --- a/server/www/scripts/statusReport/partials/modalEdit.html +++ b/server/www/scripts/statusReport/partials/modalEdit.html @@ -42,7 +42,6 @@
    Status
    Confirmed
    - Confirmed
    @@ -54,7 +53,7 @@
    Confirmed
    - +
    diff --git a/server/www/scripts/statusReport/partials/modalNew.html b/server/www/scripts/statusReport/partials/modalNew.html index 50fe7e617c2..5fff16af5e8 100644 --- a/server/www/scripts/statusReport/partials/modalNew.html +++ b/server/www/scripts/statusReport/partials/modalNew.html @@ -120,7 +120,7 @@
    Ease of Resolution
    - +
    From 23a0e4eeec5fefc31af1f2b3fa6a051acaa94da9 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Thu, 14 Jun 2018 17:04:11 -0300 Subject: [PATCH 1352/1506] Fix impact position in WebUi status report --- server/www/estilos.css | 21 +++++++++++++++++++ .../statusReport/partials/modalEdit.html | 7 ++++--- .../statusReport/partials/modalNew.html | 8 +++---- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/server/www/estilos.css b/server/www/estilos.css index e5c1d780b0b..789185937ea 100644 --- a/server/www/estilos.css +++ b/server/www/estilos.css @@ -1288,3 +1288,24 @@ a.button-disable{cursor: not-allowed;pointer-events: none;opacity: 0.5} .tooltip-inner { white-space: pre-wrap; } + +/* deduplication of CSS - StatusResport */ +.label-impact { + display: inline; + padding: .2em .6em .3em; + font-size: 100%; + font-weight: 700; + line-height: 1; + color: #fff; + text-align: center; + white-space: nowrap; + vertical-align: baseline; + -webkit-border-radius: .25em; + border-radius: .25em + } + .label-default-impact { + background-color: #777 + } + .label-success-impact { + background-color: #5cb85c + } \ No newline at end of file diff --git a/server/www/scripts/statusReport/partials/modalEdit.html b/server/www/scripts/statusReport/partials/modalEdit.html index b6724a36418..61cf8ea4ba7 100644 --- a/server/www/scripts/statusReport/partials/modalEdit.html +++ b/server/www/scripts/statusReport/partials/modalEdit.html @@ -145,9 +145,10 @@
    Confirmed

    Impact

    -
    -

    {{key}}

    -
    +

    + {{key}} +

    +

    Evidence

    diff --git a/server/www/scripts/statusReport/partials/modalNew.html b/server/www/scripts/statusReport/partials/modalNew.html index 5fff16af5e8..63a3d4bb9db 100644 --- a/server/www/scripts/statusReport/partials/modalNew.html +++ b/server/www/scripts/statusReport/partials/modalNew.html @@ -218,10 +218,10 @@
    Ease of Resolution

    Impact

    -
    -

    {{key}}

    -
    - +

    + {{key}} +

    +

    Evidence

    -

    Vuln edit

    +

    Vulnerability edit

    From 7e28aa8918fad27ba3d83af4bc26731534dc3e30 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Thu, 14 Jun 2018 17:15:32 -0300 Subject: [PATCH 1356/1506] Fix status padding --- server/www/scripts/statusReport/partials/modalEdit.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/statusReport/partials/modalEdit.html b/server/www/scripts/statusReport/partials/modalEdit.html index 7d409733474..aa1dd01cc4f 100644 --- a/server/www/scripts/statusReport/partials/modalEdit.html +++ b/server/www/scripts/statusReport/partials/modalEdit.html @@ -36,7 +36,7 @@
    Ease of Resolution
    Status
    -
    From 7fce18d32a5ffd5c1140b986822f7b8c316e1f4a Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Thu, 14 Jun 2018 17:42:13 -0300 Subject: [PATCH 1357/1506] Fix padding modal new status report --- server/www/scripts/statusReport/partials/modalNew.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/statusReport/partials/modalNew.html b/server/www/scripts/statusReport/partials/modalNew.html index 63a3d4bb9db..13e880c77a5 100644 --- a/server/www/scripts/statusReport/partials/modalNew.html +++ b/server/www/scripts/statusReport/partials/modalNew.html @@ -92,7 +92,7 @@
    Search vulnerability database templates
    Type
    -
    @@ -106,7 +106,7 @@
    Severity
    Ease of Resolution
    -
    From 72c40c15f66337495d65c43590d816e48a61dbd9 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Thu, 14 Jun 2018 17:44:00 -0300 Subject: [PATCH 1358/1506] Fix padding modal edit status report --- server/www/scripts/statusReport/partials/modalEdit.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/statusReport/partials/modalEdit.html b/server/www/scripts/statusReport/partials/modalEdit.html index aa1dd01cc4f..1b689079016 100644 --- a/server/www/scripts/statusReport/partials/modalEdit.html +++ b/server/www/scripts/statusReport/partials/modalEdit.html @@ -30,7 +30,7 @@
    Severity
    Ease of Resolution
    -
    From 8621d11be1c9f352741ab322be43275f294e93f2 Mon Sep 17 00:00:00 2001 From: fedek Date: Thu, 14 Jun 2018 17:47:30 -0300 Subject: [PATCH 1359/1506] Added class 'date-cell' for dates in the host list partial --- server/www/scripts/hosts/partials/list.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/hosts/partials/list.html b/server/www/scripts/hosts/partials/list.html index bd4fd9ad24d..1908de7d21e 100644 --- a/server/www/scripts/hosts/partials/list.html +++ b/server/www/scripts/hosts/partials/list.html @@ -198,14 +198,14 @@ {{host.description}}
    -
    From e0d08da2ff95d036529f8efcf18a2c8e1cbf6446 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 14 Jun 2018 20:47:36 +0000 Subject: [PATCH 1360/1506] [FIX] PEP8 and create host even if no vulns were found --- plugins/repo/xsssniper/plugin.py | 87 +++++++++++++++----------------- 1 file changed, 42 insertions(+), 45 deletions(-) diff --git a/plugins/repo/xsssniper/plugin.py b/plugins/repo/xsssniper/plugin.py index d2c09f0c829..fef691d896e 100644 --- a/plugins/repo/xsssniper/plugin.py +++ b/plugins/repo/xsssniper/plugin.py @@ -7,65 +7,62 @@ See the file 'doc/LICENSE' for the license information ''' -from plugins import core import re import socket +from plugins import core __author__ = "Roberto Focke" __copyright__ = "Copyright (c) 2017, Infobyte LLC" __license__ = "" __version__ = "1.0.0" + class xsssniper (core.PluginBase): def __init__(self): - - core.PluginBase.__init__(self) - self.id = "xsssniper" - self.name = "xsssniper" - self.plugin_version = "0.0.1" - self.version = "1.0.0" - self.protocol="tcp" - self._command_regex = re.compile(r'^(sudo xsssniper|xsssniper|sudo xsssniper\.py|xsssniper\.py|sudo python xsssniper\.py|.\/xsssniper\.py|python xsssniper\.py)') + core.PluginBase.__init__(self) + self.id = "xsssniper" + self.name = "xsssniper" + self.plugin_version = "0.0.1" + self.version = "1.0.0" + self.protocol="tcp" + self._command_regex = re.compile(r'^(sudo xsssniper|xsssniper|sudo xsssniper\.py|xsssniper\.py|sudo python xsssniper\.py|.\/xsssniper\.py|python xsssniper\.py)') def parseOutputString(self, output, debug=False): - j=0 - parametro=[] - lineas = output.split("\n") - for linea in lineas: - if ((linea.find("--[!] Target:")>0)): - url = re.findall('(?:[-\w.]|(?:%[\da-fA-F]{2}))+', linea) - print url - - - - if ((linea.find("Method")>0)): - list = re.findall("\w+", linea) - print list[1] - metodo= list[1] - if ((linea.find("Query String:")>0)): - lista_parametros=linea.split('=') - print lista_parametros - aux=len(lista_parametros) - print aux - if ((linea.find("--[!] Param:")>0)): - list2= re.findall("\w+",linea) - print list2[1] - parametro.append(list2[1]) - host_id = self.createAndAddHost(url[3]) - address=socket.gethostbyname(url[3]) - interface_id = self.createAndAddInterface(host_id,address,ipv4_address=address,hostname_resolution=url[3]) - service_id = self.createAndAddServiceToInterface(host_id,interface_id,self.protocol,'tcp',ports=['80'],status='Open',version="", description="") - if aux !=0: - print "entro al if" - print parametro - - self.createAndAddVulnWebToService(host_id,service_id,name="xss",desc="XSS",ref='',severity='med',website=url[0],path='',method=metodo,pname='',params=parametro,request='',response='') - - - def processCommandString(self, username, current_path, command_string): - return None + parametro=[] + lineas = output.split("\n") + aux = 0 + for linea in lineas: + if not linea: + continue + linea = linea.lower() + if ((linea.find("target:")>0)): + url = re.findall('(?:[-\w.]|(?:%[\da-fA-F]{2}))+', linea) + host_id = self.createAndAddHost(url[3]) + address=socket.gethostbyname(url[3]) + interface_id = self.createAndAddInterface(host_id,address,ipv4_address=address,hostname_resolution=url[3]) + if ((linea.find("method")>0)): + list_a = re.findall("\w+", linea) + metodo= list_a[1] + if ((linea.find("query string:")>0)): + lista_parametros=linea.split('=') + aux=len(lista_parametros) + if ((linea.find("param:")>0)): + list2= re.findall("\w+",linea) + parametro.append(list2[1]) + service_id = self.createAndAddServiceToInterface(host_id,interface_id,self.protocol,'tcp',ports=['80'],status='Open',version="", description="") + if aux !=0: + self.createAndAddVulnWebToService(host_id,service_id,name="xss",desc="XSS",ref='',severity='med',website=url[0],path='',method=metodo,pname='',params=''.join(parametro),request='',response='') + + def processCommandString(self, username, current_path, command_string): + return None def createPlugin(): return xsssniper() + + +if __name__ == '__main__': + plugin_xss = xsssniper() + with open('xsssniper_out', 'r') as xsssniper_file: + plugin_xss.parseOutputString(xsssniper_file.read()) From fc7b736adcf3fcecd2b7a66d82f600ea6e617ca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 14 Jun 2018 18:21:07 -0300 Subject: [PATCH 1361/1506] Disgusting fix to the vulns counter of the header It wasn't working well when filter vulns, due to the innecesary use of global variables and function calls from angular partials. It is horrible. --- server/www/scripts/commons/controllers/headerCtrl.js | 9 ++++++--- server/www/scripts/commons/partials/header.html | 2 +- server/www/scripts/vulns/providers/vulns.js | 3 ++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/server/www/scripts/commons/controllers/headerCtrl.js b/server/www/scripts/commons/controllers/headerCtrl.js index 9212d80c6b3..c9a4e5e7db7 100644 --- a/server/www/scripts/commons/controllers/headerCtrl.js +++ b/server/www/scripts/commons/controllers/headerCtrl.js @@ -17,9 +17,12 @@ angular.module('faradayApp') return noSwitcher.indexOf($scope.component) < 0; }; - $scope.getVulnsNum = function() { - return vulnsManager.getVulnsNum($routeParams.wsId); - }; + // Ugly, ugly, ugly hack + $scope.vulnsNum = vulnsManager.getVulnsNum($routeParams.wsId); + setInterval(function(){ + $scope.vulnsNum = vulnsManager.getVulnsNum($routeParams.wsId); + $scope.$apply(); + }, 500) $scope.toggleConfirmed = function() { $scope.confirmed = !$scope.confirmed; diff --git a/server/www/scripts/commons/partials/header.html b/server/www/scripts/commons/partials/header.html index 67bd227ab0d..c33f0a8615d 100644 --- a/server/www/scripts/commons/partials/header.html +++ b/server/www/scripts/commons/partials/header.html @@ -25,7 +25,7 @@
    - {{getVulnsNum()}} vulns total + {{vulnsNum}} vulns total
    diff --git a/server/www/scripts/vulns/providers/vulns.js b/server/www/scripts/vulns/providers/vulns.js index a28195e2089..284970b59f1 100644 --- a/server/www/scripts/vulns/providers/vulns.js +++ b/server/www/scripts/vulns/providers/vulns.js @@ -75,7 +75,8 @@ angular.module('faradayApp') vulnsManager.loadVulnsCounter = function(ws){ // Ugly hack to populate the vulnsCounter global variable workspacesFact.get(ws).then(function(data){ - vulnsCounter = data.stats.total_vulns; + // Commenting this line worked. I'm not sure why + // vulnsCounter = data.stats.total_vulns; totalVulns = data.stats.total_vulns; }) }; From b3bbbc2d42b88b90d3de6162db9ea21328c9b1c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 14 Jun 2018 18:29:51 -0300 Subject: [PATCH 1362/1506] Add header to common workspace selector --- server/www/scripts/commons/partials/header.html | 2 +- server/www/scripts/commons/partials/workspaces.html | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/commons/partials/header.html b/server/www/scripts/commons/partials/header.html index c33f0a8615d..a44e2853ce7 100644 --- a/server/www/scripts/commons/partials/header.html +++ b/server/www/scripts/commons/partials/header.html @@ -1,5 +1,5 @@
    -
    +
    @@ -25,7 +25,8 @@

    Services for host {{hostName}}

    -
    +
    +
    From f50cc90484c96f2d36f2d458ffb9e1b0608bf773 Mon Sep 17 00:00:00 2001 From: fedek Date: Thu, 14 Jun 2018 18:56:32 -0300 Subject: [PATCH 1365/1506] Added header icon for missing sections, in case no icon is available section name is set --- server/www/scripts/commons/partials/header.html | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/commons/partials/header.html b/server/www/scripts/commons/partials/header.html index a44e2853ce7..ffc9ee0b188 100644 --- a/server/www/scripts/commons/partials/header.html +++ b/server/www/scripts/commons/partials/header.html @@ -11,9 +11,15 @@ Credentials Executive Report Tasks + Workspaces Comparison + Webshell Vulns -

    Licenses

    + Data Analysis

    Workspaces

    +

    Users

    +

    Licenses

    +

    Help

    +
    @@ -43,9 +49,15 @@ Credentials Executive Report Tasks + Workspaces Comparison + Webshell Vulns -

    Licenses

    + Data Analysis

    Workspaces

    +

    Users

    +

    Licenses

    +

    Help

    +
    From aef1f4f9780250f6941fb42ac6e4b5a9c9040114 Mon Sep 17 00:00:00 2001 From: fedek Date: Thu, 14 Jun 2018 19:00:24 -0300 Subject: [PATCH 1366/1506] Oops, header had a bad definition from previous commit --- server/www/scripts/commons/partials/header.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/commons/partials/header.html b/server/www/scripts/commons/partials/header.html index ffc9ee0b188..4346db648a2 100644 --- a/server/www/scripts/commons/partials/header.html +++ b/server/www/scripts/commons/partials/header.html @@ -56,7 +56,7 @@

    Workspaces

    Users

    Licenses

    -

    Help

    +

    Help

    From 8a265039611e74d78715d77f10c035513e2b5ffb Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Thu, 14 Jun 2018 19:20:05 -0300 Subject: [PATCH 1367/1506] Change ngClip to angular-clipboard, now "Copy to Clipboard" works! --- server/www/index.html | 3 +- server/www/script/ZeroClipboard.min.js | 9 -- server/www/script/ZeroClipboard.swf | Bin 4036 -> 0 bytes server/www/script/angular-clipboard.js | 101 ++++++++++++++++++ server/www/script/ngClip.js | 84 --------------- server/www/scripts/app.js | 7 +- .../controllers/summarizedCtrlHostsModal.js | 4 +- .../partials/modal-hosts-by-service.html | 2 +- 8 files changed, 108 insertions(+), 102 deletions(-) delete mode 100644 server/www/script/ZeroClipboard.min.js delete mode 100644 server/www/script/ZeroClipboard.swf create mode 100644 server/www/script/angular-clipboard.js delete mode 100644 server/www/script/ngClip.js diff --git a/server/www/index.html b/server/www/index.html index fb98c77883a..7d5f5fd27ec 100644 --- a/server/www/index.html +++ b/server/www/index.html @@ -69,12 +69,11 @@ - - + diff --git a/server/www/script/ZeroClipboard.min.js b/server/www/script/ZeroClipboard.min.js deleted file mode 100644 index 365737f559a..00000000000 --- a/server/www/script/ZeroClipboard.min.js +++ /dev/null @@ -1,9 +0,0 @@ -/*! - * ZeroClipboard - * The ZeroClipboard library provides an easy way to copy text to the clipboard using an invisible Adobe Flash movie and a JavaScript interface. - * Copyright (c) 2014 Jon Rohan, James M. Greene - * Licensed MIT - * http://zeroclipboard.org/ - * v2.1.6 - */ -!function(a,b){"use strict";var c,d,e=a,f=e.document,g=e.navigator,h=e.setTimeout,i=e.encodeURIComponent,j=e.ActiveXObject,k=e.Error,l=e.Number.parseInt||e.parseInt,m=e.Number.parseFloat||e.parseFloat,n=e.Number.isNaN||e.isNaN,o=e.Math.round,p=e.Date.now,q=e.Object.keys,r=e.Object.defineProperty,s=e.Object.prototype.hasOwnProperty,t=e.Array.prototype.slice,u=function(){var a=function(a){return a};if("function"==typeof e.wrap&&"function"==typeof e.unwrap)try{var b=f.createElement("div"),c=e.unwrap(b);1===b.nodeType&&c&&1===c.nodeType&&(a=e.unwrap)}catch(d){}return a}(),v=function(a){return t.call(a,0)},w=function(){var a,c,d,e,f,g,h=v(arguments),i=h[0]||{};for(a=1,c=h.length;c>a;a++)if(null!=(d=h[a]))for(e in d)s.call(d,e)&&(f=i[e],g=d[e],i!==g&&g!==b&&(i[e]=g));return i},x=function(a){var b,c,d,e;if("object"!=typeof a||null==a)b=a;else if("number"==typeof a.length)for(b=[],c=0,d=a.length;d>c;c++)s.call(a,c)&&(b[c]=x(a[c]));else{b={};for(e in a)s.call(a,e)&&(b[e]=x(a[e]))}return b},y=function(a,b){for(var c={},d=0,e=b.length;e>d;d++)b[d]in a&&(c[b[d]]=a[b[d]]);return c},z=function(a,b){var c={};for(var d in a)-1===b.indexOf(d)&&(c[d]=a[d]);return c},A=function(a){if(a)for(var b in a)s.call(a,b)&&delete a[b];return a},B=function(a,b){if(a&&1===a.nodeType&&a.ownerDocument&&b&&(1===b.nodeType&&b.ownerDocument&&b.ownerDocument===a.ownerDocument||9===b.nodeType&&!b.ownerDocument&&b===a.ownerDocument))do{if(a===b)return!0;a=a.parentNode}while(a);return!1},C=function(a){var b;return"string"==typeof a&&a&&(b=a.split("#")[0].split("?")[0],b=a.slice(0,a.lastIndexOf("/")+1)),b},D=function(a){var b,c;return"string"==typeof a&&a&&(c=a.match(/^(?:|[^:@]*@|.+\)@(?=http[s]?|file)|.+?\s+(?: at |@)(?:[^:\(]+ )*[\(]?)((?:http[s]?|file):\/\/[\/]?.+?\/[^:\)]*?)(?::\d+)(?::\d+)?/),c&&c[1]?b=c[1]:(c=a.match(/\)@((?:http[s]?|file):\/\/[\/]?.+?\/[^:\)]*?)(?::\d+)(?::\d+)?/),c&&c[1]&&(b=c[1]))),b},E=function(){var a,b;try{throw new k}catch(c){b=c}return b&&(a=b.sourceURL||b.fileName||D(b.stack)),a},F=function(){var a,c,d;if(f.currentScript&&(a=f.currentScript.src))return a;if(c=f.getElementsByTagName("script"),1===c.length)return c[0].src||b;if("readyState"in c[0])for(d=c.length;d--;)if("interactive"===c[d].readyState&&(a=c[d].src))return a;return"loading"===f.readyState&&(a=c[c.length-1].src)?a:(a=E())?a:b},G=function(){var a,c,d,e=f.getElementsByTagName("script");for(a=e.length;a--;){if(!(d=e[a].src)){c=null;break}if(d=C(d),null==c)c=d;else if(c!==d){c=null;break}}return c||b},H=function(){var a=C(F())||G()||"";return a+"ZeroClipboard.swf"},I={bridge:null,version:"0.0.0",pluginType:"unknown",disabled:null,outdated:null,unavailable:null,deactivated:null,overdue:null,ready:null},J="11.0.0",K={},L={},M=null,N={ready:"Flash communication is established",error:{"flash-disabled":"Flash is disabled or not installed","flash-outdated":"Flash is too outdated to support ZeroClipboard","flash-unavailable":"Flash is unable to communicate bidirectionally with JavaScript","flash-deactivated":"Flash is too outdated for your browser and/or is configured as click-to-activate","flash-overdue":"Flash communication was established but NOT within the acceptable time limit"}},O={swfPath:H(),trustedDomains:a.location.host?[a.location.host]:[],cacheBust:!0,forceEnhancedClipboard:!1,flashLoadTimeout:3e4,autoActivate:!0,bubbleEvents:!0,containerId:"global-zeroclipboard-html-bridge",containerClass:"global-zeroclipboard-container",swfObjectId:"global-zeroclipboard-flash-bridge",hoverClass:"zeroclipboard-is-hover",activeClass:"zeroclipboard-is-active",forceHandCursor:!1,title:null,zIndex:999999999},P=function(a){if("object"==typeof a&&null!==a)for(var b in a)if(s.call(a,b))if(/^(?:forceHandCursor|title|zIndex|bubbleEvents)$/.test(b))O[b]=a[b];else if(null==I.bridge)if("containerId"===b||"swfObjectId"===b){if(!cb(a[b]))throw new Error("The specified `"+b+"` value is not valid as an HTML4 Element ID");O[b]=a[b]}else O[b]=a[b];{if("string"!=typeof a||!a)return x(O);if(s.call(O,a))return O[a]}},Q=function(){return{browser:y(g,["userAgent","platform","appName"]),flash:z(I,["bridge"]),zeroclipboard:{version:Fb.version,config:Fb.config()}}},R=function(){return!!(I.disabled||I.outdated||I.unavailable||I.deactivated)},S=function(a,b){var c,d,e,f={};if("string"==typeof a&&a)e=a.toLowerCase().split(/\s+/);else if("object"==typeof a&&a&&"undefined"==typeof b)for(c in a)s.call(a,c)&&"string"==typeof c&&c&&"function"==typeof a[c]&&Fb.on(c,a[c]);if(e&&e.length){for(c=0,d=e.length;d>c;c++)a=e[c].replace(/^on/,""),f[a]=!0,K[a]||(K[a]=[]),K[a].push(b);if(f.ready&&I.ready&&Fb.emit({type:"ready"}),f.error){var g=["disabled","outdated","unavailable","deactivated","overdue"];for(c=0,d=g.length;d>c;c++)if(I[g[c]]===!0){Fb.emit({type:"error",name:"flash-"+g[c]});break}}}return Fb},T=function(a,b){var c,d,e,f,g;if(0===arguments.length)f=q(K);else if("string"==typeof a&&a)f=a.split(/\s+/);else if("object"==typeof a&&a&&"undefined"==typeof b)for(c in a)s.call(a,c)&&"string"==typeof c&&c&&"function"==typeof a[c]&&Fb.off(c,a[c]);if(f&&f.length)for(c=0,d=f.length;d>c;c++)if(a=f[c].toLowerCase().replace(/^on/,""),g=K[a],g&&g.length)if(b)for(e=g.indexOf(b);-1!==e;)g.splice(e,1),e=g.indexOf(b,e);else g.length=0;return Fb},U=function(a){var b;return b="string"==typeof a&&a?x(K[a])||null:x(K)},V=function(a){var b,c,d;return a=db(a),a&&!jb(a)?"ready"===a.type&&I.overdue===!0?Fb.emit({type:"error",name:"flash-overdue"}):(b=w({},a),ib.call(this,b),"copy"===a.type&&(d=pb(L),c=d.data,M=d.formatMap),c):void 0},W=function(){if("boolean"!=typeof I.ready&&(I.ready=!1),!Fb.isFlashUnusable()&&null===I.bridge){var a=O.flashLoadTimeout;"number"==typeof a&&a>=0&&h(function(){"boolean"!=typeof I.deactivated&&(I.deactivated=!0),I.deactivated===!0&&Fb.emit({type:"error",name:"flash-deactivated"})},a),I.overdue=!1,nb()}},X=function(){Fb.clearData(),Fb.blur(),Fb.emit("destroy"),ob(),Fb.off()},Y=function(a,b){var c;if("object"==typeof a&&a&&"undefined"==typeof b)c=a,Fb.clearData();else{if("string"!=typeof a||!a)return;c={},c[a]=b}for(var d in c)"string"==typeof d&&d&&s.call(c,d)&&"string"==typeof c[d]&&c[d]&&(L[d]=c[d])},Z=function(a){"undefined"==typeof a?(A(L),M=null):"string"==typeof a&&s.call(L,a)&&delete L[a]},$=function(a){return"undefined"==typeof a?x(L):"string"==typeof a&&s.call(L,a)?L[a]:void 0},_=function(a){if(a&&1===a.nodeType){c&&(xb(c,O.activeClass),c!==a&&xb(c,O.hoverClass)),c=a,wb(a,O.hoverClass);var b=a.getAttribute("title")||O.title;if("string"==typeof b&&b){var d=mb(I.bridge);d&&d.setAttribute("title",b)}var e=O.forceHandCursor===!0||"pointer"===yb(a,"cursor");Cb(e),Bb()}},ab=function(){var a=mb(I.bridge);a&&(a.removeAttribute("title"),a.style.left="0px",a.style.top="-9999px",a.style.width="1px",a.style.top="1px"),c&&(xb(c,O.hoverClass),xb(c,O.activeClass),c=null)},bb=function(){return c||null},cb=function(a){return"string"==typeof a&&a&&/^[A-Za-z][A-Za-z0-9_:\-\.]*$/.test(a)},db=function(a){var b;if("string"==typeof a&&a?(b=a,a={}):"object"==typeof a&&a&&"string"==typeof a.type&&a.type&&(b=a.type),b){!a.target&&/^(copy|aftercopy|_click)$/.test(b.toLowerCase())&&(a.target=d),w(a,{type:b.toLowerCase(),target:a.target||c||null,relatedTarget:a.relatedTarget||null,currentTarget:I&&I.bridge||null,timeStamp:a.timeStamp||p()||null});var e=N[a.type];return"error"===a.type&&a.name&&e&&(e=e[a.name]),e&&(a.message=e),"ready"===a.type&&w(a,{target:null,version:I.version}),"error"===a.type&&(/^flash-(disabled|outdated|unavailable|deactivated|overdue)$/.test(a.name)&&w(a,{target:null,minimumVersion:J}),/^flash-(outdated|unavailable|deactivated|overdue)$/.test(a.name)&&w(a,{version:I.version})),"copy"===a.type&&(a.clipboardData={setData:Fb.setData,clearData:Fb.clearData}),"aftercopy"===a.type&&(a=qb(a,M)),a.target&&!a.relatedTarget&&(a.relatedTarget=eb(a.target)),a=fb(a)}},eb=function(a){var b=a&&a.getAttribute&&a.getAttribute("data-clipboard-target");return b?f.getElementById(b):null},fb=function(a){if(a&&/^_(?:click|mouse(?:over|out|down|up|move))$/.test(a.type)){var c=a.target,d="_mouseover"===a.type&&a.relatedTarget?a.relatedTarget:b,g="_mouseout"===a.type&&a.relatedTarget?a.relatedTarget:b,h=Ab(c),i=e.screenLeft||e.screenX||0,j=e.screenTop||e.screenY||0,k=f.body.scrollLeft+f.documentElement.scrollLeft,l=f.body.scrollTop+f.documentElement.scrollTop,m=h.left+("number"==typeof a._stageX?a._stageX:0),n=h.top+("number"==typeof a._stageY?a._stageY:0),o=m-k,p=n-l,q=i+o,r=j+p,s="number"==typeof a.movementX?a.movementX:0,t="number"==typeof a.movementY?a.movementY:0;delete a._stageX,delete a._stageY,w(a,{srcElement:c,fromElement:d,toElement:g,screenX:q,screenY:r,pageX:m,pageY:n,clientX:o,clientY:p,x:o,y:p,movementX:s,movementY:t,offsetX:0,offsetY:0,layerX:0,layerY:0})}return a},gb=function(a){var b=a&&"string"==typeof a.type&&a.type||"";return!/^(?:(?:before)?copy|destroy)$/.test(b)},hb=function(a,b,c,d){d?h(function(){a.apply(b,c)},0):a.apply(b,c)},ib=function(a){if("object"==typeof a&&a&&a.type){var b=gb(a),c=K["*"]||[],d=K[a.type]||[],f=c.concat(d);if(f&&f.length){var g,h,i,j,k,l=this;for(g=0,h=f.length;h>g;g++)i=f[g],j=l,"string"==typeof i&&"function"==typeof e[i]&&(i=e[i]),"object"==typeof i&&i&&"function"==typeof i.handleEvent&&(j=i,i=i.handleEvent),"function"==typeof i&&(k=w({},a),hb(i,j,[k],b))}return this}},jb=function(a){var b=a.target||c||null,e="swf"===a._source;delete a._source;var f=["flash-disabled","flash-outdated","flash-unavailable","flash-deactivated","flash-overdue"];switch(a.type){case"error":-1!==f.indexOf(a.name)&&w(I,{disabled:"flash-disabled"===a.name,outdated:"flash-outdated"===a.name,unavailable:"flash-unavailable"===a.name,deactivated:"flash-deactivated"===a.name,overdue:"flash-overdue"===a.name,ready:!1});break;case"ready":var g=I.deactivated===!0;w(I,{disabled:!1,outdated:!1,unavailable:!1,deactivated:!1,overdue:g,ready:!g});break;case"beforecopy":d=b;break;case"copy":var h,i,j=a.relatedTarget;!L["text/html"]&&!L["text/plain"]&&j&&(i=j.value||j.outerHTML||j.innerHTML)&&(h=j.value||j.textContent||j.innerText)?(a.clipboardData.clearData(),a.clipboardData.setData("text/plain",h),i!==h&&a.clipboardData.setData("text/html",i)):!L["text/plain"]&&a.target&&(h=a.target.getAttribute("data-clipboard-text"))&&(a.clipboardData.clearData(),a.clipboardData.setData("text/plain",h));break;case"aftercopy":Fb.clearData(),b&&b!==vb()&&b.focus&&b.focus();break;case"_mouseover":Fb.focus(b),O.bubbleEvents===!0&&e&&(b&&b!==a.relatedTarget&&!B(a.relatedTarget,b)&&kb(w({},a,{type:"mouseenter",bubbles:!1,cancelable:!1})),kb(w({},a,{type:"mouseover"})));break;case"_mouseout":Fb.blur(),O.bubbleEvents===!0&&e&&(b&&b!==a.relatedTarget&&!B(a.relatedTarget,b)&&kb(w({},a,{type:"mouseleave",bubbles:!1,cancelable:!1})),kb(w({},a,{type:"mouseout"})));break;case"_mousedown":wb(b,O.activeClass),O.bubbleEvents===!0&&e&&kb(w({},a,{type:a.type.slice(1)}));break;case"_mouseup":xb(b,O.activeClass),O.bubbleEvents===!0&&e&&kb(w({},a,{type:a.type.slice(1)}));break;case"_click":d=null,O.bubbleEvents===!0&&e&&kb(w({},a,{type:a.type.slice(1)}));break;case"_mousemove":O.bubbleEvents===!0&&e&&kb(w({},a,{type:a.type.slice(1)}))}return/^_(?:click|mouse(?:over|out|down|up|move))$/.test(a.type)?!0:void 0},kb=function(a){if(a&&"string"==typeof a.type&&a){var b,c=a.target||null,d=c&&c.ownerDocument||f,g={view:d.defaultView||e,canBubble:!0,cancelable:!0,detail:"click"===a.type?1:0,button:"number"==typeof a.which?a.which-1:"number"==typeof a.button?a.button:d.createEvent?0:1},h=w(g,a);c&&d.createEvent&&c.dispatchEvent&&(h=[h.type,h.canBubble,h.cancelable,h.view,h.detail,h.screenX,h.screenY,h.clientX,h.clientY,h.ctrlKey,h.altKey,h.shiftKey,h.metaKey,h.button,h.relatedTarget],b=d.createEvent("MouseEvents"),b.initMouseEvent&&(b.initMouseEvent.apply(b,h),b._source="js",c.dispatchEvent(b)))}},lb=function(){var a=f.createElement("div");return a.id=O.containerId,a.className=O.containerClass,a.style.position="absolute",a.style.left="0px",a.style.top="-9999px",a.style.width="1px",a.style.height="1px",a.style.zIndex=""+Db(O.zIndex),a},mb=function(a){for(var b=a&&a.parentNode;b&&"OBJECT"===b.nodeName&&b.parentNode;)b=b.parentNode;return b||null},nb=function(){var a,b=I.bridge,c=mb(b);if(!b){var d=ub(e.location.host,O),g="never"===d?"none":"all",h=sb(O),i=O.swfPath+rb(O.swfPath,O);c=lb();var j=f.createElement("div");c.appendChild(j),f.body.appendChild(c);var k=f.createElement("div"),l="activex"===I.pluginType;k.innerHTML='"+(l?'':"")+'',b=k.firstChild,k=null,u(b).ZeroClipboard=Fb,c.replaceChild(b,j)}return b||(b=f[O.swfObjectId],b&&(a=b.length)&&(b=b[a-1]),!b&&c&&(b=c.firstChild)),I.bridge=b||null,b},ob=function(){var a=I.bridge;if(a){var b=mb(a);b&&("activex"===I.pluginType&&"readyState"in a?(a.style.display="none",function c(){if(4===a.readyState){for(var d in a)"function"==typeof a[d]&&(a[d]=null);a.parentNode&&a.parentNode.removeChild(a),b.parentNode&&b.parentNode.removeChild(b)}else h(c,10)}()):(a.parentNode&&a.parentNode.removeChild(a),b.parentNode&&b.parentNode.removeChild(b))),I.ready=null,I.bridge=null,I.deactivated=null}},pb=function(a){var b={},c={};if("object"==typeof a&&a){for(var d in a)if(d&&s.call(a,d)&&"string"==typeof a[d]&&a[d])switch(d.toLowerCase()){case"text/plain":case"text":case"air:text":case"flash:text":b.text=a[d],c.text=d;break;case"text/html":case"html":case"air:html":case"flash:html":b.html=a[d],c.html=d;break;case"application/rtf":case"text/rtf":case"rtf":case"richtext":case"air:rtf":case"flash:rtf":b.rtf=a[d],c.rtf=d}return{data:b,formatMap:c}}},qb=function(a,b){if("object"!=typeof a||!a||"object"!=typeof b||!b)return a;var c={};for(var d in a)if(s.call(a,d)){if("success"!==d&&"data"!==d){c[d]=a[d];continue}c[d]={};var e=a[d];for(var f in e)f&&s.call(e,f)&&s.call(b,f)&&(c[d][b[f]]=e[f])}return c},rb=function(a,b){var c=null==b||b&&b.cacheBust===!0;return c?(-1===a.indexOf("?")?"?":"&")+"noCache="+p():""},sb=function(a){var b,c,d,f,g="",h=[];if(a.trustedDomains&&("string"==typeof a.trustedDomains?f=[a.trustedDomains]:"object"==typeof a.trustedDomains&&"length"in a.trustedDomains&&(f=a.trustedDomains)),f&&f.length)for(b=0,c=f.length;c>b;b++)if(s.call(f,b)&&f[b]&&"string"==typeof f[b]){if(d=tb(f[b]),!d)continue;if("*"===d){h.length=0,h.push(d);break}h.push.apply(h,[d,"//"+d,e.location.protocol+"//"+d])}return h.length&&(g+="trustedOrigins="+i(h.join(","))),a.forceEnhancedClipboard===!0&&(g+=(g?"&":"")+"forceEnhancedClipboard=true"),"string"==typeof a.swfObjectId&&a.swfObjectId&&(g+=(g?"&":"")+"swfObjectId="+i(a.swfObjectId)),g},tb=function(a){if(null==a||""===a)return null;if(a=a.replace(/^\s+|\s+$/g,""),""===a)return null;var b=a.indexOf("//");a=-1===b?a:a.slice(b+2);var c=a.indexOf("/");return a=-1===c?a:-1===b||0===c?null:a.slice(0,c),a&&".swf"===a.slice(-4).toLowerCase()?null:a||null},ub=function(){var a=function(a){var b,c,d,e=[];if("string"==typeof a&&(a=[a]),"object"!=typeof a||!a||"number"!=typeof a.length)return e;for(b=0,c=a.length;c>b;b++)if(s.call(a,b)&&(d=tb(a[b]))){if("*"===d){e.length=0,e.push("*");break}-1===e.indexOf(d)&&e.push(d)}return e};return function(b,c){var d=tb(c.swfPath);null===d&&(d=b);var e=a(c.trustedDomains),f=e.length;if(f>0){if(1===f&&"*"===e[0])return"always";if(-1!==e.indexOf(b))return 1===f&&b===d?"sameDomain":"always"}return"never"}}(),vb=function(){try{return f.activeElement}catch(a){return null}},wb=function(a,b){if(!a||1!==a.nodeType)return a;if(a.classList)return a.classList.contains(b)||a.classList.add(b),a;if(b&&"string"==typeof b){var c=(b||"").split(/\s+/);if(1===a.nodeType)if(a.className){for(var d=" "+a.className+" ",e=a.className,f=0,g=c.length;g>f;f++)d.indexOf(" "+c[f]+" ")<0&&(e+=" "+c[f]);a.className=e.replace(/^\s+|\s+$/g,"")}else a.className=b}return a},xb=function(a,b){if(!a||1!==a.nodeType)return a;if(a.classList)return a.classList.contains(b)&&a.classList.remove(b),a;if("string"==typeof b&&b){var c=b.split(/\s+/);if(1===a.nodeType&&a.className){for(var d=(" "+a.className+" ").replace(/[\n\t]/g," "),e=0,f=c.length;f>e;e++)d=d.replace(" "+c[e]+" "," ");a.className=d.replace(/^\s+|\s+$/g,"")}}return a},yb=function(a,b){var c=e.getComputedStyle(a,null).getPropertyValue(b);return"cursor"!==b||c&&"auto"!==c||"A"!==a.nodeName?c:"pointer"},zb=function(){var a,b,c,d=1;return"function"==typeof f.body.getBoundingClientRect&&(a=f.body.getBoundingClientRect(),b=a.right-a.left,c=f.body.offsetWidth,d=o(b/c*100)/100),d},Ab=function(a){var b={left:0,top:0,width:0,height:0};if(a.getBoundingClientRect){var c,d,g,h=a.getBoundingClientRect();"pageXOffset"in e&&"pageYOffset"in e?(c=e.pageXOffset,d=e.pageYOffset):(g=zb(),c=o(f.documentElement.scrollLeft/g),d=o(f.documentElement.scrollTop/g));var i=f.documentElement.clientLeft||0,j=f.documentElement.clientTop||0;b.left=h.left+c-i,b.top=h.top+d-j,b.width="width"in h?h.width:h.right-h.left,b.height="height"in h?h.height:h.bottom-h.top}return b},Bb=function(){var a;if(c&&(a=mb(I.bridge))){var b=Ab(c);w(a.style,{width:b.width+"px",height:b.height+"px",top:b.top+"px",left:b.left+"px",zIndex:""+Db(O.zIndex)})}},Cb=function(a){I.ready===!0&&(I.bridge&&"function"==typeof I.bridge.setHandCursor?I.bridge.setHandCursor(a):I.ready=!1)},Db=function(a){if(/^(?:auto|inherit)$/.test(a))return a;var b;return"number"!=typeof a||n(a)?"string"==typeof a&&(b=Db(l(a,10))):b=a,"number"==typeof b?b:"auto"},Eb=function(a){function b(a){var b=a.match(/[\d]+/g);return b.length=3,b.join(".")}function c(a){return!!a&&(a=a.toLowerCase())&&(/^(pepflashplayer\.dll|libpepflashplayer\.so|pepperflashplayer\.plugin)$/.test(a)||"chrome.plugin"===a.slice(-13))}function d(a){a&&(i=!0,a.version&&(l=b(a.version)),!l&&a.description&&(l=b(a.description)),a.filename&&(k=c(a.filename)))}var e,f,h,i=!1,j=!1,k=!1,l="";if(g.plugins&&g.plugins.length)e=g.plugins["Shockwave Flash"],d(e),g.plugins["Shockwave Flash 2.0"]&&(i=!0,l="2.0.0.11");else if(g.mimeTypes&&g.mimeTypes.length)h=g.mimeTypes["application/x-shockwave-flash"],e=h&&h.enabledPlugin,d(e);else if("undefined"!=typeof a){j=!0;try{f=new a("ShockwaveFlash.ShockwaveFlash.7"),i=!0,l=b(f.GetVariable("$version"))}catch(n){try{f=new a("ShockwaveFlash.ShockwaveFlash.6"),i=!0,l="6.0.21"}catch(o){try{f=new a("ShockwaveFlash.ShockwaveFlash"),i=!0,l=b(f.GetVariable("$version"))}catch(p){j=!1}}}}I.disabled=i!==!0,I.outdated=l&&m(l)c;c++)a=e[c].replace(/^on/,""),f[a]=!0,g[a]||(g[a]=[]),g[a].push(b);if(f.ready&&I.ready&&this.emit({type:"ready",client:this}),f.error){var h=["disabled","outdated","unavailable","deactivated","overdue"];for(c=0,d=h.length;d>c;c++)if(I[h[c]]){this.emit({type:"error",name:"flash-"+h[c],client:this});break}}}return this},Nb=function(a,b){var c,d,e,f,g,h=Hb[this.id]&&Hb[this.id].handlers;if(0===arguments.length)f=q(h);else if("string"==typeof a&&a)f=a.split(/\s+/);else if("object"==typeof a&&a&&"undefined"==typeof b)for(c in a)s.call(a,c)&&"string"==typeof c&&c&&"function"==typeof a[c]&&this.off(c,a[c]);if(f&&f.length)for(c=0,d=f.length;d>c;c++)if(a=f[c].toLowerCase().replace(/^on/,""),g=h[a],g&&g.length)if(b)for(e=g.indexOf(b);-1!==e;)g.splice(e,1),e=g.indexOf(b,e);else g.length=0;return this},Ob=function(a){var b=null,c=Hb[this.id]&&Hb[this.id].handlers;return c&&(b="string"==typeof a&&a?c[a]?c[a].slice(0):[]:x(c)),b},Pb=function(a){if(Ub.call(this,a)){"object"==typeof a&&a&&"string"==typeof a.type&&a.type&&(a=w({},a));var b=w({},db(a),{client:this});Vb.call(this,b)}return this},Qb=function(a){a=Wb(a);for(var b=0;b0,d=!a.target||c&&-1!==b.indexOf(a.target),e=a.relatedTarget&&c&&-1!==b.indexOf(a.relatedTarget),f=a.client&&a.client===this;return d||e||f?!0:!1},Vb=function(a){if("object"==typeof a&&a&&a.type){var b=gb(a),c=Hb[this.id]&&Hb[this.id].handlers["*"]||[],d=Hb[this.id]&&Hb[this.id].handlers[a.type]||[],f=c.concat(d);if(f&&f.length){var g,h,i,j,k,l=this;for(g=0,h=f.length;h>g;g++)i=f[g],j=l,"string"==typeof i&&"function"==typeof e[i]&&(i=e[i]),"object"==typeof i&&i&&"function"==typeof i.handleEvent&&(j=i,i=i.handleEvent),"function"==typeof i&&(k=w({},a),hb(i,j,[k],b))}return this}},Wb=function(a){return"string"==typeof a&&(a=[]),"number"!=typeof a.length?[a]:a},Xb=function(a){if(a&&1===a.nodeType){var b=function(a){(a||(a=e.event))&&("js"!==a._source&&(a.stopImmediatePropagation(),a.preventDefault()),delete a._source)},c=function(c){(c||(c=e.event))&&(b(c),Fb.focus(a))};a.addEventListener("mouseover",c,!1),a.addEventListener("mouseout",b,!1),a.addEventListener("mouseenter",b,!1),a.addEventListener("mouseleave",b,!1),a.addEventListener("mousemove",b,!1),Kb[a.zcClippingId]={mouseover:c,mouseout:b,mouseenter:b,mouseleave:b,mousemove:b}}},Yb=function(a){if(a&&1===a.nodeType){var b=Kb[a.zcClippingId];if("object"==typeof b&&b){for(var c,d,e=["move","leave","enter","out","over"],f=0,g=e.length;g>f;f++)c="mouse"+e[f],d=b[c],"function"==typeof d&&a.removeEventListener(c,d,!1);delete Kb[a.zcClippingId]}}};Fb._createClient=function(){Lb.apply(this,v(arguments))},Fb.prototype.on=function(){return Mb.apply(this,v(arguments))},Fb.prototype.off=function(){return Nb.apply(this,v(arguments))},Fb.prototype.handlers=function(){return Ob.apply(this,v(arguments))},Fb.prototype.emit=function(){return Pb.apply(this,v(arguments))},Fb.prototype.clip=function(){return Qb.apply(this,v(arguments))},Fb.prototype.unclip=function(){return Rb.apply(this,v(arguments))},Fb.prototype.elements=function(){return Sb.apply(this,v(arguments))},Fb.prototype.destroy=function(){return Tb.apply(this,v(arguments))},Fb.prototype.setText=function(a){return Fb.setData("text/plain",a),this},Fb.prototype.setHtml=function(a){return Fb.setData("text/html",a),this},Fb.prototype.setRichText=function(a){return Fb.setData("application/rtf",a),this},Fb.prototype.setData=function(){return Fb.setData.apply(this,v(arguments)),this},Fb.prototype.clearData=function(){return Fb.clearData.apply(this,v(arguments)),this},Fb.prototype.getData=function(){return Fb.getData.apply(this,v(arguments))},"function"==typeof define&&define.amd?define(function(){return Fb}):"object"==typeof module&&module&&"object"==typeof module.exports&&module.exports?module.exports=Fb:a.ZeroClipboard=Fb}(function(){return this||window}()); diff --git a/server/www/script/ZeroClipboard.swf b/server/www/script/ZeroClipboard.swf deleted file mode 100644 index d4e2561b366e131d3bae303acce90a137a4956e1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4036 zcmV;#4?FNfS5ppe82|uy+Ko8*dmG1fGkXno36KCtQ4~c|ON%BYdPzcuWs8(#QzRjg zmIR51E{i4>z$~%SVi($7@UW~{Q6eW!(z{mg7ET*CZPcds-K0&AAKLHh2a=8WwLkW! z{uk<-UGS3Ae1SMSZ{EE3=H0Wa6(sx*LXDpx)P~V`;s8SE!{&d-2%T{Y#_;rbT3snw zwl@r`vwcP1FAon5EiW$*E}s~5+{K||r%#_AN*y0MetZBZ2E0|<*H;E??{MF_K)^Wl z@~&C-O~+Q*TF*HZ|7>4hU1}k}Ewo&5tw3ZUKSV8BqPFi19UD9bf(rRz!*NTxe@-u# zEi;8)-=gP63r>!zX^XC?C)Lp%{(w3pVZ?}njDq!Bt!bFrSIm`5l)IC?R@7d0Bc*G8t(d1pK`_5j9 zEgajobbGO)FVb@pSAxCmb)ci}({oc5OFjOErXEinI~l6MuxEx|@^&2q0X&Ds#gs$6y|9-UyY zl*!$td0(3GUDIBSq$_rwsV+Vw-x> zW^x@@2p*n~D+JzP-*e>dJo))%MlgvQ3=5V@d z>83qPSNx1^ttLa<7R$Kq>yewDH|Lv{M~b?~5OJ6p?KWehr-OOs`!sZ7!Z0mx(yWta zF2An3p6L0IKVo2Er!A_gHQm*3rjvcG}V65VLj@#_g zlCwnj5Gt0V7pOaF8;$~Qh3tSsz3xTJ$?4X>`w+S9sD!mSkaNw#B5m-NjWFnwh3Egsnx#PWeM+a`}1Mkn*=hVRIh2c8`cLwMCdjy|) zexvVJSPo@ebJ4Uttch?Eeh`s5YDBrE zs-~eH@mI^#G1NT;sGdCwlD1ClQ+s>WA4N9Es_G?LrquyG+RzbBLU$q>#y68(JQ#18gci*vlASVH$1+{*Is*~fxYM=U=y30Eqs_Ig|;@Y5f zVAf9hj40g0qcM;AJJm13X09$=nn=%zI$JgdJ2TKJ>1C%t<%yf4W3vnC$ywRk!bJm;YZajPSB3Kv zGqdD=xQHyedbw!kJvm3=n6znGO3>qOm<7Ms5P%n{xmffg1y^652D{%wRzb;8C^4FM%Buo<$hrZofiVmAp}}`9Im^@?(>=r9q)=r73H6BkIGWVo)iV#%Vpvw{8*%oDP*COwDAj$YFyvjaA8__M%_p zG&M}nL7Vz9$Xas|ifGoEo4%yLn5OqD(5D;jf;QpH(=a~59UT!DJZc$Is2Q~K6<;X& zC5v}`BRZy+^_&U0Xi|@JAnPHnRJ2UHvLej{EjQiAC zyL^7tr{N1v{Z69ZJ>MzI+wAd0cH`>SHbTt9o0c2jUZ;asUaw(x;wBVPX8Q8zEP?ac zi47NLFJG$TV$1CMoSRLv6TSHK_bmI+n}ZXh^H#2679iPS{IIFS-WOcJq~ zh%H3iN5obl?k8d!i5wu2b`m*AB1wWf2<{}fi{L{rzArC>@D&pK3O)$H zGKqbR<*#D^zX9W?VEitjK2l=?_mCW7~g~OeHi}$;~#@?%}()J_Oiz z0uH_l*b_c7FcuKvLq`=I0IYE4c{p+F0aOvJ zC>YmgQDp4!XIt<*fKjX+qsD_6HFaPV@5Ct4D9SQw6}y-*hcMc& zV${}+(ScVmYCkN>4d@W)(SVw{o&?|WYC_o8!SyEi&c1}uB`8=HFiPSm=y(t}3W!tj zVF7`wF^r87t4xLOVuV(Ez^Vm!|{OWsHL(;k`V;RIEsU1wd_%&zh=ZUuW>-b z^Pp}dtF84u(+6NJ2%Hjq3=(hj|1d?d`baEcHt{8a;_k;Kku4%6Xl3AToPgJ98e`EHQloCuK&!7EkIz zPVN?RG7xfjmynYnM0+d!I%aBRPZ@7yPXS)ayb0WWB|e}zt)-v!C*&<5r+c3zMJA%R z#MiV7+QllMxvDl<)uyW2J5}vcRl8i(u2i*5Rl8c%u2r?^sx}i~W~%pr zs;*F6OD7O>QEk2vB=J*?SdZ1_qfF{l6zdH&qaodnk{#fEzzLas7%4xMjD13MI(rl&>6&YNJ9dT&LYtJ94iJ<k#eHA=7 zo1Q~e{pz~*HLPE+ts7U@L(63zVsWjOTFVN?jhfK{rNzLTJK(Jy@b(V)?hbgTW+ZpO z01KW_IH>Q#wG>5JF_oQ9W#3C>7gE`5D$B#)P1G6uiZbZXfH5zGogm6_g4hstg0;*6 z45%0{h>FXq`Z-kfS_#*(x{>RB)}Mq{cog6~IAedkKA0oe%+SxE_n}_B73j5Jg_vRJ z6Z_*W5VX{K>Ne`%>`%Q0()#p5XN0fSj8>2X(LY)<_CM9zF5r=im#Bdkq31@UjT{!l z1n9M!Y6;A2;z7M6Y_J@GZlBfH)}b;$@3;dcyKRSW7O0;_MLU^EEi5gNsv^e{G5P_K4GP zKp%%bBX4BQb;yUi>3g{PWHTr9;Oe?zMM0Ad?VGr|`2!&*0Ak;NzIZWTnzyyj@LE;- zU0mI$YTw4yDI06QhchK+OcJ8WH`P+}4vVIE6L%J)fNtK2TnoqX!gWZbPqg315FDPR zCz$;`kwb~?mZ!S7NO6{UwZzN z%3j*scp8dYXXmsEkbm6}&a7wnJ*Y=aS2(B~?T_)Zr^T)?D-W@(+|P4i+-pV`$PRVU zyI`k{mjrPo2wYJ7LG&Ky#rPps#bQqAda}6zjva&*1?1WyeKMfv3l7I>xJU=5|>>L zu8(qNg*LQeYc>dbu+qT)^>Hda1#!vOfMky!W7X0i=G9+Y+lj zdcf|dZRsBv4>Mmt;Sr&OBU?9Cc`5mNP?P~x_Cw%DcZM&wWXryOb<2dGimZ9-z=HE4+H$%$G3Z<{?^2NX{Ep5Be@^H@g)0 zazGRu?`|K@?LFRQUgb|uU#G^$_?!4MOpX7-pK{;gsKM`e`3Xk+19XrTTO5k*BOF#6 z*I8_Y%<9;wwoU=6ZJ?IYh67{jr2uNJ!5hgxVeOysc@F0ES1|V^4sVYdhRo=~z$8G4 q0(=NwVMQruv>;C1I`OOZ6Zy{KVOI9d?ElO6{y#btH2OaX)bM*lY1Xp< diff --git a/server/www/script/angular-clipboard.js b/server/www/script/angular-clipboard.js new file mode 100644 index 00000000000..ab976c0d7c4 --- /dev/null +++ b/server/www/script/angular-clipboard.js @@ -0,0 +1,101 @@ +(function (root, factory) { + /* istanbul ignore next */ + if (typeof define === 'function' && define.amd) { + define(['angular'], factory); + } else if (typeof module === 'object' && module.exports) { + module.exports = factory(require('angular')); + } else { + root.angularClipboard = factory(root.angular); + } +}(this, function (angular) { + +return angular.module('angular-clipboard', []) + .factory('clipboard', ['$document', '$window', function ($document, $window) { + function createNode(text, context) { + var node = $document[0].createElement('textarea'); + node.style.position = 'absolute'; + node.style.fontSize = '12pt'; + node.style.border = '0'; + node.style.padding = '0'; + node.style.margin = '0'; + node.style.left = '-10000px'; + node.style.top = ($window.pageYOffset || $document[0].documentElement.scrollTop) + 'px'; + node.textContent = text; + return node; + } + + function copyNode(node) { + try { + // Set inline style to override css styles + $document[0].body.style.webkitUserSelect = 'initial'; + + var selection = $document[0].getSelection(); + selection.removeAllRanges(); + + var range = document.createRange(); + range.selectNodeContents(node); + selection.addRange(range); + // This makes it work in all desktop browsers (Chrome) + node.select(); + // This makes it work on Mobile Safari + node.setSelectionRange(0, 999999); + + try { + if(!$document[0].execCommand('copy')) { + throw('failure copy'); + } + } finally { + selection.removeAllRanges(); + } + } finally { + // Reset inline style + $document[0].body.style.webkitUserSelect = ''; + } + } + + function copyText(text, context) { + var left = $window.pageXOffset || $document[0].documentElement.scrollLeft; + var top = $window.pageYOffset || $document[0].documentElement.scrollTop; + + var node = createNode(text, context); + $document[0].body.appendChild(node); + copyNode(node); + + $window.scrollTo(left, top); + $document[0].body.removeChild(node); + } + + return { + copyText: copyText, + supported: 'queryCommandSupported' in $document[0] && $document[0].queryCommandSupported('copy') + }; + }]) + .directive('clipboard', ['clipboard', function (clipboard) { + return { + restrict: 'A', + scope: { + onCopied: '&', + onError: '&', + text: '=', + supported: '=?' + }, + link: function (scope, element) { + scope.supported = clipboard.supported; + + element.on('click', function (event) { + try { + clipboard.copyText(scope.text, element[0]); + if (angular.isFunction(scope.onCopied)) { + scope.$evalAsync(scope.onCopied()); + } + } catch (err) { + if (angular.isFunction(scope.onError)) { + scope.$evalAsync(scope.onError({err: err})); + } + } + }); + } + }; + }]); + +})); diff --git a/server/www/script/ngClip.js b/server/www/script/ngClip.js deleted file mode 100644 index 280e394ff3f..00000000000 --- a/server/www/script/ngClip.js +++ /dev/null @@ -1,84 +0,0 @@ -/*jslint node: true */ -/*global ZeroClipboard */ - -(function(window, angular, undefined) { - 'use strict'; - - angular.module('ngClipboard', []). - provider('ngClip', function() { - var self = this; - this.path = '//cdnjs.cloudflare.com/ajax/libs/zeroclipboard/2.1.6/ZeroClipboard.swf'; - return { - setPath: function(newPath) { - self.path = newPath; - }, - setConfig: function(config) { - self.config = config; - }, - $get: function() { - return { - path: self.path, - config: self.config - }; - } - }; - }). - run(['ngClip', function(ngClip) { - var config = { - swfPath: ngClip.path, - trustedDomains: ["*"], - allowScriptAccess: "always", - forceHandCursor: true, - }; - ZeroClipboard.config(angular.extend(config,ngClip.config || {})); - }]). - directive('clipCopy', ['ngClip', function (ngClip) { - return { - scope: { - clipCopy: '&', - clipClick: '&', - clipClickFallback: '&' - }, - restrict: 'A', - link: function (scope, element, attrs) { - // Bind a fallback function if flash is unavailable - if (ZeroClipboard.isFlashUnusable()) { - element.bind('click', function($event) { - // Execute the expression with local variables `$event` and `copy` - scope.$apply(scope.clipClickFallback({ - $event: $event, - copy: scope.$eval(scope.clipCopy) - })); - }); - - return; - } - - // Create the client object - var client = new ZeroClipboard(element); - if (attrs.clipCopy === "") { - scope.clipCopy = function(scope) { - return element[0].previousElementSibling.innerText; - }; - } - client.on( 'ready', function(readyEvent) { - - client.on('copy', function (event) { - var clipboard = event.clipboardData; - clipboard.setData(attrs.clipCopyMimeType || 'text/plain', scope.$eval(scope.clipCopy)); - }); - - client.on( 'aftercopy', function(event) { - if (angular.isDefined(attrs.clipClick)) { - scope.$apply(scope.clipClick); - } - }); - - scope.$on('$destroy', function() { - client.destroy(); - }); - }); - } - }; - }]); -})(window, window.angular); \ No newline at end of file diff --git a/server/www/scripts/app.js b/server/www/scripts/app.js index 88be6e820e2..dfe93ec6e98 100644 --- a/server/www/scripts/app.js +++ b/server/www/scripts/app.js @@ -9,7 +9,7 @@ $.ajaxSetup({ }); var faradayApp = angular.module('faradayApp', ['ngRoute', 'selectionModel', 'ui.bootstrap', 'angularFileUpload', - 'filter', 'ngClipboard', 'ngCookies', 'cfp.hotkeys', 'chart.js', + 'filter', 'angular-clipboard', 'ngCookies', 'cfp.hotkeys', 'chart.js', 'ui.grid', 'ui.grid.selection', 'ui.grid.grouping', 'ngSanitize', 'ui.grid.pagination', 'ui.grid.pinning', 'angularMoment', 'ui-notification', 'tandibar/ng-rollbar', 'ui.grid.resizeColumns']) @@ -71,12 +71,11 @@ var faradayApp = angular.module('faradayApp', ['ngRoute', 'selectionModel', 'ui. return statuses; })()); -faradayApp.config(['$routeProvider', 'ngClipProvider', '$uibTooltipProvider', 'RollbarProvider', - function($routeProvider, ngClipProvider, $uibTooltipProvider, RollbarProvider) { +faradayApp.config(['$routeProvider', '$uibTooltipProvider', 'RollbarProvider', + function($routeProvider, $uibTooltipProvider, RollbarProvider) { $uibTooltipProvider.options({ appendToBody: true }); - ngClipProvider.setPath("script/ZeroClipboard.swf"); $routeProvider. when('/dashboard/ws/:wsId', { templateUrl: 'scripts/dashboard/partials/dashboard.html', diff --git a/server/www/scripts/dashboard/controllers/summarizedCtrlHostsModal.js b/server/www/scripts/dashboard/controllers/summarizedCtrlHostsModal.js index b0bf10e6cb5..6e5b0d4bf10 100644 --- a/server/www/scripts/dashboard/controllers/summarizedCtrlHostsModal.js +++ b/server/www/scripts/dashboard/controllers/summarizedCtrlHostsModal.js @@ -4,9 +4,9 @@ angular.module('faradayApp') .controller('summarizedCtrlHostsModal', - ['$scope', '$modalInstance', 'dashboardSrv', 'workspace', 'srv_name', 'osint', + ['$scope', '$modalInstance', 'dashboardSrv', 'workspace', 'srv_name', 'osint', function($scope, $modalInstance, dashboardSrv, workspace, srv_name, osint) { - + $scope.osint = osint; $scope.sortField = 'name'; $scope.sortReverse = false; diff --git a/server/www/scripts/dashboard/partials/modal-hosts-by-service.html b/server/www/scripts/dashboard/partials/modal-hosts-by-service.html index c1484809457..7474300ebbd 100644 --- a/server/www/scripts/dashboard/partials/modal-hosts-by-service.html +++ b/server/www/scripts/dashboard/partials/modal-hosts-by-service.html @@ -37,7 +37,7 @@

    Services for {{name}} ({{hosts.length}} total)

    - +
    From a52d52e8917c7e7b9c46dfd7aa04177f0e290fc4 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 15 Jun 2018 12:24:19 -0300 Subject: [PATCH 1368/1506] Update version on default.xml --- config/default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/default.xml b/config/default.xml index a557e32e7f3..efe8d0a3802 100644 --- a/config/default.xml +++ b/config/default.xml @@ -2,7 +2,7 @@ Faraday - Penetration Test IDE - 2.7.2 + 3.0 0 -Misc-Fixed-medium-r-normal-*-12-100-100-100-c-70-iso8859-1 ~/ From 1ef2fdaae1c27562311d738d0eeef152cf8456a4 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 15 Jun 2018 13:05:51 -0300 Subject: [PATCH 1369/1506] [FIX] if faraday can't access the plugins allow the server to run. --- plugins/manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/manager.py b/plugins/manager.py index 13160a9c01d..1e01a6a96b9 100644 --- a/plugins/manager.py +++ b/plugins/manager.py @@ -16,7 +16,7 @@ from plugins.controller import PluginController from config.configuration import getInstanceConfiguration -from utils.logs import getLogger +import server.utils.logger CONF = getInstanceConfiguration() @@ -96,6 +96,9 @@ def _loadPlugins(self, plugin_repo_path): sys.path.append(plugin_repo_path) dir_name_regexp = re.compile(r"^[\d\w\-\_]+$") + if not os.path.exists(plugin_repo_path): + server.utils.logger.get_logger(self).error('Plugins path could not be opened, no pluging will be available!') + return for name in os.listdir(plugin_repo_path): if dir_name_regexp.match(name): try: From 03bfc8c722936657746efb6c3a3708efb20f05fc Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 15 Jun 2018 13:20:51 -0300 Subject: [PATCH 1370/1506] [FIX] show only workspace controls if the workspace is already selected --- server/www/scripts/commons/partials/header.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/www/scripts/commons/partials/header.html b/server/www/scripts/commons/partials/header.html index 4346db648a2..992fb31c9b6 100644 --- a/server/www/scripts/commons/partials/header.html +++ b/server/www/scripts/commons/partials/header.html @@ -57,17 +57,17 @@

    Users

    Licenses

    Help

    - +
    - - - + From 2cae1bc942302f4f28f51152f69b94d732fb64bb Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 15 Jun 2018 15:30:58 -0300 Subject: [PATCH 1371/1506] Fix leo bug :p --- plugins/manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/manager.py b/plugins/manager.py index 1e01a6a96b9..9e1bad0be5d 100644 --- a/plugins/manager.py +++ b/plugins/manager.py @@ -117,12 +117,12 @@ def _loadPlugins(self, plugin_repo_path): elif file_ext.lower() == '.pyc': self._plugin_modules[name] = imp.load_compiled(name, module_filename) - getLogger(self).debug('Loading plugin {0}'.format(name)) + server.utils.logger.get_logger(self).debug('Loading plugin {0}'.format(name)) except Exception as e: msg = "An error ocurred while loading plugin %s.\n%s" % ( module_filename, traceback.format_exc()) - getLogger(self).debug(msg) - getLogger(self).warn(e) + server.utils.logger.get_logger(self).debug(msg) + server.utils.logger.get_logger(self).warn(e) def getPlugins(self): plugins = self._instancePlugins() From c52a5394f873f866094af97730d5701a1aea6849 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 15 Jun 2018 16:15:53 -0300 Subject: [PATCH 1372/1506] Change severity of importer log message It wasn't so important so I moved it from warning to debug --- server/importer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/importer.py b/server/importer.py index 024a8193883..310937223dd 100644 --- a/server/importer.py +++ b/server/importer.py @@ -346,8 +346,8 @@ def update_command_tools(workspace, command_tool_map, id_map): missing_tool_count = Command.query.filter_by( workspace=workspace, tool="unknown").count() if missing_tool_count: - logger.warn("Couldn't find the tool name of {} commands in " - "workspace {}".format( + logger.debug("Couldn't find the tool name of {} commands in " + "workspace {}".format( missing_tool_count, workspace.name)) From cfe720becd4c7a0dca0f541fae13c2358e1305b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 15 Jun 2018 16:35:13 -0300 Subject: [PATCH 1373/1506] Add plugins to RELEASE.md --- RELEASE.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index fce8929411a..5d29fbe4fe9 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -12,7 +12,7 @@ TBA: --- * Allow to upload report file from external tools from the web * Fix sshcheck import file from GTK -* Add reconng plugin +* Add reconng, sublist3r, HP Webinspect, dirsearch and ip360 plugins * CouchDB was replaced by PostgreSQL :) * Host object changed, now the name property is called ip * Interface object was removed @@ -28,7 +28,6 @@ TBA: * Exploitation and severity fields only allow certain values. CWE CVEs were fixed to be valid. A script to convert custom CSVs was added. * Web UI path changed from /_ui/ to / (_ui has now a redirection to / for keeping backwards compatibility) * dirb plugin should creates a vulnerability type information instead of a note. -* Add new plugin ip360 * Add confirmed column to exported csv from webui * Fixes in Arachni plugin * Add new parameters --keep-old and --keep-new for faraday CLI @@ -37,7 +36,6 @@ TBA: * Removed Chat feature (data is kept inside notes) * Plugin reports now can be imported in the server, from the Web UI * Add CVSS score to reference field in Nessus plugin. -* Add HP Webinspect plugin. * Fix unicode characters bug in Netsparker plugin. * Fix qualys plugin. * Fix bugs with MACOS and GTK. From 21508ba9748047061c7cf05e643579e5d5d18627 Mon Sep 17 00:00:00 2001 From: montive Date: Fri, 15 Jun 2018 17:37:50 -0300 Subject: [PATCH 1374/1506] Fix path calculation on vulndbToCsv.py --- helpers/vulndbToCsv.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/helpers/vulndbToCsv.py b/helpers/vulndbToCsv.py index e8fa5e10392..ca3c30ab802 100755 --- a/helpers/vulndbToCsv.py +++ b/helpers/vulndbToCsv.py @@ -3,14 +3,13 @@ ''' Copyright (C) 2016 Infobyte LLC (http://www.infobytesec.com/) Author: Ezequiel Tavella -See the file 'doc/LICENSE' for the license information This script generate a CSV file with information about the vulndb database. CSV Format: cwe,name,desc_summary,description,resolution,exploitation,references ''' from subprocess import call -from os import walk +from os import walk, path import json import csv @@ -100,18 +99,17 @@ def main(): for file_db in files: print '[*]Parsing ' + file_db - with open(root + file_db, 'r') as file_object: + with open(path.join(root, file_db), 'r') as file_object: csv_content = JsonToCsv(file_object) - result = ( - csv_content.cwe, - csv_content.name, - csv_content.description, - csv_content.resolution, - '', - ' '.join(csv_content.references), - csv_content.severity + csv_content.cwe, + csv_content.name, + csv_content.description, + csv_content.resolution, + '', + ' '.join(csv_content.references or []), + csv_content.severity ) writer.writerow(result) From b65576b014ce1ca55a1eec28fa48db5e7492e3d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 15 Jun 2018 18:17:10 -0300 Subject: [PATCH 1375/1506] Fix requests error catching on the importer With the previous way it wasn't working properly with some requests versions --- server/importer.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/server/importer.py b/server/importer.py index 310937223dd..e6188d247ce 100644 --- a/server/importer.py +++ b/server/importer.py @@ -17,6 +17,7 @@ import requests +from requests.exceptions import HTTPError, RequestException from tempfile import NamedTemporaryFile from collections import ( @@ -213,7 +214,7 @@ def get_children_from_couch(workspace, parent_couchdb_id, child_type): try: r = requests.put(view_url, json=view_data) - except requests.exceptions.RequestException as e: + except RequestException as e: logger.exception(e) return [] @@ -227,7 +228,7 @@ def get_children_from_couch(workspace, parent_couchdb_id, child_type): try: r = requests.get(couch_url) - except requests.exceptions.RequestException as e: + except RequestException as e: logger.error('Network error in CouchDB request {}'.format( couch_url, r.status_code, @@ -237,7 +238,7 @@ def get_children_from_couch(workspace, parent_couchdb_id, child_type): try: r.raise_for_status() - except requests.exceptions.RequestException as e: + except RequestException as e: logger.error('Error in CouchDB request {}. ' 'Status code: {}. ' 'Body: {}'.format(couch_url, @@ -1215,10 +1216,10 @@ def run(self): try: cwes = requests.get(cwe_url) cwes.raise_for_status() - except requests.exceptions.HTTPError: + except HTTPError: logger.warn('Unable to retrieve Vulnerability Templates Database. Moving on.') return - except requests.exceptions.RequestException as e: + except RequestException as e: logger.exception(e) return @@ -1298,10 +1299,10 @@ def run(self): try: licenses = requests.get(licenses_url) licenses.raise_for_status() - except requests.exceptions.HTTPError: + except HTTPError: logger.warn('Unable to retrieve Licenses Database. Moving on.') return - except requests.exceptions.RequestException as e: + except RequestException as e: logger.exception(e) return From 5a30353d609ad691e830b6824b4e35c04cf10ae1 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 15 Jun 2018 19:15:13 -0300 Subject: [PATCH 1376/1506] Fix bug with hostnames, put hostnames in Host object --- model/api.py | 8 ++++---- model/controller.py | 4 ++-- persistence/server/models.py | 7 +++++++ persistence/server/server.py | 6 ++++-- persistence/server/utils.py | 1 + plugins/plugin.py | 5 +++-- test_cases/test_api_hosts.py | 24 +++++++++++++++++++++++- 7 files changed, 44 insertions(+), 11 deletions(-) diff --git a/model/api.py b/model/api.py index 6402ed01a7b..5ecee3cceb0 100644 --- a/model/api.py +++ b/model/api.py @@ -138,8 +138,8 @@ def _setUpAPIServer(hostname=None, port=None): # plugin created the object -def createAndAddHost(ip, os="Unknown"): - host = newHost(ip, os) +def createAndAddHost(ip, os="Unknown", hostnames=None): + host = newHost(ip, os, hostnames=hostnames) if addHost(host): return host.getID() return None @@ -328,12 +328,12 @@ def delCredFromService(cred, hostname, srvname): # CREATION APIS #------------------------------------------------------------------------------- -def newHost(ip, os = "Unknown"): +def newHost(ip, os = "Unknown", hostnames=None): """ It creates and returns a Host object. The object created is not added to the model. """ - return __model_controller.newHost(ip, os) + return __model_controller.newHost(ip, os, hostnames=hostnames) def newService(name, protocol = "tcp?", ports = [], status = "running", diff --git a/model/controller.py b/model/controller.py index 2c9895b0863..e288f14fc37 100644 --- a/model/controller.py +++ b/model/controller.py @@ -447,10 +447,10 @@ def _log(self, msg, *args, **kwargs): api.log(msg, *args[:-1]) return True - def newHost(self, ip, os="Unknown"): + def newHost(self, ip, os="Unknown", hostnames=None): return model.common.factory.createModelObject( models.Host.class_signature, ip, - workspace_name=self.mappers_manager.workspace_name, os=os, parent_id=None) + workspace_name=self.mappers_manager.workspace_name, os=os, parent_id=None, hostnames=hostnames) def newService(self, name, protocol="tcp?", ports=[], status="running", version="unknown", description="", parent_id=None): diff --git a/persistence/server/models.py b/persistence/server/models.py index da4f1fdb6ee..f45b330ba85 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -857,6 +857,7 @@ def __init__(self, host, workspace_name): self.os = host.get('os') if host.get('os') else 'unknown' self.vuln_amount = int(host.get('vulns', 0)) self.ip = host.get('ip', self.name) + self.hostnames = host.get('hostnames', []) if host.get('hostnames') else [] def getName(self): return self.ip @@ -890,6 +891,12 @@ def getVulnsAmount(self): def getDefaultGateway(self): return self.default_gateway + def getHostnames(self): + return self.hostnames + + def setHostnames(self, hostnames): + self.hostnames = hostnames + def getVulns(self): """ Get all vulns of this host. diff --git a/persistence/server/server.py b/persistence/server/server.py index cbf12168665..8059314ab13 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -889,7 +889,7 @@ def get_commands_number(workspace_name, **params): def create_host(workspace_name, command_id, ip, os, default_gateway=None, description="", metadata=None, owned=False, owner="", - parent=None): + parent=None, hostnames=None): """Create a host. Args: @@ -918,11 +918,12 @@ def create_host(workspace_name, command_id, ip, os, default_gateway=None, owner=owner, parent=parent, description=description, + hostnames=hostnames, type="Host") def update_host(workspace_name, command_id, id, ip, os, default_gateway="", description="", metadata=None, owned=False, owner="", - parent=None): + parent=None, hostnames=None): """Updates a host. Args: @@ -952,6 +953,7 @@ def update_host(workspace_name, command_id, id, ip, os, default_gateway="", owner=owner, parent=parent, description=description, + hostnames=hostnames, type="Host") diff --git a/persistence/server/utils.py b/persistence/server/utils.py index 36197f98d73..7a9c5e8893d 100644 --- a/persistence/server/utils.py +++ b/persistence/server/utils.py @@ -47,6 +47,7 @@ def get_object_properties(obj): def get_host_properties(host): host_dict = { 'os': host.getOS(), + 'hostnames': host.getHostnames() } if host.getDefaultGateway(): host_dict['default_gateway'] = host.getDefaultGateway() diff --git a/plugins/plugin.py b/plugins/plugin.py index b45b4804933..2f8936e9008 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -171,14 +171,15 @@ def __addPendingAction(self, *args): logger.debug('AddPendingAction', args) self._pending_actions.put(args) - def createAndAddHost(self, name, os="unknown"): + def createAndAddHost(self, name, os="unknown", hostnames=None): host_obj = factory.createModelObject( Host.class_signature, name, os=os, parent_id=None, - workspace_name=self.workspace) + workspace_name=self.workspace, + hostnames=hostnames) host_obj._metadata.creatoserverr = self.id self.__addPendingAction(Modelactions.ADDHOST, host_obj) diff --git a/test_cases/test_api_hosts.py b/test_cases/test_api_hosts.py index c4e2ee63977..514ad209a5a 100644 --- a/test_cases/test_api_hosts.py +++ b/test_cases/test_api_hosts.py @@ -26,7 +26,7 @@ ReadWriteAPITests, PaginationTestsMixin, ) -from server.models import db, Host +from server.models import db, Host, Hostname from server.api.modules.hosts import HostsView from test_cases.factories import HostFactory, CommandFactory, \ EmptyCommandFactory, WorkspaceFactory @@ -651,6 +651,28 @@ def test_delete_host_with_blank_ip(self, session, test_client): res = test_client.delete(self.url(host, workspace=host.workspace)) assert res.status_code == 204 + def test_update_hostname(self, session, test_client): + host = HostFactory.create() + session.add(host) + session.commit() + data = { + "description":"", + "default_gateway":"", + "ip":"127.0.0.1", + "owned":False, + "name":"127.0.0.1", + "mac":"", + "hostnames":["dasdas"], + "owner":"faraday", + "os":"Unknown", + } + + res = test_client.put('v2/ws/{0}/hosts/{1}/'.format(host.workspace.name, host.id), data=data) + assert res.status_code == 200 + + assert session.query(Hostname).filter_by(host=host).count() == 1 + assert session.query(Hostname).all()[0].name == 'dasdas' + def host_json(): return st.fixed_dictionaries( From 2dc666283df72b0a91993d5139e8871aefee3f1d Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 15 Jun 2018 19:53:19 -0300 Subject: [PATCH 1377/1506] Move import to avoid importerror (we need faraday as a package) --- server/api/modules/upload_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/api/modules/upload_reports.py b/server/api/modules/upload_reports.py index edd75cc080f..8017d0bef5c 100644 --- a/server/api/modules/upload_reports.py +++ b/server/api/modules/upload_reports.py @@ -26,7 +26,6 @@ from werkzeug.utils import secure_filename from wtforms import ValidationError -from faraday import setupPlugins from server.utils.logger import get_logger from server.utils.web import gzipped @@ -55,6 +54,7 @@ class RawReportProcessor(Thread): def __init__(self): super(RawReportProcessor, self).__init__() + from faraday import setupPlugins setupPlugins() self.pending_actions = Queue() From eae08a5360d47a57848b45c460726be4f7e7d6fa Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 15 Jun 2018 20:09:33 -0300 Subject: [PATCH 1378/1506] Fix bug in test hostnames.... --- test_cases/test_managers_mapper_manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test_cases/test_managers_mapper_manager.py b/test_cases/test_managers_mapper_manager.py index f47ca70ba4e..93eee811067 100644 --- a/test_cases/test_managers_mapper_manager.py +++ b/test_cases/test_managers_mapper_manager.py @@ -55,6 +55,7 @@ 'name': '192.168.0.20', 'description': 'My computer', 'default_gateway': '192.168.0.1', + 'hostnames': [], 'os': 'Debian', 'owned': False, 'owner': 'leo' @@ -65,6 +66,7 @@ 'default_gateway': '192.168.0.1', 'description': 'My computer', 'ip': '192.168.0.20', + 'hostnames': [], 'os': 'Debian', 'owned': False, 'owner': 'leo', @@ -491,6 +493,7 @@ 'description': 'Test description', 'ip': '192.168.1.1', 'os': 'Linux 2.6.9', + 'hostnames': [], 'owned': False, 'owner': 'leonardo'} @@ -874,4 +877,4 @@ def mock_unsafe_io_with_server(host, test_data, server_io_function, server_expec serialized_obj = test_data['get_properties_function'](found_obj) if obj_class not in [Command]: metadata = serialized_obj.pop('metadata') - assert serialized_obj == test_data['serialized_expected_results'] + assert serialized_obj == test_data['serialized_expected_results'] \ No newline at end of file From 298d2eff02b3295fcf7a451a3a3ef1b398529ef4 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Mon, 18 Jun 2018 12:27:26 -0300 Subject: [PATCH 1379/1506] [COM] New bug partial fix warning [MOD] README with changes --- RELEASE.md | 1 + plugins/repo/wapiti/plugin.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index a3d286c53cd..f4cc462cea1 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -10,6 +10,7 @@ New features in the latest update TBA: --- +* Fix W3af, Zap * Add brutexss plugin * Allow to upload report file from external tools from the web * Fix sshcheck import file from GTK diff --git a/plugins/repo/wapiti/plugin.py b/plugins/repo/wapiti/plugin.py index 40d07d065d1..36295d5738f 100644 --- a/plugins/repo/wapiti/plugin.py +++ b/plugins/repo/wapiti/plugin.py @@ -244,6 +244,9 @@ def parseOutputString(self, output, debug=False): """ self._output_file_path = "/root/dev/faraday/trunk/src/report/wapiti2.3.0_abaco.xml" + #TODO FOR mig_white_4896... if NOT debug + # OR replace all if/else with + # parser = WapitiXmlParser(output) if debug: parser = WapitiXmlParser(output) else: From 9e9a111de30dded9b7814c02bceec09602335411 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 18 Jun 2018 15:25:38 -0300 Subject: [PATCH 1380/1506] Update release.md --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index f4cc462cea1..ec0108588fc 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -10,6 +10,7 @@ New features in the latest update TBA: --- +* Add xsssniper plugin * Fix W3af, Zap * Add brutexss plugin * Allow to upload report file from external tools from the web From b9b384017e7bbb9fe0eef937782da3e18d4ac295 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 18 Jun 2018 17:29:50 -0300 Subject: [PATCH 1381/1506] PEP8 fixes --- plugins/repo/wfuzz/plugin.py | 71 +++++++++++++++++------------------- 1 file changed, 34 insertions(+), 37 deletions(-) diff --git a/plugins/repo/wfuzz/plugin.py b/plugins/repo/wfuzz/plugin.py index c9558eedd01..778822f84e7 100644 --- a/plugins/repo/wfuzz/plugin.py +++ b/plugins/repo/wfuzz/plugin.py @@ -1,8 +1,9 @@ -import sys import re -from plugins import core +import sys from urlparse import urljoin, urlparse +from plugins import core + class WfuzzPlugin(core.PluginBase): @@ -23,51 +24,48 @@ def __init__(self): def parseData(self, output): - data = { - 'target' : '', - 'findings' : [ - - ] - } - for line in output: - - if(line.startswith('Target')): + data = { + 'target' : '', + 'findings' : [] + } + for line in output: + + if line.startswith('Target'): data['target'] = line[8:].rstrip() - if(line.startswith('0')): - aux = line.split(' ') - res = {} - for item in aux: - if 'C=' in item: - res['response'] = int(item.replace('C=', '')) - elif 'L' in item and ' ' in item: - res['lines'] = int(item.replace('L', '')) - elif 'W' in item and ' ' in item: - res['words'] = int(item.replace('W', '')) - elif 'Ch' in item and ' ' in item: - res['chars'] = int(item.replace('Ch', '')) - else: - res['request'] = item.rstrip().replace('"', '') - data['findings'].append(res) + if line.startswith('0'): + aux = line.split(' ') + res = {} + for item in aux: + if 'C=' in item: + res['response'] = int(item.replace('C=', '')) + elif 'L' in item and ' ' in item: + res['lines'] = int(item.replace('L', '')) + elif 'W' in item and ' ' in item: + res['words'] = int(item.replace('W', '')) + elif 'Ch' in item and ' ' in item: + res['chars'] = int(item.replace('Ch', '')) + else: + res['request'] = item.rstrip().replace('"', '') + data['findings'].append(res) return data - def parseOutputString(self, output, debug=False): output_list = output.split('\n') - info = self.parseData(output_list) + info = self.parseData(output_list) target = info['target'] target_url = urlparse(target) port = 80 - + if target_url.scheme == 'https': port = 443 custom_port = target_url.netloc.split(':') if len(custom_port) > 1: port = custom_port[1] - + host_id = self.createAndAddHost(target) service_id = self.createAndAddServiceToHost(host_id,name="http",protocol="tcp", ports=[port] ) @@ -87,17 +85,16 @@ def parseOutputString(self, output, debug=False): chars=chars, status=status ) - vuln = self.createAndAddVulnWebToService(host_id, + self.createAndAddVulnWebToService(host_id, service_id, - name, - desc, - severity="info", - website=target, + name, + desc, + severity="info", + website=target, path=path - ) + ) - def createPlugin(): return WfuzzPlugin() From 7f51baaf20036642098838b5f1fa07ef4e1dba21 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 18 Jun 2018 18:59:01 -0300 Subject: [PATCH 1382/1506] [FIX] remove hidden chars to make the plugin work from terminal --- plugins/repo/wfuzz/plugin.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/repo/wfuzz/plugin.py b/plugins/repo/wfuzz/plugin.py index 778822f84e7..86673b8c32b 100644 --- a/plugins/repo/wfuzz/plugin.py +++ b/plugins/repo/wfuzz/plugin.py @@ -1,5 +1,6 @@ import re import sys +import string from urlparse import urljoin, urlparse from plugins import core @@ -29,10 +30,12 @@ def parseData(self, output): 'findings' : [] } for line in output: - + # remove stdout hidden chars + line = ''.join([char for char in line if char in string.printable]) + line = line.strip('\r').replace('[0K', '').replace('[0m', '') if line.startswith('Target'): data['target'] = line[8:].rstrip() - + continue if line.startswith('0'): aux = line.split(' ') res = {} @@ -52,7 +55,6 @@ def parseData(self, output): return data def parseOutputString(self, output, debug=False): - output_list = output.split('\n') info = self.parseData(output_list) From 5d91c18d3dd086046d4bcf3078bcd43e730d2fc7 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Mon, 18 Jun 2018 19:07:51 -0300 Subject: [PATCH 1383/1506] Fix hostnames bug #4895 --- managers/reports_managers.py | 2 +- plugins/controller.py | 1 + plugins/plugin.py | 9 +++++++++ plugins/repo/burp/faraday-burp.rb | 2 +- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/managers/reports_managers.py b/managers/reports_managers.py index 45a9bf155fe..7a6625c4901 100755 --- a/managers/reports_managers.py +++ b/managers/reports_managers.py @@ -51,7 +51,7 @@ def sendReport(self, plugin_id, filename): """Sends a report to the appropiate plugin specified by plugin_id""" getLogger(self).info( 'The file is %s, %s' % (filename, plugin_id)) - command_id = self.plugin_controller.processReport(plugin_id, filename, self.ws_name) + command_id = self.plugin_controller.processReport(plugin_id, filename, ws_name=self.ws_name) if not command_id: getLogger(self).error( "Faraday doesn't have a plugin for this tool..." diff --git a/plugins/controller.py b/plugins/controller.py index 7543a7f23ef..32e596ab422 100644 --- a/plugins/controller.py +++ b/plugins/controller.py @@ -287,6 +287,7 @@ def processReport(self, plugin, filepath, ws_name=None): cmd_info.setID(command_id) if plugin in self._plugins: logger.info('Processing report with plugin {0}'.format(plugin)) + self._plugins[plugin].workspace = ws_name with open(filepath, 'rb') as output: self.processOutput(self._plugins[plugin], output.read(), cmd_info, True) return command_id diff --git a/plugins/plugin.py b/plugins/plugin.py index 2f8936e9008..7d9cd7ffe9c 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -20,6 +20,7 @@ import model.api import model.common from model.common import factory +from persistence.server.models import get_host , update_host from persistence.server.models import ( Host, Service, @@ -198,6 +199,14 @@ def createAndAddInterface( # We don't use interface anymore, so return a host id to maintain # backwards compatibility + # Little hack because we dont want change all the plugins for add hostnames in Host object. + # SHRUG + try: + host = get_host(self.workspace, host_id=host_id) + host.hostnames = hostname_resolution + update_host(self.workspace, host, command_id=self.command_id) + except: + logger.info("Error updating Host with right hostname resolution...") return host_id @deprecation.deprecated(deprecated_in="3.0", removed_in="3.5", diff --git a/plugins/repo/burp/faraday-burp.rb b/plugins/repo/burp/faraday-burp.rb index de62215ba5b..9b8d80593a4 100644 --- a/plugins/repo/burp/faraday-burp.rb +++ b/plugins/repo/burp/faraday-burp.rb @@ -243,7 +243,7 @@ def newScanIssue(issue, ctx=nil, import=nil) begin rt = @server.call("devlog", "[BURP] New issue generation") - h_id = @server.call("createAndAddHost",ip, "unknown") + h_id = @server.call("createAndAddHost",ip, "unknown", host) i_id = @server.call("createAndAddInterface",h_id, ip,"00:00:00:00:00:00", ip, "0.0.0.0", "0.0.0.0",[], "0000:0000:0000:0000:0000:0000:0000:0000","00","0000:0000:0000:0000:0000:0000:0000:0000", [],"",host) From 2e8936e05638b847cac8c77c31a5f13176bc5be2 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Mon, 18 Jun 2018 19:55:12 -0300 Subject: [PATCH 1384/1506] Fix bug in Burp plugin --- plugins/repo/burp/faraday-burp.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/repo/burp/faraday-burp.rb b/plugins/repo/burp/faraday-burp.rb index 9b8d80593a4..b71407ce63c 100644 --- a/plugins/repo/burp/faraday-burp.rb +++ b/plugins/repo/burp/faraday-burp.rb @@ -243,7 +243,7 @@ def newScanIssue(issue, ctx=nil, import=nil) begin rt = @server.call("devlog", "[BURP] New issue generation") - h_id = @server.call("createAndAddHost",ip, "unknown", host) + h_id = @server.call("createAndAddHost",ip, "unknown", [host]) i_id = @server.call("createAndAddInterface",h_id, ip,"00:00:00:00:00:00", ip, "0.0.0.0", "0.0.0.0",[], "0000:0000:0000:0000:0000:0000:0000:0000","00","0000:0000:0000:0000:0000:0000:0000:0000", [],"",host) From 18180fab1c9bf48cefdc3310f61aed0e7f25ed14 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 19 Jun 2018 12:55:01 -0300 Subject: [PATCH 1385/1506] [FIX] Remove lazy joined from Scope to avoid performance issues on workspace summary. we need to review scope query for other use cases --- server/models.py | 3 ++- test_cases/test_api_workspace.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index f3adf890ad3..d5e3856bc37 100644 --- a/server/models.py +++ b/server/models.py @@ -1240,9 +1240,10 @@ class Scope(Metadata): index=True, nullable=False ) + workspace = relationship( 'Workspace', - backref=backref('scope', lazy="joined", cascade="all, delete-orphan"), + backref=backref('scope', cascade="all, delete-orphan"), foreign_keys=[workspace_id], ) diff --git a/test_cases/test_api_workspace.py b/test_cases/test_api_workspace.py index 0b8eb1b18ce..8b26639152a 100644 --- a/test_cases/test_api_workspace.py +++ b/test_cases/test_api_workspace.py @@ -9,6 +9,7 @@ from server.models import Workspace, Scope from server.api.modules.workspaces import WorkspaceView +from test_cases.conftest import ignore_nplusone from test_cases.test_api_non_workspaced_base import ReadWriteAPITests from test_cases import factories @@ -163,3 +164,8 @@ def test_update_with_scope(self, session, test_client, workspace): assert res.status_code == 200 assert set(res.json['scope']) == set(desired_scope) assert set(s.name for s in workspace.scope) == set(desired_scope) + + @ignore_nplusone + def test_list_retrieves_all_items_from(self, test_client): + super(TestWorkspaceAPI, self).test_list_retrieves_all_items_from(test_client) + From 1e5f14da1a77070871b402d9d245726df8e66c4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 19 Jun 2018 12:56:20 -0300 Subject: [PATCH 1386/1506] Fix a potential cause for an association proxy bug The session can hold hosts inside it, which don't have a name. The bug was probably related to a race condition when doing many requests at the same time --- server/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index f3adf890ad3..3ae830270c9 100644 --- a/server/models.py +++ b/server/models.py @@ -479,7 +479,7 @@ def _create(self, value): # we need to fetch already created objs. session.rollback() for conflict_obj in conflict_objs: - if conflict_obj.name == value: + if hasattr(conflict_obj, 'name') and conflict_obj.name == value: continue persisted_conclict_obj = session.query(conflict_obj.__class__).filter_by(name=conflict_obj.name).first() if persisted_conclict_obj: From 988e71fdf3601187204da3316e77d9264c7406db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 19 Jun 2018 14:50:41 -0300 Subject: [PATCH 1387/1506] Fix importing services without parent from couch The case where parent is null wasn't handled --- server/importer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/importer.py b/server/importer.py index e6188d247ce..da36277650e 100644 --- a/server/importer.py +++ b/server/importer.py @@ -533,11 +533,11 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation command = session.query(Command).get(couchdb_relational_map[document['metadata']['command_id']][0]) except (KeyError, IndexError): command = None - try: - parent_id = document['parent'].split('.')[0] - except KeyError: - # some services are missing the parent key - parent_id = document['_id'].split('.')[0] + + # This should be safe because _id is always present and split never + # returns an empty list + parent_id = (document.get('parent') or document.get('_id')).split('.')[0] + for relational_parent_id in couchdb_relational_map[parent_id]: host, created = get_or_create(session, Host, id=relational_parent_id) if command and created: From e9f9de389a31a4be8f8665535f0839bbca0b85de Mon Sep 17 00:00:00 2001 From: montive Date: Tue, 19 Jun 2018 16:04:28 -0300 Subject: [PATCH 1388/1506] Plugin created: Sslyze --- plugins/repo/sslyze/__init__.py | 0 plugins/repo/sslyze/plugin.py | 192 ++++++++++++++++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 plugins/repo/sslyze/__init__.py create mode 100644 plugins/repo/sslyze/plugin.py diff --git a/plugins/repo/sslyze/__init__.py b/plugins/repo/sslyze/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/repo/sslyze/plugin.py b/plugins/repo/sslyze/plugin.py new file mode 100644 index 00000000000..1c88eab94a6 --- /dev/null +++ b/plugins/repo/sslyze/plugin.py @@ -0,0 +1,192 @@ +#coding=utf-8 +import re +import os +import random +from plugins import core + +try: + from lxml import etree as ET +except ImportError: + import xml.etree.ElementTree as ET + + +WEAK_CIPHER_LIST = [ + "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA", + "TLS_RSA_WITH_AES_128_CBC_SHA", + "TLS_RSA_WITH_AES_128_CBC_SHA256", + "TLS_RSA_WITH_AES_128_GCM_SHA256", + "TLS_RSA_WITH_AES_256_CBC_SHA", + "TLS_RSA_WITH_AES_256_CBC_SHA256", + "TLS_RSA_WITH_AES_256_GCM_SHA384", + "TLS_RSA_WITH_3DES_EDE_CBC_SHA", + "TLS_RSA_WITH_CAMELLIA_256_CBC_SHA", + "TLS_RSA_WITH_CAMELLIA_128_CBC_SHA" +] + + +class SslyzeXmlParser(object): + + def __init__(self, xml_output): + self.parser = self.parse_xml(xml_output) + self.target = self.get_target(self.parser) + self.certificate = self.get_hostname_validation(self.parser) + self.cipher_suite = self.get_weak_cipher_suite(self.parser) + self.heart_bleed = self.get_heartbleed(self.parser) + self.open_ssl_ccs = self.get_openssl_ccs(self.parser) + + def parse_xml(self, xml_output): + try: + tree = ET.fromstring(xml_output) + return tree + except IndexError: + print "Syntax error" + return None + + def get_target(self, tree): + return tree.xpath('//target') + + def get_hostname_validation(self, tree): + return tree.xpath('//hostnameValidation') + + def get_protocol_name(self, tree): + protocol_supported = [] + protocols = [] + protocols.append(tree.xpath('//sslv2')) + protocols.append(tree.xpath('//sslv3')) + protocols.append(tree.xpath('//tlsv1')) + protocols.append(tree.xpath('//tlsv1_1')) + protocols.append(tree.xpath('//tlsv1_2')) + protocols.append(tree.xpath('//tlsv1_3')) + + for protocol in protocols: + if protocol[0].attrib['isProtocolSupported'] == "True": + protocol_supported.append(protocol[0]) + + return protocol_supported + + def get_weak_cipher_suite(self, tree): + protocols = self.get_protocol_name(tree) + weak_cipher = {} + + for protocol in protocols: + weak_cipher[protocol.tag] = [] + for ciphers in protocol: + if ciphers.tag == 'preferredCipherSuite' or ciphers.tag == 'acceptedCipherSuites': + for cipher in ciphers: + if cipher.attrib['name'] in WEAK_CIPHER_LIST: + if not cipher.attrib['name'] in weak_cipher[protocol.tag]: + weak_cipher[protocol.tag].append(cipher.attrib['name']) + + return weak_cipher + + def get_heartbleed(self, tree): + return tree.xpath('//heartbleed') + + def get_openssl_ccs(self, tree): + return tree.xpath('//openssl_ccs') + + +class SslyzePlugin(core.PluginBase): + + def __init__(self): + core.PluginBase.__init__(self) + self.id = "Sslyze" + self.name = "Sslyze Plugin" + self.plugin_version = "0.0.1" + self.version = "1.4.2" + self.framework_version = "1.0.0" + self.options = None + self._current_output = None + self._command_regex = re.compile(r'^(sudo sslyze|sslyze|\.\/sslyze).*?') + self.xml_arg_re = re.compile(r"^.*(--xml_output\s*[^\s]+).*$") + + def parseOutputString(self, output, debug=False): + parser = SslyzeXmlParser(output) + host = parser.target[0].attrib['host'] + ip = parser.target[0].attrib['ip'] + port = parser.target[0].attrib['port'] + protocol = parser.target[0].attrib['tlsWrappedProtocol'] + cipher = parser.cipher_suite + + # Creating host + host_id = self.createAndAddHost(ip) + # Creating service CHANGE NAME + service_id = self.createAndAddServiceToHost( + host_id, + name=protocol, + protocol=protocol, + ports=[port], + ) + + # Checking if certificate matches + certificate = parser.certificate[0].attrib['certificateMatchesServerHostname'] + server_hostname = parser.certificate[0].attrib['serverHostname'] + if certificate.lower() == 'false': + self.createAndAddVulnToService( + host_id, + service_id, + name="Certificate mismatch", + desc="Certificate does not match server hostname {}".format(server_hostname), + severity="info") + #Ciphers + cipher = parser.cipher_suite + + for key in cipher: + for value in cipher[key]: + self.createAndAddVulnToService( + host_id, + service_id, + name=value, + desc="In protocol [{}], weak cipher suite: {}".format(key, value), + severity="low") + + #Heartbleed + heartbleed = parser.heart_bleed + + if heartbleed[0][0].attrib['isVulnerable'].lower() == 'true': + self.createAndAddVulnToService( + host_id, + service_id, + name="OpenSSL Heartbleed", + desc="OpenSSL Heartbleed is vulnerable", + severity="critical") + + #OpenSsl CCS Injection + openssl_ccs = parser.open_ssl_ccs + + if openssl_ccs[0][0].attrib['isVulnerable'].lower() == 'true': + self.createAndAddVulnToService( + host_id, + service_id, + name="OpenSSL CCS Injection", + desc="OpenSSL CCS Injection is vulnerable", + severity="medium") + + def processCommandString(self, username, current_path, command_string): + self._output_file_path = os.path.join( + self.data_path, + "%s_%s_output-%s.xml" % ( + self.get_ws(), + self.id, + random.uniform(1, 10)) + ) + + arg_match = self.xml_arg_re.match(command_string) + + if arg_match is None: + return re.sub(r"(^.*?sslyze)", + r"\1 --xml_out %s" % self._output_file_path, + command_string) + else: + return re.sub(arg_match.group(1), + r"--xml_out %s" % self._output_file_path, + command_string) + + +def createPlugin(): + return SslyzePlugin() + +if __name__ == '__main__': + parser = SslyzePlugin() + with open("/home/javier/expired.xml","r") as report: + parser.parseOutputString(report.read()) \ No newline at end of file From eba1a8a51930c96f33721e24210a535f879105a3 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Tue, 19 Jun 2018 16:09:46 -0300 Subject: [PATCH 1389/1506] Fix bug #4903 --- faraday.py | 3 +-- persistence/server/server.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/faraday.py b/faraday.py index 1d457ae21d7..ba30d562b4a 100755 --- a/faraday.py +++ b/faraday.py @@ -490,8 +490,7 @@ def doLoginLoop(): CONF.saveConfig() user_info = get_user_info() - - if user_info is None or 'username' not in user_info: + if (user_info is None) or (not user_info) or ('username' not in user_info): print("You can't login as a client. You have %s attempt(s) left." % (3 - attempt)) continue diff --git a/persistence/server/server.py b/persistence/server/server.py index 8059314ab13..ed1217be770 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -1577,7 +1577,7 @@ def check_server_url(url_to_test): def get_user_info(): try: resp = requests.get(urlparse.urljoin(_get_base_server_url(), "/_api/session"), cookies=_conf().getDBSessionCookies(), timeout=1) - if resp.status_code != 403: + if (resp.status_code != 401) and (resp.status_code != 403): return resp.json() else: return False From a67da8ff2c83a08693a43e335a9cf8c4226c88f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 19 Jun 2018 16:24:03 -0300 Subject: [PATCH 1390/1506] Fix potential race condition when importing users I wasn't able to reproduce the bug properly but I think this makes it more resistant to race conditions --- server/importer.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/server/importer.py b/server/importer.py index da36277650e..976232545e7 100644 --- a/server/importer.py +++ b/server/importer.py @@ -597,18 +597,20 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation yield service +user_lock = threading.Lock() def get_or_create_user(session, username): - rng = SystemRandom() - password = "".join( - [rng.choice(string.ascii_letters + string.digits) for _ in - xrange(12)]) - creator, created = get_or_create(session, User, username=username) - if created: - creator.active = False - creator.password = password - session.add(creator) # remove me - session.commit() # remove me - return creator + with user_lock: + rng = SystemRandom() + password = "".join( + [rng.choice(string.ascii_letters + string.digits) for _ in + xrange(12)]) + creator, created = get_or_create(session, User, username=username) + if created: + creator.active = False + creator.password = password + session.add(creator) # remove me + session.commit() # remove me + return creator class VulnerabilityImporter(object): From 629a54b15919d9494325bdbaea1f0095932a6c8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 19 Jun 2018 16:35:08 -0300 Subject: [PATCH 1391/1506] Fix importer error when the workspace has duplicate scope elements --- server/importer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/importer.py b/server/importer.py index 976232545e7..cd60352cb22 100644 --- a/server/importer.py +++ b/server/importer.py @@ -898,6 +898,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation workspace.end_date = datetime.datetime.fromtimestamp(float(document.get('duration')['end'])/1000) for scope in [x.strip() for x in document.get('scope', '').split('\n') if x.strip()]: scope_obj, created = get_or_create(session, Scope, name=scope, workspace=workspace) + session.flush() # This fixes integrity errors for duplicate scope elements users = document.get('users', []) if not users: workspace.public = True From b7aea3aee9065b505f32d4492c8442b360fa6c56 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 19 Jun 2018 16:59:44 -0300 Subject: [PATCH 1392/1506] Update release.md --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index fb1b1304b30..6fb9ca095c4 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -10,6 +10,7 @@ New features in the latest update TBA: --- +* Add new plugin sslyze * Add new plugin wfuzz * Add xsssniper plugin * Fix W3af, Zap From 4054b47e389eefafee7c7842c49d4fcdf4222539 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Tue, 19 Jun 2018 17:20:26 -0300 Subject: [PATCH 1393/1506] Fix bug #4906 --- manage.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/manage.py b/manage.py index eeee42afed9..c1b1172d16e 100755 --- a/manage.py +++ b/manage.py @@ -25,8 +25,9 @@ from server.web import app from utils.logs import setUpLogger +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) -@click.group() +@click.group(context_settings=CONTEXT_SETTINGS) def cli(): pass @@ -35,7 +36,7 @@ def check_faraday_server(url): return requests.get(url) -@click.command() +@click.command(help="Enable importation of plugins reports in ~/.faraday folder") @click.option('--debug/--no-debug', default=False) @click.option('--workspace', default=None) @click.option('--polling/--no-polling', default=True) @@ -64,15 +65,11 @@ def process_reports(debug, workspace, polling): print('Can\'t connect to {0}. Please check if the server is running.'.format(url)) -@click.command() +@click.command(help="Show all URLs in Faraday Server API") def show_urls(): show_all_urls() -@click.command() -def faraday_schema_display(): - DatabaseSchema().run() - -@click.command() +@click.command(help="Create Faraday DB in Postgresql, also tables and indexes") def initdb(): with app.app_context(): InitDB().run() @@ -82,16 +79,16 @@ def initdb(): ImportCouchDB().run() print('All users from CouchDB were imported. You can login with your old username/password to faraday now.') -@click.command() +@click.command(help="Import all your data from Couchdb Faraday databases") def import_from_couchdb(): with app.app_context(): ImportCouchDB().run() -@click.command() +@click.command(help="Create a PNG image with Faraday model object") def database_schema(): DatabaseSchema().run() -@click.command() +@click.command(help="Open a SQL Shell connected to postgresql 'Faraday DB'") def sql_shell(): try: from pgcli.main import PGCli @@ -105,7 +102,7 @@ def sql_shell(): pgcli.run_cli() -@click.command() +@click.command(help="Check critical modules in Faraday server application") def status_check(): full_status_check() @@ -125,7 +122,7 @@ def validate_email(ctx, param, value): return validate_user_unique_field(ctx, param, value) -@click.command() +@click.command(help="Create ADMIN user for Faraday application") @click.option('--username', prompt=True, callback=validate_user_unique_field) @click.option('--email', prompt=True, callback=validate_email) @click.option('--password', prompt=True, hide_input=True, @@ -148,7 +145,6 @@ def createsuperuser(username, email, password): cli.add_command(process_reports) cli.add_command(show_urls) -cli.add_command(faraday_schema_display) cli.add_command(initdb) cli.add_command(import_from_couchdb) cli.add_command(database_schema) @@ -158,5 +154,4 @@ def createsuperuser(username, email, password): if __name__ == '__main__': - cli() - + cli() \ No newline at end of file From 24e4af3a2c4da1490efa76fc4ecedd113503819b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 19 Jun 2018 17:23:23 -0300 Subject: [PATCH 1394/1506] Fix importer bug when hosts has to interfaces with same ip --- server/importer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/importer.py b/server/importer.py index cd60352cb22..c1905c21bed 100644 --- a/server/importer.py +++ b/server/importer.py @@ -452,12 +452,14 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation if check_ip_address(interface['ipv4']['address']): interface_ip = interface['ipv4']['address'] host, created = get_or_create(session, Host, ip=interface_ip, workspace=workspace) + session.flush() host.default_gateway_ip = interface['ipv4']['gateway'] self.merge_with_host(host, interface, workspace) hosts.append((host, created)) if check_ip_address(interface['ipv6']['address']): interface_ip = interface['ipv6']['address'] host, created = get_or_create(session, Host, ip=interface_ip, workspace=workspace) + session.flush() host.default_gateway_ip = interface['ipv6']['gateway'] self.merge_with_host(host, interface, workspace) hosts.append((host, created)) From ec458a99b5de5cd9eb932a79deb7766c87c70ca4 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Tue, 19 Jun 2018 18:35:29 -0300 Subject: [PATCH 1395/1506] Fix bug #4905 --- persistence/server/models.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/persistence/server/models.py b/persistence/server/models.py index f45b330ba85..15f39f68468 100644 --- a/persistence/server/models.py +++ b/persistence/server/models.py @@ -1035,6 +1035,8 @@ def tieBreakable(self, key): return True if key == "status": return True + if key == "refs": + return True return False def tieBreak(self, key, prop1, prop2): @@ -1043,6 +1045,10 @@ def tieBreak(self, key, prop1, prop2): Return a tuple with prop1, prop2 if we cant resolve conflict. """ + if key == "refs": + prop1.extend([x for x in prop2 if x not in prop1]) + return prop1 + if key == "confirmed": return True @@ -1268,6 +1274,9 @@ def tieBreakable(self, key): return True if key == "status": return True + if key == "refs": + return True + return False def tieBreak(self, key, prop1, prop2): @@ -1276,6 +1285,10 @@ def tieBreak(self, key, prop1, prop2): Return a tuple with prop1, prop2 if we cant resolve conflict. """ + if key == "refs": + prop1.extend([x for x in prop2 if x not in prop1]) + return prop1 + if key == "response": return self._resolve_response(prop1, prop2) From 36fd645600ef15786d3abc0a17825734b8608e80 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Tue, 19 Jun 2018 19:17:42 -0300 Subject: [PATCH 1396/1506] Fix bug #4913 --- gui/gtk/dialogs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gui/gtk/dialogs.py b/gui/gtk/dialogs.py index a4979099898..3f62c789956 100644 --- a/gui/gtk/dialogs.py +++ b/gui/gtk/dialogs.py @@ -6,12 +6,13 @@ See the file 'doc/LICENSE' for the license information ''' -import gi import webbrowser +import gi import os gi.require_version('Gtk', '3.0') +from persistence.server.server import ResourceDoesNotExist from gi.repository import Gtk, GdkPixbuf, Gdk from config.configuration import getInstanceConfiguration from persistence.server.server import is_authenticated, login_user, get_user_info, check_server_url @@ -1262,7 +1263,7 @@ def save(self, button, keeper): dialog.run() dialog.destroy() - except KeyError: # TODO: revert this hack to prevent exception when + except ResourceDoesNotExist: # TODO: revert this hack to prevent exception when # fixing conflict of non existent object dialog = Gtk.MessageDialog(self, 0, Gtk.MessageType.INFO, From 4cab3d068b709160e970d11498b3a1e00dabe94f Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Tue, 19 Jun 2018 19:49:34 -0300 Subject: [PATCH 1397/1506] Fix bug #4914 --- gui/gtk/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gui/gtk/server.py b/gui/gtk/server.py index 6fc35e50bf6..1d516fed523 100644 --- a/gui/gtk/server.py +++ b/gui/gtk/server.py @@ -113,10 +113,11 @@ def get_changes(): obj_id = obj_information.get('id') obj_type = obj_information.get('type') obj_name = obj_information.get('name') - obj = self.get_object(obj_type, obj_id) if action == 'CREATE': + obj = self.get_object(obj_type, obj_id) notification_center.addObject(obj) elif action == 'UPDATE': + obj = self.get_object(obj_type, obj_id) notification_center.editObject(obj) elif action == 'DELETE': notification_center.deleteObject(obj_id, obj_type) From 7c4a05200f8a739420f5f4176afccb205617fd25 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Tue, 19 Jun 2018 19:55:30 -0300 Subject: [PATCH 1398/1506] Fix bug with get a deleted service in GTK #4914 --- gui/gtk/mainwidgets.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gui/gtk/mainwidgets.py b/gui/gtk/mainwidgets.py index 8aeadeae2f2..47b92552ed2 100644 --- a/gui/gtk/mainwidgets.py +++ b/gui/gtk/mainwidgets.py @@ -402,8 +402,10 @@ def remove_object(self, obj_id, obj_type): """Remove an obj of id obj_id from the model, if found there""" if obj_type == 'Host': self.remove_host(host_id=obj_id) - # elif not is_host and self._is_vuln_of_host(vuln_id=obj_id, host_id=potential_host_id): - # self.remove_vuln(vuln_id=obj_id) + elif obj_type == 'Service': + # Yeah, we query to server about services + # We are not using a cached version of model + pass else: # Since we don't know the type of the delete object, # we have to assume it's a vulnerability so the host's From f34baf42735d2509e1ae394fec8f27126a2890b4 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 19 Jun 2018 19:56:53 -0300 Subject: [PATCH 1399/1506] [ADD] test for search by id --- test_cases/test_api_vulnerability.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 75a59b75a67..5013c25a502 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1439,6 +1439,16 @@ def test_after_deleting_vuln_ref_and_policies_remains(self, session, test_client assert PolicyViolation.query.count() == 1 assert Vulnerability.query.count() == 5 + def test_search_by_id(self, session, test_client): + vuln = VulnerabilityFactory.create() + vuln2 = VulnerabilityFactory.create(workspace=vuln.workspace) + session.add(vuln) + session.add(vuln2) + session.commit() + res = test_client.get(self.url(workspace=vuln.workspace) + '?id={0}'.format(vuln.id)) + assert res.json['count'] == 1 + res.json['vulnerabilities'][0]['value']['name'] == vuln.name + def test_type_filter(workspace, session, vulnerability_factory, From 69b28ce6bd9a726b78d9db0f610c544862f15d79 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 19 Jun 2018 20:00:01 -0300 Subject: [PATCH 1400/1506] [FIX] Search by vuln id --- server/api/modules/vulns.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 2817e40c256..c34a187bf87 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -264,6 +264,11 @@ class Meta: _strict_filtering = {'default_operator': operators.Equal} +class IDFilter(Filter): + def filter(self, query, model, attr, value): + return query.filter(model.id == value) + + class TargetFilter(Filter): def filter(self, query, model, attr, value): return query.filter(model.target_host_ip == value) @@ -311,7 +316,7 @@ class Meta(FilterSetMeta): # command, impact, issuetracker, tags, date, host # evidence, policy violations, hostnames fields = ( - "status", "website", "pname", "query", "path", "service", + "id", "status", "website", "pname", "query", "path", "service", "data", "severity", "confirmed", "name", "request", "response", "parameters", "params", "resolution", "ease_of_resolution", "description", "command_id", "target", "creator", "method", @@ -325,10 +330,12 @@ class Meta(FilterSetMeta): ) default_operator = CustomILike + # next line uses dict comprehensions! column_overrides = { field: _strict_filtering for field in strict_fields } operators = (CustomILike, operators.Equal) + id = IDFilter(fields.Int()) target = TargetFilter(fields.Str()) type = TypeFilter(fields.Str(validate=[OneOf(['Vulnerability', 'VulnerabilityWeb'])])) From 31867f2a141db093cbb24b0932a51c299bbc75d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 21 Jun 2018 11:18:54 -0300 Subject: [PATCH 1401/1506] Add create_tables command for users not using the initdb Advanced PostgreSQL configurations are not contemplated by the initdb script, so allow the user to configure it manually and then create the tables with this command --- manage.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/manage.py b/manage.py index c1b1172d16e..72a4836904e 100755 --- a/manage.py +++ b/manage.py @@ -143,6 +143,20 @@ def createsuperuser(username, email, password): fg='green', bold=True)) +@click.command(help="Create database tables. Requires a functional " + "PostgreSQL database configured in the server.ini") +def create_tables(): + with app.app_context(): + # Ugly hack to create tables and also setting alembic revision + import server.config + conn_string = server.config.database.connection_string + from server.commands.initdb import InitDB + InitDB()._create_tables(conn_string) + click.echo(click.style( + 'Tables created successfully!', + fg='green', bold=True)) + + cli.add_command(process_reports) cli.add_command(show_urls) cli.add_command(initdb) @@ -151,7 +165,8 @@ def createsuperuser(username, email, password): cli.add_command(createsuperuser) cli.add_command(sql_shell) cli.add_command(status_check) +cli.add_command(create_tables) if __name__ == '__main__': - cli() \ No newline at end of file + cli() From f3b2d1050e5d1400eb716aed112953ca5b801e1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 21 Jun 2018 11:49:59 -0300 Subject: [PATCH 1402/1506] Use ilike filtering for target field in vulns api Instead of exact matching --- server/api/modules/vulns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index c34a187bf87..154006fb198 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -271,7 +271,7 @@ def filter(self, query, model, attr, value): class TargetFilter(Filter): def filter(self, query, model, attr, value): - return query.filter(model.target_host_ip == value) + return query.filter(model.target_host_ip.ilike("%" + value + "%")) class TypeFilter(Filter): From 6796c2f397a5b45a005804187383901df983c84b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 21 Jun 2018 12:21:27 -0300 Subject: [PATCH 1403/1506] Fix filtering by status Allow using "open" and "opened" as values --- server/api/modules/vulns.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 154006fb198..918c64b1430 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -349,6 +349,10 @@ class Meta(FilterSetMeta): pname = Filter(fields.String(attribute='parameter_name')) query = Filter(fields.String(attribute='query_string')) params = Filter(fields.String(attribute='parameters')) + status = Filter(fields.Function( + deserialize=lambda val: 'open' if val == 'opened' else val, + validate=OneOf(Vulnerability.STATUSES + ['opened']) + )) def filter(self): """Generate a filtered query from request parameters. From aea752ebac389acabd0e2182838b52512abbab73 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 21 Jun 2018 12:32:16 -0300 Subject: [PATCH 1404/1506] [FIX] Update code to work with pip 10 --- utils/dependencies.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/utils/dependencies.py b/utils/dependencies.py index 8d80fd42ad9..2ff854b7306 100644 --- a/utils/dependencies.py +++ b/utils/dependencies.py @@ -5,7 +5,11 @@ ''' import sys -import pip +try: + from pip import main +except ImportError: + # pip 10 compat + from pip._internal import main import pkg_resources @@ -37,4 +41,4 @@ def install_packages(packages): pip_cmd = ['install', package, '-U'] if not hasattr(sys, 'real_prefix'): pip_cmd.append('--user') - pip.main(pip_cmd) + main(pip_cmd) From 541e9d5860d42808d7c85fe61f336f0297f0cae6 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 21 Jun 2018 14:25:16 -0300 Subject: [PATCH 1405/1506] [FIX] Fix daemon start and stop --- server/web.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/server/web.py b/server/web.py index df409425dc9..3ac73cad144 100644 --- a/server/web.py +++ b/server/web.py @@ -5,7 +5,7 @@ import os import sys import functools -from signal import SIGABRT, SIGILL, SIGINT, SIGSEGV, SIGTERM, signal +from signal import SIGABRT, SIGILL, SIGINT, SIGSEGV, SIGTERM, SIG_DFL, signal import twisted.web from twisted.web.resource import Resource, ForbiddenResource @@ -134,15 +134,17 @@ def __build_websockets_resource(self): def install_signal(self): for sig in (SIGABRT, SIGILL, SIGINT, SIGSEGV, SIGTERM): - def signal_handler(*args): - logger.info("Stopping threads, please wait...") - # teardown() - self.raw_report_processor.stop() - reactor.stop() + signal(sig, SIG_DFL) + - signal(sig, signal_handler) def run(self): + def signal_handler(*args): + logger.info("Stopping threads, please wait...") + # teardown() + self.raw_report_processor.stop() + reactor.stop() + site = twisted.web.server.Site(self.__root_resource) if self.__ssl_enabled: ssl_context = self.__load_ssl_certs() @@ -167,6 +169,7 @@ def run(self): except : logger.warn('Could not start websockets, address already open. This is ok is you wan to run multiple instances.') logger.info('Faraday Server is ready') + reactor.addSystemEventTrigger('before', 'shutdown', signal_handler) reactor.run() except error.CannotListenError as e: From 4f8d3b1963b1ee564d7a7fd6c0b066fa9b60a316 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 21 Jun 2018 17:32:48 -0300 Subject: [PATCH 1406/1506] Fix to allow faraday-server on multiple ports. Daemon will only work in one port for now --- faraday-server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/faraday-server.py b/faraday-server.py index 470199d7278..2ede16436c8 100755 --- a/faraday-server.py +++ b/faraday-server.py @@ -131,7 +131,7 @@ def main(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) result = sock.connect_ex((args.bind_address or server.config.faraday_server.bind_address, int(args.port or server.config.faraday_server.port))) - if is_server_running(): + if is_server_running() and result == 0: sys.exit(1) if result == 0: From 1b329a4ae655f2dc98f7eace99a788e805e3a328 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 21 Jun 2018 18:32:01 -0300 Subject: [PATCH 1407/1506] update release --- RELEASE.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 6fb9ca095c4..554bb84082b 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -10,14 +10,25 @@ New features in the latest update TBA: --- +* Add hostname to host +* Interface removed from model and from persistence server lib (fplugin) +* Performance iprovements on the backend +* Add quick change workspace name (from all views) +* Allow user to change workspace +* New faraday styles in all webui views +* Add search by id for vulnerabilities * Add new plugin sslyze * Add new plugin wfuzz * Add xsssniper plugin -* Fix W3af, Zap +* Fix W3af, Zap plugins * Add brutexss plugin * Allow to upload report file from external tools from the web * Fix sshcheck import file from GTK -* Add reconng, sublist3r, HP Webinspect, dirsearch and ip360 plugins +* Add reconng plugin +* Add sublist3r plugin +* Add HP Webinspect plugin +* Add dirsearch plugin +* Add ip360 plugin * CouchDB was replaced by PostgreSQL :) * Host object changed, now the name property is called ip * Interface object was removed From 324c4630b201a062a339095d0933e68494136dad Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Thu, 21 Jun 2018 18:43:51 -0300 Subject: [PATCH 1408/1506] Fix bug #4904 --- server/www/scripts/statusReport/controllers/statusReport.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index e047e7a8114..3cf5a8e87c4 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -211,7 +211,7 @@ angular.module("faradayApp") "service": "110", "hostnames": "130", "target": "100", - "desc": "200", + "desc": "600", "resolution": "170", "data": "170", "easeofresolution": "140", @@ -332,7 +332,9 @@ angular.module("faradayApp") cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/desccolumn.html', headerCellTemplate: header, sort: getColumnSort('desc'), - visible: $scope.columns["desc"] + visible: $scope.columns["desc"], + minWidth: '300', + maxWidth: '600', }); $scope.gridOptions.columnDefs.push({ name : 'resolution', cellTemplate: 'scripts/statusReport/partials/ui-grid/columns/resolutioncolumn.html', From 2a1ad520b28c253aa64983151d9f027e470ff21a Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Thu, 21 Jun 2018 20:19:25 -0300 Subject: [PATCH 1409/1506] Fix bug #4907 status href with \ --- .../statusReport/partials/ui-grid/columns/statuscolumn.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html index 9b4ba5fee07..1abaf9841fd 100644 --- a/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html +++ b/server/www/scripts/statusReport/partials/ui-grid/columns/statuscolumn.html @@ -1,12 +1,12 @@
    - + {{COL_FIELD | uppercase}}
    - + {{COL_FIELD | uppercase}}
    \ No newline at end of file From 65ddb4401001a5fe8f9b58e15c585fea93ea94dd Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 22 Jun 2018 14:21:05 -0300 Subject: [PATCH 1410/1506] Disable flask session due to a bug with gtk client --- server/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/app.py b/server/app.py index 680995514d3..801e49af1cf 100644 --- a/server/app.py +++ b/server/app.py @@ -207,7 +207,7 @@ def create_app(db_connection_string=None, testing=None): from server.models import db db.init_app(app) - Session(app) + #Session(app) # Setup Flask-Security app.user_datastore = SQLAlchemyUserDatastore( From 80c04e8b5cbe544483af5bb44aa7d1df9e2b8e75 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 22 Jun 2018 16:55:49 -0300 Subject: [PATCH 1411/1506] Fix bug in hostnames API #4907 --- server/api/modules/vulns.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 918c64b1430..4a646a2f009 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -33,7 +33,8 @@ Vulnerability, VulnerabilityWeb, VulnerabilityGeneric, - Workspace + Workspace, + Hostname ) from server.utils.database import get_or_create @@ -298,6 +299,21 @@ def filter(self, query, model, attr, value): alias.name == value ) +class HostnamesFilter(Filter): + def filter(self, query, model, attr, value): + alias = aliased(Hostname, name='hostname_filter') + + service_hostnames_query = query.join(Service, Service.id == Vulnerability.service_id).\ + join(Host).\ + join(alias).\ + filter(alias.name == value) + + host_hostnames_query = query.join(Host, Host.id == Vulnerability.host_id).\ + join(alias).\ + filter(alias.name == value) + + query = service_hostnames_query.union(host_hostnames_query) + return query class CustomILike(operators.Operator): """A filter operator that puts a % in the beggining and in the @@ -353,6 +369,7 @@ class Meta(FilterSetMeta): deserialize=lambda val: 'open' if val == 'opened' else val, validate=OneOf(Vulnerability.STATUSES + ['opened']) )) + hostnames = HostnamesFilter(fields.Str()) def filter(self): """Generate a filtered query from request parameters. From c220dec957ffedd23b908d37b580f057e006ac3f Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 22 Jun 2018 16:57:14 -0300 Subject: [PATCH 1412/1506] Commit test for hostnames --- test_cases/test_api_vulnerability.py | 43 ++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 5013c25a502..f38e1b5e7e8 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -45,7 +45,7 @@ from test_cases.factories import ServiceFactory, CommandFactory, \ CommandObjectFactory, HostFactory, EmptyCommandFactory, \ UserFactory, VulnerabilityWebFactory, VulnerabilityFactory, \ - ReferenceFactory, PolicyViolationFactory + ReferenceFactory, PolicyViolationFactory, HostnameFactory, WorkspaceFactory CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) @@ -1447,7 +1447,46 @@ def test_search_by_id(self, session, test_client): session.commit() res = test_client.get(self.url(workspace=vuln.workspace) + '?id={0}'.format(vuln.id)) assert res.json['count'] == 1 - res.json['vulnerabilities'][0]['value']['name'] == vuln.name + assert res.json['vulnerabilities'][0]['value']['name'] == vuln.name + + def test_search_by_hostnames_service_case(self, session, test_client): + workspace = WorkspaceFactory.create() + vuln2 = VulnerabilityFactory.create(workspace=workspace) + hostname = HostnameFactory.create(workspace=workspace, name='test.com') + host = HostFactory.create(workspace=workspace) + host.hostnames.append(hostname) + service = ServiceFactory.create(workspace=workspace, host=host) + vuln = VulnerabilityFactory.create(service=service, workspace=workspace) + session.add(vuln) + session.add(vuln2) + session.add(hostname) + session.commit() + url = self.url(workspace=workspace) + '?hostnames={0}'.format(hostname.name) + res = test_client.get(url) + import ipdb; ipdb.set_trace() + + assert res.status_code == 200 + assert res.json['count'] == 1 + assert res.json['vulnerabilities'][0]['value']['name'] == vuln.name + + def test_search_by_hostnames_host_case(self, session, test_client): + workspace = WorkspaceFactory.create() + vuln2 = VulnerabilityFactory.create(workspace=workspace) + hostname = HostnameFactory.create(workspace=workspace, name='test.com') + host = HostFactory.create(workspace=workspace) + host.hostnames.append(hostname) + vuln = VulnerabilityFactory.create(host=host, workspace=workspace) + session.add(vuln) + session.add(vuln2) + session.add(hostname) + session.commit() + import ipdb; ipdb.set_trace() + url = self.url(workspace=workspace) + '?hostnames={0}'.format(hostname.name) + res = test_client.get(url) + + assert res.status_code == 200 + assert res.json['count'] == 1 + assert res.json['vulnerabilities'][0]['value']['name'] == vuln.name def test_type_filter(workspace, session, From c4bd5a7433bb2354a5c01a28af4e1489fb29eb73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 22 Jun 2018 17:56:13 -0300 Subject: [PATCH 1413/1506] Add extra flushs to couchdb importer This probably doesn't make any difference --- server/importer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/importer.py b/server/importer.py index c1905c21bed..437b2b0910a 100644 --- a/server/importer.py +++ b/server/importer.py @@ -592,8 +592,8 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation service.status = status_mapper.get(couchdb_status, 'open') service.version = document.get('version') service.workspace = workspace + session.flush() if command and created: - session.flush() CommandObject.create(service, command) yield service @@ -690,8 +690,8 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation vulnerability.impact_availability = document.get('impact', {}).get('availability') or False vulnerability.impact_confidentiality = document.get('impact', {}).get('confidentiality') or False vulnerability.impact_integrity = document.get('impact', {}).get('integrity') or False + session.flush() if command and created: - session.flush() CommandObject.create(vulnerability, command) if document['type'] == 'VulnerabilityWeb': vulnerability.query_string = document.get('query') From 2590008752383b3a40d6481a6c42cdfd1f71044b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 22 Jun 2018 17:56:53 -0300 Subject: [PATCH 1414/1506] Add vulnerability.type to unique index Otherwise, the importer would break with a vuln web with similar data to a service vulnerability --- server/models.py | 4 ++-- server/utils/database.py | 1 + test_cases/test_api_vulnerability.py | 6 +++--- test_cases/test_utils_database.py | 3 ++- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/server/models.py b/server/models.py index 821d0317bcc..3168cad4b42 100644 --- a/server/models.py +++ b/server/models.py @@ -1610,14 +1610,14 @@ def attachments(self): vulnerability_uniqueness = DDL( "CREATE UNIQUE INDEX uix_vulnerability ON %(fullname)s " - "(md5(name), md5(description), COALESCE(host_id, -1), COALESCE(service_id, -1), " + "(md5(name), md5(description), type, COALESCE(host_id, -1), COALESCE(service_id, -1), " "COALESCE(md5(method), ''), COALESCE(md5(parameter_name), ''), COALESCE(md5(path), ''), " "COALESCE(md5(website), ''), workspace_id, COALESCE(source_code_id, -1));" ) vulnerability_uniqueness_sqlite = DDL( "CREATE UNIQUE INDEX uix_vulnerability ON %(fullname)s " - "(name, description, COALESCE(host_id, -1), COALESCE(service_id, -1), " + "(name, description, type, COALESCE(host_id, -1), COALESCE(service_id, -1), " "COALESCE(method, ''), COALESCE(parameter_name, ''), COALESCE(path, ''), " "COALESCE(website, ''), workspace_id, COALESCE(source_code_id, -1));" ) diff --git a/server/utils/database.py b/server/utils/database.py index ed9f0a08646..2552165e004 100644 --- a/server/utils/database.py +++ b/server/utils/database.py @@ -235,6 +235,7 @@ def get_unique_fields(session, instance): 'column_names': [ 'name', 'description', + 'type', 'host_id', 'service_id', 'method', diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 5013c25a502..4c050582061 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1242,7 +1242,7 @@ def test_create_webvuln_multiple_times_returns_conflict(self, host_with_hostname data=raw_data) assert res.status_code == 409 - def test_create_vuln_service_vuln_web_conflict( + def test_create_similar_vuln_service_and_vuln_web_conflict_succeed( self, service, vulnerability_factory, vulnerability_web_factory, session, test_client, workspace): service_vuln = vulnerability_factory.create( @@ -1263,8 +1263,8 @@ def test_create_vuln_service_vuln_web_conflict( ) raw_data['type'] = 'VulnerabilityWeb' res = test_client.post(self.url(), data=raw_data) - assert res.status_code == 409 - assert VulnerabilityGeneric.query.count() == old_count + assert res.status_code == 201 + assert VulnerabilityGeneric.query.count() == old_count + 1 def test_update_conflict(self, host, vulnerability_factory, session, test_client): diff --git a/test_cases/test_utils_database.py b/test_cases/test_utils_database.py index 4fa88dba492..26ed99c497b 100644 --- a/test_cases/test_utils_database.py +++ b/test_cases/test_utils_database.py @@ -23,6 +23,7 @@ Vulnerability: [ 'name', 'description', + 'type', 'host_id', 'service_id', 'method', @@ -53,7 +54,7 @@ def test_vulnerability_ddl_invariant(session): for statement_clean in statements_clean: statement_clean = statement_clean if statement_clean not in unique_constraint: - raise Exception('Please check server.data.utils.get_unique_fields. Vulnerability DDL changed?') + raise Exception('Please check server.utils.database.get_unique_fields. Vulnerability DDL changed?') @pytest.mark.parametrize("obj_class, expected_unique_fields", UNIQUE_FIELDS.items()) From 04f1c7d7c9351a45c2e1972e5a531602a8f8ff79 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 22 Jun 2018 18:36:48 -0300 Subject: [PATCH 1415/1506] [ADD] if beta in version, skip version check --- faraday.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/faraday.py b/faraday.py index ba30d562b4a..202b74290ba 100755 --- a/faraday.py +++ b/faraday.py @@ -420,6 +420,9 @@ def checkUpdates(): resp = resp.text.strip() except Exception as e: logger.error(e) + version = getInstanceConfiguration().getVersion() + if 'b' in version.split("+")[0]: + return if not resp == u'OK': logger.info("You have available updates. Run ./faraday.py --update to catchup!") else: From f50ad9c3a74607a7da764d7086072a8256d158f5 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 22 Jun 2018 18:42:07 -0300 Subject: [PATCH 1416/1506] Update version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2e2c8b81e03..391435a30dc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0a1+white +3.0b1+white From 2a16dbcf39ea3e52c4375750cfa7799bd7ceb6bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Fri, 22 Jun 2018 19:06:21 -0300 Subject: [PATCH 1417/1506] Fix command importing with duplicated couch databases It was mixing data from both workspaces. This isn't a security vulnerability, since it will only happen on duplicated databases --- server/importer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/importer.py b/server/importer.py index 437b2b0910a..9bdd26a5111 100644 --- a/server/importer.py +++ b/server/importer.py @@ -336,7 +336,7 @@ def update_command_tools(workspace, command_tool_map, id_map): command_id )) continue - assert workspace.id == command.workspace_id + assert workspace.id == command.workspace_id, (workspace.id, command.workspace_id) if command.tool and command.tool != 'unknown': logger.warn("Command {} (Couch ID {}) has already a tool. " "Overriding it".format(command_id, @@ -812,6 +812,7 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation Command, command=document.get('command', None), start_date=start_date, + workspace=workspace, ) if document.get('duration'): From 9c6de0c4d42e692de0b7dde0561233611de430de Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 25 Jun 2018 15:02:20 +0000 Subject: [PATCH 1418/1506] Show a message to the user when xdot is missing --- server/commands/faraday_schema_display.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/commands/faraday_schema_display.py b/server/commands/faraday_schema_display.py index e62181e75d3..68ae99aca3f 100644 --- a/server/commands/faraday_schema_display.py +++ b/server/commands/faraday_schema_display.py @@ -38,7 +38,14 @@ def _draw_entity_diagrama(self): rankdir='LR', # From left to right (instead of top to bottom) concentrate=False # Don't try to join the relation lines together ) - graph.write_png('entity_dbschema.png') # write out the file + try: + graph.write_png('entity_dbschema.png') # write out the file + except OSError as ex: + if 'dot' in ex.strerror: + print('Rendering entity scheam requires dot. Please install it with: sudo apt install xdot') + sys.exit(1) + raise + def _draw_uml_class_diagram(self): # lets find all the mappers in our model From c738443230e43a36e7a7647044f9bbe6ab9c063f Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Mon, 25 Jun 2018 13:16:07 -0300 Subject: [PATCH 1419/1506] [MOD] Take the hostnames search parameter to a list comma separated --- server/api/modules/vulns.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index 4a646a2f009..f13a58f2ea1 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -303,14 +303,16 @@ class HostnamesFilter(Filter): def filter(self, query, model, attr, value): alias = aliased(Hostname, name='hostname_filter') + value_list = value.split(",") + service_hostnames_query = query.join(Service, Service.id == Vulnerability.service_id).\ join(Host).\ join(alias).\ - filter(alias.name == value) + filter(alias.name.in_(value_list)) host_hostnames_query = query.join(Host, Host.id == Vulnerability.host_id).\ join(alias).\ - filter(alias.name == value) + filter(alias.name.in_(value_list)) query = service_hostnames_query.union(host_hostnames_query) return query From f1cebfb5ee8a29add525850dd0ec459f7d6671c2 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Mon, 25 Jun 2018 14:18:43 -0300 Subject: [PATCH 1420/1506] [ADD] Comma-separated list hostname test --- test_cases/test_api_vulnerability.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 4c050582061..7eab88f7e52 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -45,7 +45,7 @@ from test_cases.factories import ServiceFactory, CommandFactory, \ CommandObjectFactory, HostFactory, EmptyCommandFactory, \ UserFactory, VulnerabilityWebFactory, VulnerabilityFactory, \ - ReferenceFactory, PolicyViolationFactory + ReferenceFactory, PolicyViolationFactory, HostnameFactory CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) @@ -1449,6 +1449,27 @@ def test_search_by_id(self, session, test_client): assert res.json['count'] == 1 res.json['vulnerabilities'][0]['value']['name'] == vuln.name + def test_hostnames_comma_separated(self, test_client, session): + #Create Host A with hostname HA + hostnameA = HostnameFactory.create() + hostnameA.host.workspace = hostnameA.workspace + #Create Host B with hostname HB + hostnameB = HostnameFactory.create(workspace=hostnameA.workspace) + hostnameB.host.workspace = hostnameA.workspace + #Create Vuln with Host A + vuln = VulnerabilityFactory.create(host=hostnameA.host, workspace=hostnameA.workspace) + #Create Vuln with Host B + vuln2 = VulnerabilityFactory.create(host=hostnameB.host, workspace=hostnameA.workspace) + session.add(hostnameA) + session.add(hostnameB) + session.add(vuln) + session.add(vuln2) + session.commit() + + #Search with hosnames=HA,HB + res = test_client.get(self.url(workspace=vuln.workspace) + '?hostname={0},{1}'.format(hostnameA,hostnameB)) + assert res.status_code == 200 + assert res.json['count'] == 2 def test_type_filter(workspace, session, vulnerability_factory, From a31d453657c4913aa3fb0f093acdf2865ba37ba8 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 25 Jun 2018 17:39:27 +0000 Subject: [PATCH 1421/1506] [FIX] remove duplicate parameter --- server/www/scripts/services/controllers/serviceModalEdit.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/services/controllers/serviceModalEdit.js b/server/www/scripts/services/controllers/serviceModalEdit.js index 16e95deddaf..3e1c8cdda67 100644 --- a/server/www/scripts/services/controllers/serviceModalEdit.js +++ b/server/www/scripts/services/controllers/serviceModalEdit.js @@ -4,8 +4,8 @@ angular.module('faradayApp') .controller('serviceModalEdit', - ['$q', '$scope', '$modalInstance', '$routeParams', 'SERVICE_STATUSES', 'service', 'servicesManager', 'commonsFact', 'servicesManager', - function($q, $scope, $modalInstance, $routeParams, SERVICE_STATUSES, service, servicesManager, commonsFact, servicesManager) { + ['$q', '$scope', '$modalInstance', '$routeParams', 'SERVICE_STATUSES', 'service', 'servicesManager', 'commonsFact', + function($q, $scope, $modalInstance, $routeParams, SERVICE_STATUSES, service, servicesManager, commonsFact) { init = function() { // current Workspace From f994c41c2d04ff1d6cefbbdb839e019d2a997367 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 26 Jun 2018 19:56:40 -0300 Subject: [PATCH 1422/1506] Update RELEASE.md --- RELEASE.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 554bb84082b..30b9245c8a5 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -10,11 +10,10 @@ New features in the latest update TBA: --- -* Add hostname to host * Interface removed from model and from persistence server lib (fplugin) * Performance iprovements on the backend * Add quick change workspace name (from all views) -* Allow user to change workspace +* Changed the scope field of a workspace from a free text input to a list of targets * New faraday styles in all webui views * Add search by id for vulnerabilities * Add new plugin sslyze @@ -40,17 +39,15 @@ TBA: * Allow the user to specify the desired fields of the host list table * Add optional hostnames, services, MAC and description fields to the host list * Workspace names can be changed from the Web UI -* Changed the scope field of a workspace from a free text input to a list of targets * Exploitation and severity fields only allow certain values. CWE CVEs were fixed to be valid. A script to convert custom CSVs was added. * Web UI path changed from /_ui/ to / (_ui has now a redirection to / for keeping backwards compatibility) -* dirb plugin should creates a vulnerability type information instead of a note. +* dirb plugin creates an informational vulnerability instead of a note. * Add confirmed column to exported csv from webui * Fixes in Arachni plugin * Add new parameters --keep-old and --keep-new for faraday CLI * Add new screenshot fplugin which takes a screenshot of the ip:ports of a given protocol * Add fix for net sparker regular and cloud fix on severity * Removed Chat feature (data is kept inside notes) -* Plugin reports now can be imported in the server, from the Web UI * Add CVSS score to reference field in Nessus plugin. * Fix unicode characters bug in Netsparker plugin. * Fix qualys plugin. From ab316b82a74c6874fe9769aa36f8019457eb6ffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 26 Jun 2018 19:57:00 -0300 Subject: [PATCH 1423/1506] Fix typo in vuln create error message --- server/www/scripts/statusReport/controllers/modalNew.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/www/scripts/statusReport/controllers/modalNew.js b/server/www/scripts/statusReport/controllers/modalNew.js index 41d9f4567c4..faed51fcea9 100644 --- a/server/www/scripts/statusReport/controllers/modalNew.js +++ b/server/www/scripts/statusReport/controllers/modalNew.js @@ -139,7 +139,7 @@ angular.module('faradayApp') $modalInstance.close(vm.data); }, function(response){ if (response.status == 409) { - commonsFact.showMessage("Error while creating a new Vulnerability " + vm.data.name + " Conflicting Vulnarability with id: " + response.data.object._id + ". " + response.data.message); + commonsFact.showMessage("Error while creating a new Vulnerability " + vm.data.name + " Conflicting Vulnerability with id: " + response.data.object._id + ". " + response.data.message); } else { commonsFact.showMessage("Error from backend: " + response.status); } From fd1939026eb9e8b6e32724ea9b60ccef2f5b1a50 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 27 Jun 2018 15:41:11 -0300 Subject: [PATCH 1424/1506] [FIX] version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 391435a30dc..566a271a6f5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0b1+white +3.0b2 From 374e24d8cb1196f8e4f8993f45af0a5ae4ccb0a7 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 27 Jun 2018 15:48:26 -0300 Subject: [PATCH 1425/1506] Update version to beta3 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 566a271a6f5..e536b8a2f28 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0b2 +3.0b3 From e47c68dfee80bec6e72880d3c1016b412fc76430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 27 Jun 2018 15:55:36 -0300 Subject: [PATCH 1426/1506] Revert "Add hardcoded rollbar credentials" This reverts commit 108c2a17832e91b461001095765dc2fd06772412. --- server/www/index.html | 1 - server/www/script/ng-rollbar.min.js | 1 - server/www/scripts/app.js | 12 +++--------- 3 files changed, 3 insertions(+), 11 deletions(-) delete mode 100644 server/www/script/ng-rollbar.min.js diff --git a/server/www/index.html b/server/www/index.html index 7d5f5fd27ec..b34bf0826f5 100644 --- a/server/www/index.html +++ b/server/www/index.html @@ -83,7 +83,6 @@ - diff --git a/server/www/script/ng-rollbar.min.js b/server/www/script/ng-rollbar.min.js deleted file mode 100644 index 535ad1bee45..00000000000 --- a/server/www/script/ng-rollbar.min.js +++ /dev/null @@ -1 +0,0 @@ -(function(angular){angular.module("tandibar/ng-rollbar",[]);angular.module("tandibar/ng-rollbar").config(["$provide",function($provide){$provide.decorator("$exceptionHandler",["$delegate","$injector","$window",function($delegate,$injector,$window){return function(exception,cause){if($window.Rollbar){$window.Rollbar.error(exception,{cause:cause},function(err,data){var $rootScope=$injector.get("$rootScope");$rootScope.$emit("rollbar:exception",{exception:exception,err:err,data:data.result})})}$delegate(exception,cause)}}])}]);angular.module("tandibar/ng-rollbar").provider("Rollbar",function RollbarProvider(){var rollbarProvider=this;var rollbarActivated=true;this.init=function(config){var _rollbarConfig=config;if(rollbarActivated){!function(r){function o(n){if(e[n])return e[n].exports;var t=e[n]={exports:{},id:n,loaded:!1};return r[n].call(t.exports,t,t.exports,o),t.loaded=!0,t.exports}var e={};return o.m=r,o.c=e,o.p="",o(0)}([function(r,o,e){"use strict";var n=e(1),t=e(4);_rollbarConfig=_rollbarConfig||{},_rollbarConfig.rollbarJsUrl=_rollbarConfig.rollbarJsUrl||"https://cdnjs.cloudflare.com/ajax/libs/rollbar.js/2.3.1/rollbar.min.js",_rollbarConfig.async=void 0===_rollbarConfig.async||_rollbarConfig.async;var a=n.setupShim(window,_rollbarConfig),l=t(_rollbarConfig);window.rollbar=n.Rollbar,a.loadFull(window,document,!_rollbarConfig.async,_rollbarConfig,l)},function(r,o,e){"use strict";function n(r){return function(){try{return r.apply(this,arguments)}catch(r){try{console.error("[Rollbar]: Internal error",r)}catch(r){}}}}function t(r,o){this.options=r,this._rollbarOldOnError=null;var e=s++;this.shimId=function(){return e},window&&window._rollbarShims&&(window._rollbarShims[e]={handler:o,messages:[]})}function a(r,o){var e=o.globalAlias||"Rollbar";if("object"==typeof r[e])return r[e];r._rollbarShims={},r._rollbarWrappedError=null;var t=new p(o);return n(function(){o.captureUncaught&&(t._rollbarOldOnError=r.onerror,i.captureUncaughtExceptions(r,t,!0),i.wrapGlobals(r,t,!0)),o.captureUnhandledRejections&&i.captureUnhandledRejections(r,t,!0);var n=o.autoInstrument;return o.enabled!==!1&&(void 0===n||n===!0||"object"==typeof n&&n.network)&&r.addEventListener&&(r.addEventListener("load",t.captureLoad.bind(t)),r.addEventListener("DOMContentLoaded",t.captureDomContentLoaded.bind(t))),r[e]=t,t})()}function l(r){return n(function(){var o=this,e=Array.prototype.slice.call(arguments,0),n={shim:o,method:r,args:e,ts:new Date};window._rollbarShims[this.shimId()].messages.push(n)})}var i=e(2),s=0,d=e(3),c=function(r,o){return new t(r,o)},p=d.bind(null,c);t.prototype.loadFull=function(r,o,e,t,a){var l=function(){var o;if(void 0===r._rollbarDidLoad){o=new Error("rollbar.js did not load");for(var e,n,t,l,i=0;e=r._rollbarShims[i++];)for(e=e.messages||[];n=e.shift();)for(t=n.args||[],i=0;i Date: Wed, 27 Jun 2018 15:59:55 -0300 Subject: [PATCH 1427/1506] Update version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e536b8a2f28..70a5c59c74c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0b3 +3.0b4 From b0998c327f1cfc1442615a6c0bd5e1f1f09a89e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 27 Jun 2018 16:54:42 -0300 Subject: [PATCH 1428/1506] Adapt service important to actual unique constraints --- server/importer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/importer.py b/server/importer.py index 9bdd26a5111..001e3a892b1 100644 --- a/server/importer.py +++ b/server/importer.py @@ -564,13 +564,13 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation port = 65535 service, created = get_or_create(session, Service, - name=document.get('name'), + protocol=document.get('protocol'), port=port, host=host) service.description = document.get('description') service.owned = document.get('owned', False) service.banner = document.get('banner') - service.protocol = document.get('protocol') + service.name = document.get('name') if not document.get('status'): logger.warning('Service {0} with empty status. Using \'open\' as status'.format(document['_id'])) document['status'] = 'open' From 27aa1e74d95ec1e6386bf59ad8e5cc49a305a05b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 27 Jun 2018 16:56:31 -0300 Subject: [PATCH 1429/1506] Fix string encoding bug on importer I don't know what caused it, just skipped the conflicting characters :P --- server/utils/invalid_chars.py | 44 +++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/server/utils/invalid_chars.py b/server/utils/invalid_chars.py index 97238a5cfa5..db0278c2f9c 100644 --- a/server/utils/invalid_chars.py +++ b/server/utils/invalid_chars.py @@ -46,26 +46,30 @@ def clean_string(s): def clean_char(char): - #Get rid of the ctrl characters first. - #http://stackoverflow.com/questions/1833873/python-regex-escape-characters - char = re.sub('\x1b[^m]*m', '', char) - #Clean up invalid xml - char = remove_invalid_chars(char) - replacements = [ - (u'\u201c', '\"'), - (u'\u201d', '\"'), - (u"\u001B", ' '), #http://www.fileformat.info/info/unicode/char/1b/index.htm - (u"\u0019", ' '), #http://www.fileformat.info/info/unicode/char/19/index.htm - (u"\u0016", ' '), #http://www.fileformat.info/info/unicode/char/16/index.htm - (u"\u001C", ' '), #http://www.fileformat.info/info/unicode/char/1c/index.htm - (u"\u0003", ' '), #http://www.utf8-chartable.de/unicode-utf8-table.pl?utf8=0x - (u"\u000C", ' ') - ] - for rep, new_char in replacements: - if char == rep: - #print ord(char), char.encode('ascii', 'ignore') - return new_char - return char + try: + #Get rid of the ctrl characters first. + #http://stackoverflow.com/questions/1833873/python-regex-escape-characters + char = re.sub('\x1b[^m]*m', '', char) + #Clean up invalid xml + char = remove_invalid_chars(char) + replacements = [ + (u'\u201c', '\"'), + (u'\u201d', '\"'), + (u"\u001B", ' '), #http://www.fileformat.info/info/unicode/char/1b/index.htm + (u"\u0019", ' '), #http://www.fileformat.info/info/unicode/char/19/index.htm + (u"\u0016", ' '), #http://www.fileformat.info/info/unicode/char/16/index.htm + (u"\u001C", ' '), #http://www.fileformat.info/info/unicode/char/1c/index.htm + (u"\u0003", ' '), #http://www.utf8-chartable.de/unicode-utf8-table.pl?utf8=0x + (u"\u000C", ' ') + ] + for rep, new_char in replacements: + if char == rep: + #print ord(char), char.encode('ascii', 'ignore') + return new_char + return char + except UnicodeEncodeError: + # Ugly hack triggered when importing some strange objects + return '' def remove_invalid_chars(c): From c57fe314ea01160a66d8fb4fa44d948ad51b19b5 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Wed, 27 Jun 2018 18:04:44 -0300 Subject: [PATCH 1430/1506] Fix bug Object Object in Workspace edit Webui #4911 --- server/www/scripts/commons/controllers/headerCtrl.js | 4 +++- server/www/scripts/statusReport/controllers/statusReport.js | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/server/www/scripts/commons/controllers/headerCtrl.js b/server/www/scripts/commons/controllers/headerCtrl.js index c9a4e5e7db7..e4e733d4a11 100644 --- a/server/www/scripts/commons/controllers/headerCtrl.js +++ b/server/www/scripts/commons/controllers/headerCtrl.js @@ -87,7 +87,9 @@ angular.module('faradayApp') // copy pasted from server/www/scripts/workspaces/controllers/workspaces.js // it makes scope work properly (i think) workspace.scope = workspace.scope.map(function(scope){ - return {key: scope} + if(scope.key === undefined) + return {key: scope}; + return scope; }); if (workspace.scope.length == 0) workspace.scope.push({key: ''}); diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 3cf5a8e87c4..a1b3cf90b45 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -46,7 +46,6 @@ angular.module("faradayApp") var init = function() { $scope.baseurl = BASEURL; - console.log($scope.baseurl); $scope.severities = SEVERITIES; $scope.easeofresolution = EASEOFRESOLUTION; $scope.propertyGroupBy = $routeParams.groupbyId; From 47517018d40e31c9ec47c6628d5f4dff20d356bb Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Thu, 28 Jun 2018 17:28:09 -0300 Subject: [PATCH 1431/1506] [FIX] If server is not available, print an message and exit --- faraday.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/faraday.py b/faraday.py index 202b74290ba..afd8ec49a2b 100755 --- a/faraday.py +++ b/faraday.py @@ -449,6 +449,12 @@ def checkServerUrl(): sys.exit(-1) except requests.exceptions.MissingSchema as ex: print("Check ~/.faraday/config/user.xml server url, the following error was found: {0} ".format(ex)) + except requests.exceptions.ConnectionError: + print("\n\n" + "Can't connect to server. \nCheck if server is running or the file ~/.faraday/config/user.xml server url." + "\n\n" + ) + sys.exit(-1) def check_faraday_version(): From 0dfe2ab6b1c8ab776bd5838ce80eca3c5c93581d Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 29 Jun 2018 17:42:07 -0300 Subject: [PATCH 1432/1506] Support for MAC OS in initdb --- server/commands/initdb.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/commands/initdb.py b/server/commands/initdb.py index ef8fb96a85f..a5dbcf74cb1 100644 --- a/server/commands/initdb.py +++ b/server/commands/initdb.py @@ -163,11 +163,12 @@ def _configure_new_postgres_user(self, psql_log_file): """ print('This script will {blue} create a new postgres user {white} and {blue} save faraday-server settings {white}(server.ini). '.format(blue=Fore.BLUE, white=Fore.WHITE)) username = 'faraday_postgresql' - postgres_command = ['sudo', '-u', 'postgres'] + postgres_command = ['sudo', '-u', 'postgres', 'psql'] if sys.platform == 'darwin': - postgres_command = [] + print('{blue}MAC OS detected{white}'.format(blue=Fore.BLUE, white=Fore.WHITE)) + postgres_command = ['psql', 'postgres'] password = self.generate_random_pw(25) - command = postgres_command + ['psql', '-c', 'CREATE ROLE {0} WITH LOGIN PASSWORD \'{1}\';'.format(username, password)] + command = postgres_command + [ '-c', 'CREATE ROLE {0} WITH LOGIN PASSWORD \'{1}\';'.format(username, password)] p = Popen(command, stderr=psql_log_file, stdout=psql_log_file) p.wait() psql_log_file.seek(0) From 5b834c0000484feb1805ca590183094bf3b436db Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 29 Jun 2018 19:55:18 -0300 Subject: [PATCH 1433/1506] Fix various login issues in faraday client --- faraday.py | 184 +++++++++++++++++++++++++---------------------------- 1 file changed, 88 insertions(+), 96 deletions(-) diff --git a/faraday.py b/faraday.py index afd8ec49a2b..5fab2278d3e 100755 --- a/faraday.py +++ b/faraday.py @@ -1,15 +1,16 @@ #!/usr/bin/env python2.7 ''' Faraday Penetration Test IDE -Copyright (C) 2014 Infobyte LLC (http://www.infobytesec.com/) +Copyright (C) 2018 Infobyte LLC (http://www.infobytesec.com/) See the file 'doc/LICENSE' for the license information - ''' import os import sys import shutil +import getpass import argparse +import requests.exceptions from config.configuration import getInstanceConfiguration from config.globals import ( @@ -70,8 +71,8 @@ def getParserArgs(): - """Parser setup for faraday launcher arguments. - + """ + Parser setup for faraday launcher arguments. """ parser = argparse.ArgumentParser( @@ -176,15 +177,13 @@ def getParserArgs(): parser.add_argument('-v', '--version', action='version', version='Faraday v{version}'.format(version=f_version)) - # args = parser.parse_args(['@parser_args.cfg']) return parser.parse_args() def check_dependencies_or_exit(): - """Dependency resolver based on a previously specified CONST_REQUIREMENTS_FILE. - + """ + Dependency resolver based on a previously specified CONST_REQUIREMENTS_FILE. Currently checks a list of dependencies from a file and exits if they are not met. - """ installed_deps, missing_deps, conflict_deps = dependencies.check_dependencies(requirements_file=FARADAY_REQUIREMENTS_FILE) @@ -210,11 +209,10 @@ def check_dependencies_or_exit(): logger.info("Dependencies met") def setConf(): - """User configuration management and instantiation. - + """ + User configuration management and instantiation. Setting framework configuration based either on previously user saved settings or default ones. - """ logger.info("Setting configuration.") @@ -260,7 +258,6 @@ def startFaraday(): start = main_app.start from colorama import Fore, Back, Style - import string serverURL = getInstanceConfiguration().getServerURI() if serverURL: url = "%s/_ui" % serverURL @@ -276,7 +273,8 @@ def startFaraday(): def setupPlugins(dev_mode=False): - """Checks and handles Faraday's plugin status. + """ + Checks and handles Faraday's plugin status. When dev_mode is True, the user enters in development mode and the plugins will be replaced with the latest ones. @@ -286,7 +284,6 @@ def setupPlugins(dev_mode=False): TODO: When dependencies are not satisfied ask user if he wants to try and run faraday with a inestability warning. - """ if dev_mode: @@ -303,11 +300,11 @@ def setupPlugins(dev_mode=False): def setupZSH(): - """Cheks and handles Faraday's integration with ZSH. + """ + Checks and handles Faraday's integration with ZSH. If the user has a .zshrc file, it gets copied and integrated with faraday's zsh plugin. - """ if os.path.isfile(USER_ZSHRC): @@ -325,7 +322,8 @@ def setupZSH(): def setupXMLConfig(): - """Checks user configuration file status. + """ + Checks user configuration file status. If there is no custom config the default one will be copied as a default. """ @@ -338,7 +336,8 @@ def setupXMLConfig(): def setupImages(): - """ Copy png icons + """ + Copy png icons """ if os.path.exists(FARADAY_USER_IMAGES): shutil.rmtree(FARADAY_USER_IMAGES) @@ -346,7 +345,8 @@ def setupImages(): def checkConfiguration(gui_type): - """Checks if the environment is ready to run Faraday. + """ + Checks if the environment is ready to run Faraday. Checks different environment requirements and sets them before starting Faraday. This includes checking for plugin folders, libraries, @@ -364,8 +364,8 @@ def checkConfiguration(gui_type): def setupFolders(folderlist): - """Checks if a list of folders exists and creates them otherwise. - + """ + Checks if a list of folders exists and creates them otherwise. """ for folder in folderlist: @@ -374,8 +374,8 @@ def setupFolders(folderlist): def checkFolder(folder): - """Checks whether a folder exists and creates it if it doesn't. - + """ + Checks whether a folder exists and creates it if it doesn't. """ if not os.path.isdir(folder): @@ -385,8 +385,8 @@ def checkFolder(folder): def printBanner(): - """Prints Faraday's ascii banner. - + """ + Prints Faraday's ascii banner. """ from colorama import Fore, Back, Style print (Fore.RED + """ @@ -429,72 +429,61 @@ def checkUpdates(): logger.info("No updates available, enjoy Faraday.") -def checkServerUrl(): - import requests - CONF = getInstanceConfiguration() - server_url = CONF.getServerURI() - - if server_url is None or CONF.getAPIUsername() is None or CONF.getAPIUsername() is None: - doLoginLoop() - server_url = CONF.getServerURI() - - try: - requests.get(server_url, timeout=5) - except requests.exceptions.SSLError: - print(""" - SSL certificate validation failed. - You can use the --cert option in Faraday - to set the path of the cert - """) - sys.exit(-1) - except requests.exceptions.MissingSchema as ex: - print("Check ~/.faraday/config/user.xml server url, the following error was found: {0} ".format(ex)) - except requests.exceptions.ConnectionError: - print("\n\n" - "Can't connect to server. \nCheck if server is running or the file ~/.faraday/config/user.xml server url." - "\n\n" - ) - sys.exit(-1) - - def check_faraday_version(): try: server.check_faraday_version() except RuntimeError: - getLogger("launcher").error("The server is running a different Faraday version than the client " - "you are running. Version numbers must match!") - + getLogger("launcher").error( + "The server is running a different Faraday version than the client you are running. Version numbers must match!") sys.exit(2) -def doLoginLoop(): - """ Sets the username and passwords from the command line. - If --login flag is set then username and password is set """ +def try_login_user(server_uri, api_username, api_password): + + try: + session_cookie = login_user(server_uri, api_username, api_password) + return session_cookie + except requests.exceptions.SSLError: + print("SSL certificate validation failed.\nYou can use the --cert option in Faraday to set the path of the cert") + sys.exit(-1) + except requests.exceptions.MissingSchema: + print("The Faraday server URL is incorrect, please try again.") + sys.exit(-2) - import getpass - print("""\nTo login please provide your valid DB Credentials.\n -You have 3 attempts.""") +def doLoginLoop(): + """ + Sets the username and passwords from the command line. + If --login flag is set then username and password is set + """ try: CONF = getInstanceConfiguration() - server_url = CONF.getAPIUrl() - if server_url is None: - server_url = raw_input( - "Please enter the faraday server url (press enter for http://localhost:5985): ") or "http://localhost:5985" - CONF.setAPIUrl(server_url) + old_server_url = CONF.getAPIUrl() + + if old_server_url is None: + new_server_url = raw_input( + "\nPlease enter the Faraday server URL (Press enter for http://localhost:5985): ") or "http://localhost:5985" + else: + new_server_url = raw_input( + "\nPlease enter the Faraday server URL (Press enter for last used: {}): ".format(old_server_url)) or old_server_url + + CONF.setAPIUrl(new_server_url) + + print("""\nTo login please provide your valid Faraday credentials.\nYou have 3 attempts.""") for attempt in range(1, 4): - username = raw_input("Username (press enter for faraday): ") or "faraday" - password = getpass.getpass('Password: ') + api_username = raw_input("Username (press enter for faraday): ") or "faraday" + api_password = getpass.getpass('Password: ') + + session_cookie = try_login_user(new_server_url, api_username, api_password) - session_cookie = login_user(server_url, username, password) if session_cookie: - CONF.setAPIUsername(username) - CONF.setAPIPassword(password) + CONF.setAPIUsername(api_username) + CONF.setAPIPassword(api_password) CONF.setDBSessionCookies(session_cookie) CONF.saveConfig() @@ -503,18 +492,42 @@ def doLoginLoop(): print("You can't login as a client. You have %s attempt(s) left." % (3 - attempt)) continue - logger.info('Login successful') - + logger.info('Login successful: {0}'.format(api_username)) break + print('Login failed, please try again. You have %d more attempts' % (3 - attempt)) else: logger.fatal('Invalid credentials, 3 attempts failed. Quitting Faraday...') sys.exit(-1) + except KeyboardInterrupt: sys.exit(0) +def login(forced_login): + + CONF = getInstanceConfiguration() + server_uri = CONF.getServerURI() + api_username = CONF.getAPIUsername() + api_password = CONF.getAPIPassword() + + if forced_login: + doLoginLoop() + return + + if server_uri and api_username and api_password: + + session_cookie = try_login_user(server_uri, api_username, api_password) + + if session_cookie: + CONF.setDBSessionCookies(session_cookie) + logger.info('Login successful: {0}'.format(api_username)) + return + + doLoginLoop() + + def main(): """ Main function for launcher. @@ -524,38 +537,17 @@ def main(): global logger, args logger = getLogger("launcher") - args = getParserArgs() setupFolders(CONST_FARADAY_FOLDER_LIST) setUpLogger(args.debug) - if not args.nodeps: check_dependencies_or_exit() - printBanner() if args.cert_path: os.environ[REQUESTS_CA_BUNDLE_VAR] = args.cert_path checkConfiguration(args.gui) setConf() - checkServerUrl() - CONF = getInstanceConfiguration() - if args.login: - if not CONF.getServerURI(): - couchURI = raw_input("Enter the Faraday server [http://127.0.0.1:5985]: ") or "http://127.0.0.1:5985" - - if couchURI: - CONF.setAPIUrl(couchURI) - checkServerUrl() - else: - logger.fatal('Please configure Faraday server to authenticate (--login)') - sys.exit(-1) - - doLoginLoop() - else: - session_cookie = login_user(CONF.getServerURI(), CONF.getAPIUsername(), CONF.getAPIPassword()) - if session_cookie: - CONF.setDBSessionCookies(session_cookie) - + login(args.login) check_faraday_version() checkUpdates() startFaraday() From 5869f93773cc17d5516443b7327819e956293443 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 29 Jun 2018 20:03:33 -0300 Subject: [PATCH 1434/1506] Remove bad print in faraday client, related to client role --- faraday.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/faraday.py b/faraday.py index 5fab2278d3e..ed80e1165ed 100755 --- a/faraday.py +++ b/faraday.py @@ -489,7 +489,7 @@ def doLoginLoop(): user_info = get_user_info() if (user_info is None) or (not user_info) or ('username' not in user_info): - print("You can't login as a client. You have %s attempt(s) left." % (3 - attempt)) + print('Login failed, please try again. You have %d more attempts' % (3 - attempt)) continue logger.info('Login successful: {0}'.format(api_username)) From 6de6d5af59d0229fb23d2134cf56783cb6007e66 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 29 Jun 2018 20:27:49 -0300 Subject: [PATCH 1435/1506] Remove duplicated item in requirements_extras --- requirements_extras.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements_extras.txt b/requirements_extras.txt index 14967431c7f..1fa335710dc 100644 --- a/requirements_extras.txt +++ b/requirements_extras.txt @@ -1,4 +1,3 @@ beautifulsoup4>=4.6.0 -psycopg2>=2.7.3 w3af_api_client>=1.1.7 selenium>=3.9.0 From 9d25e866b664c754ed7e71e57056f07dd0c0f7e5 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 2 Jul 2018 12:27:49 -0300 Subject: [PATCH 1436/1506] [FIX] design changes. All remove rounded borders from all button. add some padding in vuln by severity --- server/www/estilos-v3.css | 4 ++++ server/www/estilos.css | 22 +++++++++---------- .../partials/modal-hosts-by-service.html | 2 +- .../dashboard/partials/vulns-by-severity.html | 2 +- .../statusReport/partials/modalNew.html | 4 ++-- .../scripts/vulndb/partials/modalEdit.html | 4 ++-- .../www/scripts/vulndb/partials/modalNew.html | 4 ++-- .../workspaces/partials/modalEdit.html | 4 ++-- .../scripts/workspaces/partials/modalNew.html | 4 ++-- 9 files changed, 27 insertions(+), 23 deletions(-) diff --git a/server/www/estilos-v3.css b/server/www/estilos-v3.css index c7eda195dd5..79d88a2c6e7 100644 --- a/server/www/estilos-v3.css +++ b/server/www/estilos-v3.css @@ -217,3 +217,7 @@ .jumbotron { background-color: #f4f3f4; } + +.btn { + border-radius: 0px; +} diff --git a/server/www/estilos.css b/server/www/estilos.css index 789185937ea..726f6328a80 100644 --- a/server/www/estilos.css +++ b/server/www/estilos.css @@ -196,16 +196,16 @@ article.untercio{ .untercio article{ height: 20%; } - .seccion article { - min-height: 85px; - font-size: 11px; - text-align: left; - } - .seccion article header { - background: #FFFFFF; - min-height: 40px; - text-align: left; - } +.seccion article { + min-height: 85px; + font-size: 11px; + text-align: left; +} +.seccion article header { + background: #FFFFFF; + min-height: 40px; + text-align: left; +} .seccion article header h2 { color: #101010; font-size: 14px; @@ -1308,4 +1308,4 @@ a.button-disable{cursor: not-allowed;pointer-events: none;opacity: 0.5} } .label-success-impact { background-color: #5cb85c - } \ No newline at end of file + } diff --git a/server/www/scripts/dashboard/partials/modal-hosts-by-service.html b/server/www/scripts/dashboard/partials/modal-hosts-by-service.html index 7474300ebbd..6792129482c 100644 --- a/server/www/scripts/dashboard/partials/modal-hosts-by-service.html +++ b/server/www/scripts/dashboard/partials/modal-hosts-by-service.html @@ -41,6 +41,6 @@

    Services for {{name}} ({{hosts.length}} total)

    -
    +
    diff --git a/server/www/scripts/dashboard/partials/vulns-by-severity.html b/server/www/scripts/dashboard/partials/vulns-by-severity.html index 4c164501e0b..bf17afd84c7 100644 --- a/server/www/scripts/dashboard/partials/vulns-by-severity.html +++ b/server/www/scripts/dashboard/partials/vulns-by-severity.html @@ -15,7 +15,7 @@

    -
    +
    {{count}}
    {{severity}}
    diff --git a/server/www/scripts/statusReport/partials/modalNew.html b/server/www/scripts/statusReport/partials/modalNew.html index 13e880c77a5..50ae9a068fe 100644 --- a/server/www/scripts/statusReport/partials/modalNew.html +++ b/server/www/scripts/statusReport/partials/modalNew.html @@ -4,7 +4,7 @@
    -
    +
    @@ -248,7 +248,7 @@

    Evidence

    -
    +
    diff --git a/server/www/scripts/vulndb/partials/modalEdit.html b/server/www/scripts/vulndb/partials/modalEdit.html index ea3e0214e2d..7f7425af90c 100644 --- a/server/www/scripts/vulndb/partials/modalEdit.html +++ b/server/www/scripts/vulndb/partials/modalEdit.html @@ -4,7 +4,7 @@
    -
    +
    @@ -56,7 +56,7 @@

    Edit {{model.name}} Vulnerability Model

    -
    +
    diff --git a/server/www/scripts/vulndb/partials/modalNew.html b/server/www/scripts/vulndb/partials/modalNew.html index 1bd3c546d2f..eac36d086c7 100644 --- a/server/www/scripts/vulndb/partials/modalNew.html +++ b/server/www/scripts/vulndb/partials/modalNew.html @@ -4,7 +4,7 @@
    -
    +
    @@ -62,7 +62,7 @@

    New Vulnerability Model

    -
    +
    diff --git a/server/www/scripts/workspaces/partials/modalEdit.html b/server/www/scripts/workspaces/partials/modalEdit.html index 3022dc535a1..53c282aebeb 100644 --- a/server/www/scripts/workspaces/partials/modalEdit.html +++ b/server/www/scripts/workspaces/partials/modalEdit.html @@ -4,7 +4,7 @@
    -
    +
    @@ -14,7 +14,7 @@

    Edit Workspace: {{workspace.name}}

    - +
    -
    +
    @@ -14,7 +14,7 @@

    New Workspace

    - + Date: Mon, 2 Jul 2018 12:36:20 -0300 Subject: [PATCH 1437/1506] [ADD] new upload icon --- server/www/images/icon-toolbar-upload.svg | 30 +++++++++++++++++++ .../statusReport/partials/statusReport.html | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 server/www/images/icon-toolbar-upload.svg diff --git a/server/www/images/icon-toolbar-upload.svg b/server/www/images/icon-toolbar-upload.svg new file mode 100644 index 00000000000..12b38a8f3ff --- /dev/null +++ b/server/www/images/icon-toolbar-upload.svg @@ -0,0 +1,30 @@ + + + + + + + + + + diff --git a/server/www/scripts/statusReport/partials/statusReport.html b/server/www/scripts/statusReport/partials/statusReport.html index 41b62657e77..f0e1c521ccc 100644 --- a/server/www/scripts/statusReport/partials/statusReport.html +++ b/server/www/scripts/statusReport/partials/statusReport.html @@ -85,7 +85,7 @@
    From e373342a3f15085b84e444a54ee2297c79b4fa7a Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Mon, 2 Jul 2018 14:16:07 -0300 Subject: [PATCH 1438/1506] [ADD] missing bootstrap class on buttons --- server/www/scripts/hosts/partials/list.html | 4 ++-- server/www/scripts/licenses/partials/list.html | 4 ++-- .../scripts/statusReport/partials/statusReport.html | 12 ++++++------ server/www/scripts/vulndb/partials/vulndb.html | 4 ++-- server/www/scripts/workspaces/partials/list.html | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/server/www/scripts/hosts/partials/list.html b/server/www/scripts/hosts/partials/list.html index 1908de7d21e..f410a6217eb 100644 --- a/server/www/scripts/hosts/partials/list.html +++ b/server/www/scripts/hosts/partials/list.html @@ -14,12 +14,12 @@
    -
    -
    diff --git a/server/www/scripts/licenses/partials/list.html b/server/www/scripts/licenses/partials/list.html index 968e59ad43a..bce96d05f9d 100644 --- a/server/www/scripts/licenses/partials/list.html +++ b/server/www/scripts/licenses/partials/list.html @@ -15,12 +15,12 @@
    -
    -
    diff --git a/server/www/scripts/statusReport/partials/statusReport.html b/server/www/scripts/statusReport/partials/statusReport.html index f0e1c521ccc..299ac82df5e 100644 --- a/server/www/scripts/statusReport/partials/statusReport.html +++ b/server/www/scripts/statusReport/partials/statusReport.html @@ -14,7 +14,7 @@
    -
    -
    @@ -74,17 +74,17 @@
    -
    -
    -
    diff --git a/server/www/scripts/vulndb/partials/vulndb.html b/server/www/scripts/vulndb/partials/vulndb.html index ecaad043614..623b01c3bd0 100644 --- a/server/www/scripts/vulndb/partials/vulndb.html +++ b/server/www/scripts/vulndb/partials/vulndb.html @@ -15,12 +15,12 @@
    -
    -
    diff --git a/server/www/scripts/workspaces/partials/list.html b/server/www/scripts/workspaces/partials/list.html index e92e002e2bc..17bb1e3dc9f 100644 --- a/server/www/scripts/workspaces/partials/list.html +++ b/server/www/scripts/workspaces/partials/list.html @@ -18,12 +18,12 @@
    -
    -
    From fd1281b5bc59142963f99482c638d065828b868a Mon Sep 17 00:00:00 2001 From: montive Date: Mon, 2 Jul 2018 18:11:35 -0300 Subject: [PATCH 1439/1506] Nexpose update finished --- plugins/repo/nexpose/plugin.py | 251 +++++++++++++++++++++++---------- 1 file changed, 176 insertions(+), 75 deletions(-) diff --git a/plugins/repo/nexpose/plugin.py b/plugins/repo/nexpose/plugin.py index b98daa95b21..be59f3aa8a0 100644 --- a/plugins/repo/nexpose/plugin.py +++ b/plugins/repo/nexpose/plugin.py @@ -51,7 +51,9 @@ class NexposeXmlParser(object): def __init__(self, xml_output): tree = self.parse_xml(xml_output) if tree: - self.items = [data for data in self.get_items(tree)] + vulns = self.get_vulns_list(tree) + self.items = [data for data in self.get_items(tree,vulns)] + else: self.items = [] @@ -72,12 +74,22 @@ def parse_xml(self, xml_output): return tree - def get_items(self, tree): + def get_items(self, tree, vulnerabilities): """ @return items A list of Host instances """ - for node in tree.findall("devices/device"): - yield Item(node) + for node in tree.findall("nodes/node"): + yield Item(node,vulnerabilities) + + def get_vulns_list(self, tree): + """ + :param tree: + """ + vulns_list = [] + for self.issues in tree.findall("VulnerabilityDefinitions/vulnerability"): + vulns_list.append(self.issues) + + return vulns_list class Item(object): @@ -88,84 +100,185 @@ class Item(object): @param item_node A item_node taken from an nexpose xml tree """ - def __init__(self, item_node): + def __init__(self, item_node, vulnerability): self.node = item_node self.ip = item_node.get('address') - self.os = self.get_text_from_subnode("fingerprint/description") + self.os = self.get_version('fingerprints/os') + # Checking node's vulns + node_tests_list = self.get_tests('tests') + self.vulns_list = vulnerability + self.node_vulns = self.check_vulns(node_tests_list, self.vulns_list) + # Checking service's vulns + self.service = self.get_service('endpoints/endpoint', item_node) - self.vulns = self.getResults(item_node) - self.srv = [] + def get_service(self,path,item_node): + """ + Gets a service. - for srv in item_node.findall("services/service"): - item = {} + @return service + """ + self.srv = [] + for srv in item_node.findall(path): self.node = srv - item['name'] = srv.get('name') + item = {} + tests_list = self.get_tests('services/service/tests') + item['name'] = self.get_name_from_service('services/service') item['port'] = srv.get('port') item['protocol'] = srv.get('protocol') - item['version'] = self.get_text_from_subnode( - "fingerprint/description") - item['vulns'] = self.getResults(srv) + item['status'] = srv.get('status') + item['version'] = self.get_version('services/service/fingerprints/fingerprint') + item['vulns'] = self.check_vulns(tests_list, self.vulns_list) self.srv.append(item) - def getResults(self, tree): - """ - :param tree: - """ - for self.issues in tree.findall("vulnerabilities/vulnerability"): - yield Results(self.issues) + return self.srv + - def get_text_from_subnode(self, subnode_xpath_expr): + def get_name_from_service(self, service_path): """ - Finds a subnode in the host node and the retrieves a value from it. + Gets the name of a service. - @return An attribute value + @return service's attribute 'name' """ - sub_node = self.node.find(subnode_xpath_expr) + sub_node = self.node.find(service_path) if sub_node is not None: - return sub_node.text + return sub_node.get('name') + + return None + + def get_version(self, path): + """ + Gets version of a host or a service. + @return attribute 'product' from a host or a service + """ + sub_node = self.node.find(path) + if sub_node is not None: + return sub_node.attrib['product'] + return None + def get_tests(self, test_path): + """ + Gets every test in a service. + + @return a list of every test + """ + sub_node = self.node.find(test_path) + tests_list = [] + for test in sub_node: + tests_list.append(test) + return tests_list + + #Check which test of a node or service is a vuln + def check_vulns(self, list_of_tests, list_of_vulns): + """ + Checks if a test of a host or a service is in the vulns list -class Results(): + @return a list of vulns of a host or a service + """ + checked_vulns = [] + vuln_attributes = {} + for test in list_of_tests: + for vuln in list_of_vulns: + if test.attrib['id'] == vuln.attrib['id']: + vuln_attributes['id'] = vuln.attrib['id'] + vuln_attributes['ref'] = self.get_vulns_ref(vuln) + vuln_attributes['severity'] = self.severity_format(vuln.attrib['severity']) + vuln_attributes['description'] = self.convert_to_flat_text(vuln,'description') + vuln_attributes['resolution'] = self.convert_to_flat_text(vuln,'solution') + checked_vulns.append(vuln_attributes) + + return checked_vulns + + def severity_format(self,severity): + """ + Convert Nexpose severity format into Faraday API severity format - def __init__(self, issue_node): - self.node = issue_node - self.name = issue_node.get('id') - self.ref = [] - data = self.get_text_from_subnode("id/[@type='cve']") + @return a severity + """ + if severity == '1' or severity == '2': + return 'low' + elif severity == '3' or severity == '4': + return 'medium' + elif severity >= '5' or severity <= '7': + return 'high' + elif severity >= '8': + return 'critical' + + def get_vulns_ref(self, vuln): + """ + Gets the references of a vuln + + @return a list of sources of every reference + """ + # PCISeverity: Policy violations can't be added to plugins + source = [] + source.append("cvssScore-" + vuln.attrib['cvssScore']) + source.append("cvssVector-" + vuln.attrib['cvssVector']) + + data = self.get_text_from_reference(vuln, "references/reference/[@source='CVE']") if data: - self.ref.append(data) + source.append(data) - data = self.get_text_from_subnode("id/[@type='bid']") + data = self.get_text_from_reference(vuln, "references/reference/[@source='BID']") if data: - self.ref.append("bid-" + data) + source.append("bid-" + data) - data = self.get_text_from_subnode("id/[@type='osvdb']") + data = self.get_text_from_reference(vuln, "references/reference/[@source='OVAL']") if data: - self.ref.append("osvdb-" + data) + source.append("osvdb-" + data) - for v in issue_node.findall("id/[@type='secunia']"): - self.ref.append("secunia-" + v.text) + data = self.get_text_from_reference(vuln, "references/reference/[@source='SUSE']") + if data: + source.append("suse-" + data) - for v in issue_node.findall("id/[@type='url']"): - self.ref.append("url-" + v.text) + for sources in vuln.findall("references/reference/[@source='XF']"): + source.append("xf-" + sources.text) - self.url = self.get_text_from_subnode("key") + for sources in vuln.findall("references/reference/[@source='REDHAT']"): + source.append("secunia-" + sources.text) - def get_text_from_subnode(self, subnode_xpath_expr): + for sources in vuln.findall("references/reference/[@source='URL']"): + source.append("url-" + sources.text) + + return source + + def get_text_from_reference(self,vulnerability, reference_path): """ - Finds a subnode in the host node and the retrieves a value from it. + Gets text from the references of a vulnerability. - @return An attribute value + @return A attribute value """ - sub_node = self.node.find(subnode_xpath_expr) + sub_node = vulnerability.find(reference_path) if sub_node is not None: return sub_node.text return None + def convert_to_flat_text(self,vuln,tag): + """ + Converts texts from multiples elements into one flat text + + @return returns new text + """ + self.description = vuln.find(tag) + xml_str = self.description.itertext() + aux_string = [] + strings_list = [data for data in xml_str] + # Iterating each item of the list that contains the strings + for item in strings_list: + # Taking away '\n', '\t' + aux_item = item.rstrip() + if aux_item is not '': + string_text = aux_item.replace('\n','') + aux_string.append(string_text) + + flatten_text = ' '.join(aux_string) + flat_text = ' '.join(flatten_text.split()) + + return flat_text + class NexposePlugin(core.PluginBase): """ @@ -188,37 +301,23 @@ def __init__(self): "nexpose_output-%s.xml" % self._rid) def parseOutputString(self, output, debug=False): - parser = NexposeXmlParser(output) for item in parser.items: - h_id = self.createAndAddHost(item.ip, item.os) - i_id = self.createAndAddInterface( - h_id, item.ip, ipv4_address=item.ip, hostname_resolution=item.ip) + host_id = self.createAndAddHost(item.ip, item.os) - for v in item.vulns: - v_id = self.createAndAddVulnToHost(h_id, v.name, ref=v.ref) + for vuln in item.node_vulns: + vuln_id = self.createAndAddVulnToHost(host_id, vuln['id'], ref=vuln['ref'], severity=vuln['severity'], desc=vuln['description'], resolution=vuln['resolution']) - for s in item.srv: - web = False - s_id = self.createAndAddServiceToInterface(h_id, i_id, s['name'], - s['protocol'], + for srv in item.service: + service_id = self.createAndAddServiceToHost(host_id, srv['name'], + srv['protocol'], ports=[ - str(s['port'])], - status="open", - version=s['version']) - for v in s['vulns']: - if v.url: - v_id = self.createAndAddVulnWebToService( - h_id, s_id, v.name, ref=v.ref, website=item.ip, path=v.url) - if not web: - n_id = self.createAndAddNoteToService( - h_id, s_id, "website", "") - n2_id = self.createAndAddNoteToNote( - h_id, s_id, n_id, item.ip, "") - web = True - else: - v_id = self.createAndAddVulnToService( - h_id, s_id, v.name, ref=v.ref) + str(srv['port'])], + status=srv['status'], + version=srv['version']) + for vuln in srv['vulns']: + vuln_id = self.createAndAddVulnToService( + host_id, service_id, vuln['id'], ref=vuln['ref'], severity=vuln['severity'], desc=vuln['description'], resolution=vuln['resolution']) del parser def processCommandString(self, username, current_path, command_string): @@ -232,7 +331,9 @@ def createPlugin(): return NexposePlugin() if __name__ == '__main__': - parser = NexposeXmlParser(sys.argv[1]) - for item in parser.items: - if item.status == 'up': - print item + parser = NexposePlugin() + with open('/home/javier/report-xml1.0.xml', 'r') as report: + parser.parseOutputString(report.read()) + for item in parser.items: + if item.status == 'up': + print item From 8e1e262711146f98e1e1e920c303c67e64d593ec Mon Sep 17 00:00:00 2001 From: winnaz Date: Mon, 2 Jul 2018 18:20:18 -0300 Subject: [PATCH 1440/1506] fixed bug hardcored address and port on status_check.py --- server/commands/status_check.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/server/commands/status_check.py b/server/commands/status_check.py index e537ab4f570..02c24d68293 100644 --- a/server/commands/status_check.py +++ b/server/commands/status_check.py @@ -1,3 +1,9 @@ +''' +Faraday Penetration Test IDE +Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) +See the file 'doc/LICENSE' for the license information + +''' import requests import sqlalchemy import socket @@ -97,10 +103,13 @@ def check_credentials(): api_username = CONF.getAPIUsername() api_password = CONF.getAPIPassword() + address = server.config.faraday_server.bind_address + port = int(server.config.faraday_server.port) + values = {'email': api_username , 'password': api_password} try: - r = requests.post('http://localhost:5985/_api/login', json=values) + r = requests.post('http://{ADDRESS}:{PORT}/_api/login'.format(ADDRESS=address,PORT=port), json=values) if r.status_code == 200 and 'user' in r.json()['response']: return 200 From 49c151c5767a505b415119326b05257a97aad20d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 3 Jul 2018 14:26:05 -0300 Subject: [PATCH 1441/1506] Fix vulns by status to respect confirmed value It was always showing confirmed+unconfirmed vulns even if the confirmed filter was on. --- .../www/scripts/commons/providers/server.js | 5 +- .../controllers/vulnsByStatusCtrl.js | 49 ++++++++++++------- .../scripts/dashboard/providers/dashboard.js | 4 +- 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/server/www/scripts/commons/providers/server.js b/server/www/scripts/commons/providers/server.js index b0f5df62c7a..1ceb2c8f552 100644 --- a/server/www/scripts/commons/providers/server.js +++ b/server/www/scripts/commons/providers/server.js @@ -343,9 +343,12 @@ angular.module("faradayApp") return get(url, payload) } - ServerAPI.getVulnsGroupedBy = function(wsName, groupBy) { + ServerAPI.getVulnsGroupedBy = function(wsName, groupBy, confirmed) { var url = createGetUrl(wsName, 'vulns') + 'count/'; var payload = {'group_by': groupBy} + if (confirmed) { + payload.confirmed = confirmed; + } return get(url, payload) } diff --git a/server/www/scripts/dashboard/controllers/vulnsByStatusCtrl.js b/server/www/scripts/dashboard/controllers/vulnsByStatusCtrl.js index a0fb8b6f297..6283c498a06 100644 --- a/server/www/scripts/dashboard/controllers/vulnsByStatusCtrl.js +++ b/server/www/scripts/dashboard/controllers/vulnsByStatusCtrl.js @@ -12,30 +12,41 @@ angular.module('faradayApp') init = function() { if($routeParams.wsId != undefined) { $scope.workspace = $routeParams.wsId; + $scope.loadData(); + + $scope.$watch(function() { + return dashboardSrv.props.confirmed; + }, function(newValue, oldValue) { + if (oldValue != newValue) + $scope.loadData(); + }, true); + } + }; - dashboardSrv.getVulnerabilitiesGroupedBy($scope.workspace, 'status') - .then(function(vulnsByStatus) { - $scope.data = {key: [], value: [], colors: [], options: {maintainAspectRatio: false, animateRotate: true}}; - $scope.loaded = true; + $scope.loadData = function(){ + dashboardSrv.getVulnerabilitiesGroupedBy($scope.workspace, 'status', dashboardSrv.props.confirmed) + .then(function(vulnsByStatus) { + $scope.data = {key: [], value: [], colors: [], options: {maintainAspectRatio: false, animateRotate: true}}; + $scope.loaded = true; - vulnerabilityColors = { - 'open': '#e77273', - 'closed': '#bddd72', - 're-opened': '#e7d174', - 'risk-accepted': '#7aabd9' - }; + vulnerabilityColors = { + 'open': '#e77273', + 'closed': '#bddd72', + 're-opened': '#e7d174', + 'risk-accepted': '#7aabd9' + }; - vulnsByStatus.forEach(function(vuln, index) { - $scope.data.value.push(vuln.count); - $scope.data.key.push(vuln.status); + vulnsByStatus.forEach(function(vuln, index) { + $scope.data.value.push(vuln.count); + $scope.data.key.push(vuln.status); - $scope.data.colors.push(vulnerabilityColors[vuln.status]); - }); + $scope.data.colors.push(vulnerabilityColors[vuln.status]); + }); - $scope.loaded = true; - }); - } - }; + $scope.loaded = true; + }); + } + dashboardSrv.registerCallback($scope.loadData); init(); }]); diff --git a/server/www/scripts/dashboard/providers/dashboard.js b/server/www/scripts/dashboard/providers/dashboard.js index d8e9c86d8a4..504a3f31243 100644 --- a/server/www/scripts/dashboard/providers/dashboard.js +++ b/server/www/scripts/dashboard/providers/dashboard.js @@ -208,10 +208,10 @@ angular.module('faradayApp') return deferred.promise; }; - dashboardSrv.getVulnerabilitiesGroupedBy = function(ws, groupBy) { + dashboardSrv.getVulnerabilitiesGroupedBy = function(ws, groupBy, confirmed) { var deferred = $q.defer(); - ServerAPI.getVulnsGroupedBy(ws, groupBy) + ServerAPI.getVulnsGroupedBy(ws, groupBy, confirmed) .then(function(res) { deferred.resolve(res.data.groups); }, function() { From ea435ab194a94fe5d8ead1de5adfe9da1822f256 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 3 Jul 2018 15:28:32 -0300 Subject: [PATCH 1442/1506] Fix error message on save as template When clicked in the save as template button of status report, if the vuln template creation failed with a conflict (409 error) it was showing that it was created ok. I fixed some promises to avoid this --- server/www/scripts/vulndb/providers/vulnModel.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/www/scripts/vulndb/providers/vulnModel.js b/server/www/scripts/vulndb/providers/vulnModel.js index 1a35b7bd70f..966fc811196 100644 --- a/server/www/scripts/vulndb/providers/vulnModel.js +++ b/server/www/scripts/vulndb/providers/vulnModel.js @@ -99,8 +99,7 @@ angular.module('faradayApp'). } var message; if (res.status == 409) { - console.log("Vulnerability template already exists. " + res.data.message + " ID: " + res.data.object._id); - deferred.resolve(self); + message = "Vulnerability template already exists. " + res.data.message + " ID: " + res.data.object._id; } else { message = "Unable to save the Vuln Model. " + msg; } From 5f830e3b23929dab4dbddfcf9b825e761cb5a86d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Tue, 3 Jul 2018 16:10:38 -0300 Subject: [PATCH 1443/1506] Keep references when saving vulns as templates --- server/www/scripts/statusReport/controllers/statusReport.js | 1 + 1 file changed, 1 insertion(+) diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index a1b3cf90b45..b75d5d7c869 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -594,6 +594,7 @@ angular.module("faradayApp") vuln.exploitation = vuln.severity; vuln.description = vuln.desc; vuln.desc_summary = vuln.desc; + vuln.references = vuln.refs; promises.push(self.vulnModelsManager.create(vuln, true)); }); $q.all(promises).then(function(success) { From eb4e6ca7174b06d80a2b860767afd667c2451bee Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 3 Jul 2018 16:47:23 -0300 Subject: [PATCH 1444/1506] [MOD] Update icon list confirmed color --- server/www/images/icon-list-confirmed.svg | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/server/www/images/icon-list-confirmed.svg b/server/www/images/icon-list-confirmed.svg index 03f543428bc..8c0b222e380 100644 --- a/server/www/images/icon-list-confirmed.svg +++ b/server/www/images/icon-list-confirmed.svg @@ -1,11 +1,11 @@ - - - - - - - + + + + + + + From a3808e39cea212a2d6d9c685ef18906ea2e4f262 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Tue, 3 Jul 2018 18:14:57 -0300 Subject: [PATCH 1445/1506] [MOD] add click to reset_db and make it an executable for automation tests --- server/commands/reset_db.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) mode change 100644 => 100755 server/commands/reset_db.py diff --git a/server/commands/reset_db.py b/server/commands/reset_db.py old mode 100644 new mode 100755 index 9ad8278bc55..6b94cd9a631 --- a/server/commands/reset_db.py +++ b/server/commands/reset_db.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python2.7 ''' Faraday Penetration Test IDE Copyright (C) 2013 Infobyte LLC (http://www.infobytesec.com/) @@ -9,9 +10,11 @@ import os sys.path.append(os.getcwd()) +import click from server.models import db from server.web import app + def reset_db_all(): # It might be required to do a cascade delete to correctly the # vulnerability table @@ -36,16 +39,12 @@ def reset_db(): reset_db_all() -if __name__ == '__main__': - option = False - while True: - print "You are going to delete all info from the DB, this is not undoable, are you sure to follow? [Y/N]", - option = raw_input() +@click.command() +@click.option('--confirm/--no-confirme', prompt='Confirm databa reset?') +def main(confirm): + if confirm: + reset_db() - if option.upper() in ['Y', 'N', 'YES', 'NO']: - break - else: - print(str(option) + " option is invalid.") - if option.upper() in ['Y', 'YES']: - reset_db() +if __name__ == '__main__': + main() From ed6c90ab35889f50380645241d3613bca1997553 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Wed, 4 Jul 2018 16:53:15 -0300 Subject: [PATCH 1446/1506] Fix a potential cause for an association proxy bug I tried to fix this on commit 1e5f14da1a77070871b402d9d245726df8e66c4c but it didn't work there --- server/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index 3168cad4b42..95e1f0eb952 100644 --- a/server/models.py +++ b/server/models.py @@ -479,7 +479,10 @@ def _create(self, value): # we need to fetch already created objs. session.rollback() for conflict_obj in conflict_objs: - if hasattr(conflict_obj, 'name') and conflict_obj.name == value: + if not hasattr(conflict_obj, 'name'): + # The session can hold elements without a name (altough it shouldn't) + continue + if conflict_obj.name == value: continue persisted_conclict_obj = session.query(conflict_obj.__class__).filter_by(name=conflict_obj.name).first() if persisted_conclict_obj: From 195ed722f56b6179de2d4ece632dec258543aaad Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 4 Jul 2018 17:11:59 -0300 Subject: [PATCH 1447/1506] [FIX] when no policy or ref were provided the vuln api crashed. usedefault empty list when no refs or policyviolations were providad --- server/api/modules/vulns.py | 4 ++-- test_cases/test_api_vulnerability.py | 34 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/server/api/modules/vulns.py b/server/api/modules/vulns.py index f13a58f2ea1..c3bdff19dc9 100644 --- a/server/api/modules/vulns.py +++ b/server/api/modules/vulns.py @@ -425,8 +425,8 @@ def _perform_create(self, data, **kwargs): # popped object has the expected type. # This will be set after setting the workspace attachments = data.pop('_attachments', {}) - references = data.pop('references') - policyviolations = data.pop('policy_violations') + references = data.pop('references', []) + policyviolations = data.pop('policy_violations', []) obj = super(VulnerabilityView, self)._perform_create(data, **kwargs) obj.references = references diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 7eab88f7e52..f576dd0ee8d 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1471,6 +1471,40 @@ def test_hostnames_comma_separated(self, test_client, session): assert res.status_code == 200 assert res.json['count'] == 2 + def test_missing_policy_violation_case(self, test_client): + """ + bug found when a json was missing the policyviolations key + """ + host = HostFactory.create(workspace=self.workspace) + data = { + 'name': 'Test Alert policy_violations', + 'severity': 'informational', + 'creator': 'Zap', + 'parent_type': 'Host', + 'parent': 1, + 'type': 'Vulnerability', + 'refs': '' + } + res = test_client.post(self.url(), data=data) + assert res.status_code == 201 + + def test_missing_references_case(self, test_client): + """ + bug found when a json was missing the policyviolations key + """ + host = HostFactory.create(workspace=self.workspace) + data = { + 'name': 'Test Alert policy_violations', + 'severity': 'informational', + 'creator': 'Zap', + 'parent_type': 'Host', + 'parent': 1, + 'type': 'Vulnerability', + } + res = test_client.post(self.url(), data=data) + assert res.status_code == 201 + + def test_type_filter(workspace, session, vulnerability_factory, vulnerability_web_factory): From 7548ad1f9eff2e75059d8fe7b230205d9c3009de Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 4 Jul 2018 18:24:18 -0300 Subject: [PATCH 1448/1506] [FIX] fix unit test for postgresql --- test_cases/test_api_vulnerability.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index f576dd0ee8d..456726c4df8 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1471,34 +1471,36 @@ def test_hostnames_comma_separated(self, test_client, session): assert res.status_code == 200 assert res.json['count'] == 2 - def test_missing_policy_violation_case(self, test_client): + def test_missing_policy_violation_case(self, test_client, session): """ bug found when a json was missing the policyviolations key """ host = HostFactory.create(workspace=self.workspace) + session.commit() data = { 'name': 'Test Alert policy_violations', 'severity': 'informational', 'creator': 'Zap', 'parent_type': 'Host', - 'parent': 1, + 'parent': host.id, 'type': 'Vulnerability', 'refs': '' } res = test_client.post(self.url(), data=data) assert res.status_code == 201 - def test_missing_references_case(self, test_client): + def test_missing_references_case(self, test_client, session): """ bug found when a json was missing the policyviolations key """ host = HostFactory.create(workspace=self.workspace) + session.commit() data = { 'name': 'Test Alert policy_violations', 'severity': 'informational', 'creator': 'Zap', 'parent_type': 'Host', - 'parent': 1, + 'parent': host.id, 'type': 'Vulnerability', } res = test_client.post(self.url(), data=data) From d018a6be9dff41c04b3d18622ed2207ed7738b36 Mon Sep 17 00:00:00 2001 From: montive Date: Wed, 4 Jul 2018 18:39:16 -0300 Subject: [PATCH 1449/1506] Restore old check_postgresql function --- server/commands/status_check.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/server/commands/status_check.py b/server/commands/status_check.py index c516b261c9a..15a0693637a 100644 --- a/server/commands/status_check.py +++ b/server/commands/status_check.py @@ -18,12 +18,13 @@ init() -def check_postgres(): - tmp = os.popen("ps -Af").read() - if "postgresql" in tmp: - return True - else: - return False +def check_postgres(): + with app.app_context(): + try: + result = db.engine.execute("SELECT version()") + return result + except sqlalchemy.exc.OperationalError: + return None def check_active_user(): @@ -142,7 +143,7 @@ def full_status_check(): else: print('[{red}-{white}] Could not connect to PostgreSQL, please check if database is running'\ .format(red=Fore.RED, white=Fore.WHITE)) - return + if check_active_user(): print("[{green}+{white}] Active user exists".format(green=Fore.GREEN, white=Fore.WHITE)) From 240dcf47c5e5a1db4f8dd203c9889627ee20f47e Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Wed, 4 Jul 2018 19:16:34 -0300 Subject: [PATCH 1450/1506] [FIX] unit tests --- test_cases/test_api_vulnerability.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index f38e1b5e7e8..30352e58120 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1463,7 +1463,6 @@ def test_search_by_hostnames_service_case(self, session, test_client): session.commit() url = self.url(workspace=workspace) + '?hostnames={0}'.format(hostname.name) res = test_client.get(url) - import ipdb; ipdb.set_trace() assert res.status_code == 200 assert res.json['count'] == 1 @@ -1475,15 +1474,14 @@ def test_search_by_hostnames_host_case(self, session, test_client): hostname = HostnameFactory.create(workspace=workspace, name='test.com') host = HostFactory.create(workspace=workspace) host.hostnames.append(hostname) - vuln = VulnerabilityFactory.create(host=host, workspace=workspace) + vuln = VulnerabilityFactory.create(host=host, service=None, workspace=workspace) session.add(vuln) session.add(vuln2) + session.add(host) session.add(hostname) session.commit() - import ipdb; ipdb.set_trace() url = self.url(workspace=workspace) + '?hostnames={0}'.format(hostname.name) res = test_client.get(url) - assert res.status_code == 200 assert res.json['count'] == 1 assert res.json['vulnerabilities'][0]['value']['name'] == vuln.name From 12e2db99c11d3faba6dc48f74160cccb97f3a023 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Thu, 5 Jul 2018 12:46:47 -0300 Subject: [PATCH 1451/1506] [ADD] Host vuln count back and endpoint added (/ws/ws_name/hosts/countVulns?hosts=[id list]) --- server/api/modules/hosts.py | 34 +++++++++++++- server/models.py | 91 +++++++++++++++++++++++++++++++++++-- 2 files changed, 121 insertions(+), 4 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 2946072987c..49c5caef6f9 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -5,7 +5,7 @@ import flask from flask import Blueprint from flask_classful import route -from marshmallow import fields +from marshmallow import fields, Schema from filteralchemy import Filter, FilterSet, operators from server.utils.database import get_or_create @@ -85,6 +85,22 @@ class Meta(FilterSetMeta): service = ServiceFilter(fields.Str()) +class HostCountSchema(Schema): + host_id = fields.Integer(dump_only=True, allow_none=False, + attribute='id') + critical = fields.Integer(dump_only=True, allow_none=False, + attribute='vulnerability_critical_count') + high = fields.Integer(dump_only=True, allow_none=False, + attribute='vulnerability_high_count') + med = fields.Integer(dump_only=True, allow_none=False, + attribute='vulnerability_med_count') + info = fields.Integer(dump_only=True, allow_none=False, + attribute='vulnerability_info_count') + unclassified = fields.Integer(dump_only=True, allow_none=False, + attribute='vulnerability_unclassified_count') + total = fields.Integer(dump_only=True, allow_none=False, + attribute='vulnerability_total_count') + class HostsView(PaginatedMixin, FilterAlchemyMixin, ReadWriteWorkspacedView): @@ -103,6 +119,22 @@ def service_list(self, workspace_name, host_id): services = self._get_object(host_id, workspace_name).services return ServiceSchema(many=True).dump(services).data + @route('/countVulns/') + def count_vulns(self, workspace_name): + host_ids = flask.request.args.get('hosts', '') + host_id_list = host_ids.split(',') + + res_dict = {'res':[]} + + host_count_schema = HostCountSchema() + host_count = Host.query_with_count(False, host_id_list, workspace_name) + + for host in host_count.all(): + res_dict["res"].append(host_count_schema.dump(host).data) + # return counts.data + + return res_dict + def _perform_create(self, data, **kwargs): hostnames = data.pop('hostnames', []) host = super(HostsView, self)._perform_create(data, **kwargs) diff --git a/server/models.py b/server/models.py index 3168cad4b42..1699d0929a0 100644 --- a/server/models.py +++ b/server/models.py @@ -258,6 +258,79 @@ class Host(Metadata): UniqueConstraint(ip, workspace_id, name='uix_host_ip_workspace'), ) + vulnerability_info_count = query_expression() + vulnerability_med_count = query_expression() + vulnerability_high_count = query_expression() + vulnerability_critical_count = query_expression() + vulnerability_low_count = query_expression() + vulnerability_unclassified_count = query_expression() + + @classmethod + def query_with_count(cls, only_confirmed, host_ids, workspace_name): + query = cls.query.join(Workspace).filter(cls.id.in_(host_ids)).filter(Workspace.name == workspace_name) + return query.options( + with_expression( + cls.vulnerability_info_count, + _make_vuln_count_property( + type_=None, + only_confirmed = only_confirmed, + use_column_property = False, + extra_query = "vulnerability.severity='informational'", + get_hosts_vulns = True + ) + ), + with_expression( + cls.vulnerability_med_count, + _make_vuln_count_property( + type_ = None, + only_confirmed = only_confirmed, + use_column_property = False, + extra_query = "vulnerability.severity='medium'", + get_hosts_vulns = True + ) + ), + with_expression( + cls.vulnerability_high_count, + _make_vuln_count_property( + type_ = None, + only_confirmed = only_confirmed, + use_column_property = False, + extra_query = "vulnerability.severity='high'", + get_hosts_vulns = True + ) + ), + with_expression( + cls.vulnerability_critical_count, + _make_vuln_count_property( + type_ = None, + only_confirmed = only_confirmed, + use_column_property = False, + extra_query = "vulnerability.severity='critical'", + get_hosts_vulns = True + ) + ), + with_expression( + cls.vulnerability_low_count, + _make_vuln_count_property( + type_ = None, + only_confirmed = only_confirmed, + use_column_property = False, + extra_query = "vulnerability.severity='low'", + get_hosts_vulns = True + ) + ), + with_expression( + cls.vulnerability_unclassified_count, + _make_vuln_count_property( + type_ = None, + only_confirmed = only_confirmed, + use_column_property = False, + extra_query = "vulnerability.severity='unclassified'", + get_hosts_vulns = True + ) + ), + ) + @property def parent(self): return @@ -1129,11 +1202,23 @@ def parent(self): def _make_vuln_count_property(type_=None, only_confirmed=False, - use_column_property=True, extra_query=None): + use_column_property=True, extra_query=None, get_hosts_vulns=False): + from_clause = table('vulnerability') + + if get_hosts_vulns: + from_clause = from_clause.join( + Service, Vulnerability.service_id == Service.id, + isouter=True + ) + query = (select([func.count(text('vulnerability.id'))]). - select_from(table('vulnerability')). - where(text('vulnerability.workspace_id = workspace.id')) + select_from(from_clause) ) + if get_hosts_vulns: + query = query.where(text('(vulnerability.host_id = host.id OR host.id = service.host_id)')) + else: + query = query.where(text('vulnerability.workspace_id = workspace.id')) + if type_: # Don't do queries using this style! # This can cause SQL injection vulnerabilities From 36746c9dee5d4c983a6db6d5802a1019df584a43 Mon Sep 17 00:00:00 2001 From: montive Date: Thu, 5 Jul 2018 13:18:20 -0300 Subject: [PATCH 1452/1506] Print functions added at status_check.py and click.option added in function status_check() at manage.py --- manage.py | 35 ++++++- server/commands/status_check.py | 165 ++++++++++++++++---------------- 2 files changed, 116 insertions(+), 84 deletions(-) diff --git a/manage.py b/manage.py index 72a4836904e..8055c2cc278 100755 --- a/manage.py +++ b/manage.py @@ -18,7 +18,7 @@ from server.commands.faraday_schema_display import DatabaseSchema from server.commands.app_urls import show_all_urls from server.commands.reports import import_external_reports -from server.commands.status_check import full_status_check +from server.commands import status_check as status_check_functions from server.models import db, User from server.importer import ImportCouchDB @@ -102,10 +102,37 @@ def sql_shell(): pgcli.run_cli() -@click.command(help="Check critical modules in Faraday server application") -def status_check(): - full_status_check() +@click.command() +@click.option('--check_postgresql', default=False, is_flag=True) +@click.option('--check_faraday', default=False, is_flag=True) +@click.option('--check_dependencies', default=False, is_flag=True) +@click.option('--check_config', default=False, is_flag=True) +def status_check(check_postgresql, check_faraday, check_dependencies, check_config): + + selected = False + exit_code = 0 + if check_postgresql: + # exit_code is created for Faraday automation-testing purposes + exit_code = status_check_functions.print_postgresql_status() + selected = True + + if check_faraday: + status_check_functions.print_faraday_status() + selected = True + + if check_dependencies: + status_check_functions.print_depencencies_status() + selected = True + + if check_config: + status_check_functions.print_config_status() + selected = True + + if not selected: + status_check_functions.full_status_check() + + sys.exit(exit_code) def validate_user_unique_field(ctx, param, value): with app.app_context(): diff --git a/server/commands/status_check.py b/server/commands/status_check.py index 06cf9fa985f..742506dbfbb 100644 --- a/server/commands/status_check.py +++ b/server/commands/status_check.py @@ -1,49 +1,52 @@ +import os +import sys +import socket +import argparse import requests import sqlalchemy -import socket -import os import server.config from colorama import init +from server.web import app +from server.models import db +from utils import dependencies from colorama import Fore, Back, Style from server.utils.daemonize import is_server_running -from server.models import db -from server.web import app from config.configuration import getInstanceConfiguration -from utils import dependencies +from config import globals as CONSTANTS CONF = getInstanceConfiguration() - init() -def check_postgres(): +def check_server_running(): + pid = is_server_running() + return pid + + +def check_open_ports(): + address = server.config.faraday_server.bind_address + port = int(server.config.faraday_server.port) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex((address,port)) + if result == 0: + return True + else: + return False + + +def check_postgres(): with app.app_context(): try: - result = db.engine.execute("SELECT version()") + result = str(db.engine.execute("SELECT version()")) return result except sqlalchemy.exc.OperationalError: return None + +def check_client(): -def check_active_user(): - with app.app_context(): - try: - active_users = db.engine.execute('SELECT active FROM faraday_user') - for item in active_users: - if item[0] == True: - return True - except sqlalchemy.exc.OperationalError: - return 0 - - -def check_server_running(): - pid = is_server_running() - return pid - - -def check_client_running(): port_rest = CONF.getApiRestfulConInfoPort() try: @@ -56,6 +59,7 @@ def check_client_running(): def check_server_dependencies(): + installed_deps, missing_deps, conflict_deps = dependencies.check_dependencies( requirements_file='requirements_server.txt') @@ -70,6 +74,7 @@ def check_server_dependencies(): def check_client_dependencies(): + installed_deps, missing_deps, conflict_deps = dependencies.check_dependencies( requirements_file='requirements.txt') @@ -87,6 +92,7 @@ def check_client_dependencies(): def check_credentials(): + api_username = CONF.getAPIUsername() api_password = CONF.getAPIPassword() @@ -94,23 +100,21 @@ def check_credentials(): try: r = requests.post('http://localhost:5985/_api/login', json=values) - if r.status_code == 200 and 'user' in r.json()['response']: - return 200 - + return 200 elif r.status_code == 400: return 400 - elif r.status_code == 500: return 500 + except requests.exceptions.ConnectionError: return None def check_storage_permission(): - home = os.path.expanduser("~") - path = home+'/.faraday/storage/test' - + + path = os.path.join(CONSTANTS.CONST_FARADAY_HOME_PATH,'storage/test') + try: os.mkdir(path) os.rmdir(path) @@ -119,51 +123,35 @@ def check_storage_permission(): return None -def check_open_ports(): - address = server.config.faraday_server.bind_address - port = int(server.config.faraday_server.port) - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex((address,port)) - if result == 0: - return True - else: - return False - - -def full_status_check(): - - - #Checking PostgreSQL - print('\n{white}Checking if PostgreSQL is running...'.format(white=Fore.WHITE)) +def print_postgresql_status(): + """Prints the status of PostgreSQL using check_postgres()""" + exit_code = 0 result = check_postgres() if result: print('[{green}+{white}] PostgreSQL is running'.\ format(green=Fore.GREEN, white=Fore.WHITE)) - + return exit_code else: - print('[{red}-{white}] Could not connect to PostgreSQL, please check if database is running'\ + print('[{red}-{white}] Could not connect to postgresql, please check if database is running'\ .format(red=Fore.RED, white=Fore.WHITE)) + exit_code = 1 + return exit_code - if check_active_user(): - print("[{green}+{white}] Active user exists".format(green=Fore.GREEN, white=Fore.WHITE)) - elif check_active_user() == 0: - print("[{red}-{white}] Faraday database non existant".format(red=Fore.RED, white=Fore.WHITE)) - else: - print("[{red}-{white}] Active user doesn't exists".format(red=Fore.RED, white=Fore.WHITE)) - +def print_faraday_status(): + """Prints Status of farday using check_server_running() and check_client""" - #Checking status - print('\n{white}Checking if Faraday is running...'.format(white=Fore.WHITE)) + #Prints Status of the server using check_server_running() pid = check_server_running() if pid is not None: - print('[{green}+{white}] Faraday Server is Running. PID:{PID} \ + print('[{green}+{white}] Faraday Server is running. PID:{PID} \ '.format(green=Fore.GREEN, PID=pid, white=Fore.WHITE)) else: print('[{red}-{white}] Faraday Server is not running {white} \ '.format(red=Fore.RED, white=Fore.WHITE)) - if check_client_running(): + #Prints Status of the client using check_client() + if check_client(): print('[{green}+{white}] Faraday GTK is running'.\ format(green=Fore.GREEN, white=Fore.WHITE)) else: @@ -171,59 +159,76 @@ def full_status_check(): .format(yellow=Fore.YELLOW, white=Fore.WHITE)) - #Checking dependencies - print('\n{white}Checking Faraday dependencies...'.format(white=Fore.WHITE)) - status, server_dep = check_server_dependencies() +def print_depencencies_status(): + """Prints Status of the dependencies using check_server_dependencies() and check_client_dependencies()""" + status, server_dep = check_server_dependencies() if status == True: - print('[{red}-{white}] Some server\'s dependencies are old. Update them with \"pip install -r requirements_server.txt -U\": (' + ','.join(server_dep) + ')') \ + print('[{red}-{white}] Some server dependencies are old: [' + ', '.join(server_dep) + ']. Update them with \"pip install -r requirements_server.txt -U\"') \ .format(red=Fore.RED, white=Fore.WHITE) elif status == 0: - print('[{red}-{white}] Server\'s dependencies not met. Install them with \"pip install -r requirements_server.txt -U\": (' + ','.join(server_dep) + ')')\ + print('[{red}-{white}] Client dependencies not met: [' + ', '.join(server_dep) + '] Install them with \"pip install -r requirements_server.txt -U\"')\ .format(red=Fore.RED, white=Fore.WHITE) else: - print('[{green}+{white}] Server\'s dependencies met' \ + print('[{green}+{white}] Server dependencies met' \ .format(green=Fore.GREEN, white=Fore.WHITE)) status, client_dep = check_client_dependencies() if status == True: - print('[{red}-{white}] Some client\'s dependencies are old. Update them with \"pip install -r requirements.txt -U\": (' + ','.join(client_dep) + ')') \ + print('[{red}-{white}] Some client dependencies are old: [' + ', '.join(client_dep) + ']. Update them with \"pip install -r requirements.txt -U\"') \ .format(red=Fore.RED, white=Fore.WHITE) elif status == 0: - print('[{red}-{white}] Client\'s dependencies not met. Install them with \"pip install -r requirements.txt -U\": (' + ','.join(client_dep) + ')')\ + print('[{red}-{white}] Client dependencies not met: [' + ', '.join(client_dep) + ']. Install them with \"pip install -r requirements.txt -U\"')\ .format(red=Fore.RED, white=Fore.WHITE) else: - print('[{green}+{white}] Client\'s dependencies met'\ + print('[{green}+{white}] Client dependencies met'\ .format(green=Fore.GREEN, white=Fore.WHITE)) + - - #Checking config - print('\n{white}Checking Faraday config...{white}'.format(white=Fore.WHITE)) +def print_config_status(): + """Prints Status of the configuration using check_credentials(), check_storage_permission() and check_open_ports()""" + + pid = check_server_running() + result = check_postgres() if pid and result: status_code = check_credentials() if status_code == 200: - print('[{green}+{white}] User\'s credentials matched'.format(green=Fore.GREEN, white=Fore.WHITE)) + print('[{green}+{white}] Credentials matched'.format(green=Fore.GREEN, white=Fore.WHITE)) elif status_code == 400: - print('[{red}-{white}] Error. User\'s credentials do not match' \ + print('[{red}-{white}] Error. Credentials does not match' \ .format(red=Fore.RED, white=Fore.WHITE)) else: - print('[{red}-{white}] Failed checking user\'s credentials. Either Faraday Server is not running or database is not working'\ - .format(red=Fore.RED, white=Fore.WHITE)) + print('[{red}-{white}] Credentials can not be checked. Either Faraday Server not running or database not working'.format(red=Fore.RED, white=Fore.WHITE)) if check_storage_permission(): - print('[{green}+{white}] ~/.faraday/storage -> Permission accepted' \ + print('[{green}+{white}] /.faraday/storage -> Permission accepted' \ .format(green=Fore.GREEN, white=Fore.WHITE)) else: - print('[{red}-{white}] ~/.faraday/storage -> Permission denied'\ + print('[{red}-{white}] /.faraday/storage -> Permission denied'\ .format(red=Fore.RED, white=Fore.WHITE)) if check_open_ports(): print "[{green}+{white}] Port {PORT} in {ad} is open"\ .format(PORT=server.config.faraday_server.port, green=Fore.GREEN,white=Fore.WHITE,ad=server.config.faraday_server.bind_address) else: - print "[{red}-{white}] in {ad} is not open"\ + print "[{red}-{white}] Port {PORT} in {ad} is not open"\ .format(PORT=server.config.faraday_server.port,red=Fore.RED,white=Fore.WHITE,ad =server.config.faraday_server.bind_address) + + +def full_status_check(): + print('\n{white}Checking if postgreSQL is running...'.format(white=Fore.WHITE)) + print_postgresql_status() + + print('\n{white}Checking if Faraday is running...'.format(white=Fore.WHITE)) + print_faraday_status() + + print('\n{white}Checking Faraday dependencies...'.format(white=Fore.WHITE)) + print_depencencies_status() + + print('\n{white}Checking Faraday config...{white}'.format(white=Fore.WHITE)) + print_config_status() + From aa7d0a1b033a3edc81ad883db7dbf4f954c02647 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 5 Jul 2018 13:35:23 -0300 Subject: [PATCH 1453/1506] [FIX] specify host none in test --- test_cases/test_api_vulnerability.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_cases/test_api_vulnerability.py b/test_cases/test_api_vulnerability.py index 2d662695e64..ad839d0b732 100644 --- a/test_cases/test_api_vulnerability.py +++ b/test_cases/test_api_vulnerability.py @@ -1456,9 +1456,10 @@ def test_search_by_hostnames_service_case(self, session, test_client): host = HostFactory.create(workspace=workspace) host.hostnames.append(hostname) service = ServiceFactory.create(workspace=workspace, host=host) - vuln = VulnerabilityFactory.create(service=service, workspace=workspace) + vuln = VulnerabilityFactory.create(service=service, host=None, workspace=workspace) session.add(vuln) session.add(vuln2) + session.add(service) session.add(hostname) session.commit() url = self.url(workspace=workspace) + '?hostnames={0}'.format(hostname.name) From 8fc1b89c52fcf8ad60abd18e9e2cc3f8fd4fafae Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 6 Jul 2018 12:10:13 -0300 Subject: [PATCH 1454/1506] [MOD] minor pep8 fixes --- plugins/repo/nexpose/plugin.py | 55 ++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/plugins/repo/nexpose/plugin.py b/plugins/repo/nexpose/plugin.py index be59f3aa8a0..3d8791f1152 100644 --- a/plugins/repo/nexpose/plugin.py +++ b/plugins/repo/nexpose/plugin.py @@ -8,13 +8,11 @@ ''' from __future__ import with_statement -from plugins import core -from model import api import re import os -import pprint -import sys +from plugins import core +from model import api try: import xml.etree.cElementTree as ET import xml.etree.ElementTree as ET_ORIG @@ -52,7 +50,7 @@ def __init__(self, xml_output): tree = self.parse_xml(xml_output) if tree: vulns = self.get_vulns_list(tree) - self.items = [data for data in self.get_items(tree,vulns)] + self.items = [data for data in self.get_items(tree, vulns)] else: self.items = [] @@ -79,7 +77,7 @@ def get_items(self, tree, vulnerabilities): @return items A list of Host instances """ for node in tree.findall("nodes/node"): - yield Item(node,vulnerabilities) + yield Item(node, vulnerabilities) def get_vulns_list(self, tree): """ @@ -88,7 +86,7 @@ def get_vulns_list(self, tree): vulns_list = [] for self.issues in tree.findall("VulnerabilityDefinitions/vulnerability"): vulns_list.append(self.issues) - + return vulns_list @@ -107,12 +105,12 @@ def __init__(self, item_node, vulnerability): # Checking node's vulns node_tests_list = self.get_tests('tests') self.vulns_list = vulnerability - self.node_vulns = self.check_vulns(node_tests_list, self.vulns_list) + self.node_vulns = self.check_vulns(node_tests_list, self.vulns_list) # Checking service's vulns self.service = self.get_service('endpoints/endpoint', item_node) - def get_service(self,path,item_node): + def get_service(self, path, item_node): """ Gets a service. @@ -122,7 +120,7 @@ def get_service(self,path,item_node): for srv in item_node.findall(path): self.node = srv item = {} - tests_list = self.get_tests('services/service/tests') + tests_list = self.get_tests('services/service/tests') item['name'] = self.get_name_from_service('services/service') item['port'] = srv.get('port') item['protocol'] = srv.get('protocol') @@ -132,7 +130,7 @@ def get_service(self,path,item_node): self.srv.append(item) return self.srv - + def get_name_from_service(self, service_path): """ @@ -153,9 +151,9 @@ def get_version(self, path): @return attribute 'product' from a host or a service """ sub_node = self.node.find(path) - if sub_node is not None: + if sub_node is not None: return sub_node.attrib['product'] - + return None def get_tests(self, test_path): @@ -185,13 +183,13 @@ def check_vulns(self, list_of_tests, list_of_vulns): vuln_attributes['id'] = vuln.attrib['id'] vuln_attributes['ref'] = self.get_vulns_ref(vuln) vuln_attributes['severity'] = self.severity_format(vuln.attrib['severity']) - vuln_attributes['description'] = self.convert_to_flat_text(vuln,'description') - vuln_attributes['resolution'] = self.convert_to_flat_text(vuln,'solution') + vuln_attributes['description'] = self.convert_to_flat_text(vuln, 'description') + vuln_attributes['resolution'] = self.convert_to_flat_text(vuln, 'solution') checked_vulns.append(vuln_attributes) - + return checked_vulns - def severity_format(self,severity): + def severity_format(self, severity): """ Convert Nexpose severity format into Faraday API severity format @@ -244,7 +242,7 @@ def get_vulns_ref(self, vuln): return source - def get_text_from_reference(self,vulnerability, reference_path): + def get_text_from_reference(self, vulnerability, reference_path): """ Gets text from the references of a vulnerability. @@ -256,13 +254,13 @@ def get_text_from_reference(self,vulnerability, reference_path): return None - def convert_to_flat_text(self,vuln,tag): + def convert_to_flat_text(self, vuln, tag): """ Converts texts from multiples elements into one flat text @return returns new text """ - self.description = vuln.find(tag) + self.description = vuln.find(tag) xml_str = self.description.itertext() aux_string = [] strings_list = [data for data in xml_str] @@ -306,7 +304,13 @@ def parseOutputString(self, output, debug=False): host_id = self.createAndAddHost(item.ip, item.os) for vuln in item.node_vulns: - vuln_id = self.createAndAddVulnToHost(host_id, vuln['id'], ref=vuln['ref'], severity=vuln['severity'], desc=vuln['description'], resolution=vuln['resolution']) + vuln_id = self.createAndAddVulnToHost( + host_id, vuln['id'], + ref=vuln['ref'], + severity=vuln['severity'], + desc=vuln['description'], + resolution=vuln['resolution'] + ) for srv in item.service: service_id = self.createAndAddServiceToHost(host_id, srv['name'], @@ -317,7 +321,14 @@ def parseOutputString(self, output, debug=False): version=srv['version']) for vuln in srv['vulns']: vuln_id = self.createAndAddVulnToService( - host_id, service_id, vuln['id'], ref=vuln['ref'], severity=vuln['severity'], desc=vuln['description'], resolution=vuln['resolution']) + host_id, + service_id, + vuln['id'], + ref=vuln['ref'], + severity=vuln['severity'], + desc=vuln['description'], + resolution=vuln['resolution'] + ) del parser def processCommandString(self, username, current_path, command_string): From 25ff7b386ec6dae4ecfc25aeccc739f8ebb98802 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 6 Jul 2018 12:16:24 -0300 Subject: [PATCH 1455/1506] Set version as release candidate 1 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 70a5c59c74c..0964d6a2c11 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0b4 +3.0rc1 From edf6e3a8cc68973db92c1f113c516b8127007621 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Fri, 6 Jul 2018 17:01:28 -0300 Subject: [PATCH 1456/1506] [FIX] remove rc from version since distutils breaks when version is pep440 --- plugins/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/plugin.py b/plugins/plugin.py index 7d9cd7ffe9c..1a36347efb3 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -34,7 +34,7 @@ from config.configuration import getInstanceConfiguration CONF = getInstanceConfiguration() -VERSION = server.config.__get_version().split('+')[0] +VERSION = server.config.__get_version().split('-')[0].split('rc')[0] logger = logging.getLogger(__name__) From 5a9d7e3a54d2f783f4666f0923a09db69364f3d2 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Fri, 6 Jul 2018 18:03:02 -0300 Subject: [PATCH 1457/1506] FIX upload reports now set SERVER_URL ok --- persistence/server/server.py | 14 ++++++-------- server/api/modules/upload_reports.py | 20 ++++++++++++++------ 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/persistence/server/server.py b/persistence/server/server.py index ed1217be770..f05efa936f8 100644 --- a/persistence/server/server.py +++ b/persistence/server/server.py @@ -27,7 +27,6 @@ import os import json import logging -from ConfigParser import SafeConfigParser try: import urlparse @@ -57,6 +56,7 @@ FARADAY_UP = True FARADAY_UPLOAD_REPORTS_WEB_COOKIE = None +FARADAY_UPLOAD_REPORTS_OVERWRITE_SERVER_URL = None SERVER_URL = "http://127.0.0.1:5985" AUTH_USER = "" @@ -92,13 +92,11 @@ def _conf(): def _get_base_server_url(): - if FARADAY_UPLOAD_REPORTS_WEB_COOKIE: - parser = SafeConfigParser() - parser.read(LOCAL_CONFIG_FILE) - server_url = 'http://{0}:{1}'.format( - parser.get('faraday_server', 'bind_address'), - parser.get('faraday_server', 'port')) - logger.info('Detected upload cookie. Using server_url as {0}'.format(server_url)) + + # Faraday server is running, and this module is used by upload_reports... + if FARADAY_UPLOAD_REPORTS_OVERWRITE_SERVER_URL: + server_url = FARADAY_UPLOAD_REPORTS_OVERWRITE_SERVER_URL + logger.info('Detected upload cookie Using server_url as {0}'.format(server_url)) elif FARADAY_UP: server_url = _conf().getAPIUrl() logger.info('Detected faraday client running. Using server_url as {0}'.format(server_url)) diff --git a/server/api/modules/upload_reports.py b/server/api/modules/upload_reports.py index 8017d0bef5c..7b00435fe40 100644 --- a/server/api/modules/upload_reports.py +++ b/server/api/modules/upload_reports.py @@ -11,17 +11,15 @@ import string import random import logging -import model.api +import server.config as FaradayServerConfig from flask import ( redirect, request, abort, - jsonify, Blueprint, - session, - make_response ) + from flask_wtf.csrf import validate_csrf from werkzeug.utils import secure_filename from wtforms import ValidationError @@ -37,7 +35,6 @@ from managers.mapper_manager import MapperManager from managers.reports_managers import ReportProcessor -from server.models import User from persistence.server import server from config.configuration import getInstanceConfiguration @@ -49,20 +46,22 @@ upload_api = Blueprint('upload_reports', __name__) - class RawReportProcessor(Thread): def __init__(self): super(RawReportProcessor, self).__init__() from faraday import setupPlugins setupPlugins() + self.pending_actions = Queue() plugin_manager = PluginManager(os.path.join(CONF.getConfigPath(), "plugins")) mappers_manager = MapperManager() + self.model_controller = ModelController(mappers_manager, self.pending_actions) self.model_controller.start() self.end_event = Event() + plugin_controller = PluginController( 'PluginController', plugin_manager, @@ -80,24 +79,33 @@ def stop(self): def run(self): while not self._stop: try: + workspace, file_path, cookie = UPLOAD_REPORTS_QUEUE.get(False, timeout=0.1) get_logger().info('Processing raw report {0}'.format(file_path)) # Cookie of user, used to create objects in server with the right owner. server.FARADAY_UPLOAD_REPORTS_WEB_COOKIE = cookie + server.FARADAY_UPLOAD_REPORTS_OVERWRITE_SERVER_URL = "http://{0}:{1}".format( + FaradayServerConfig.faraday_server.bind_address, FaradayServerConfig.faraday_server.port) + self.processor.ws_name = workspace + command_id = self.processor.processReport(file_path) UPLOAD_REPORTS_CMD_QUEUE.put(command_id) if not command_id: continue + self.end_event.wait() get_logger().info('Report processing of report {0} finished'.format(file_path)) self.end_event.clear() + except Empty: time.sleep(0.1) + except KeyboardInterrupt as ex: get_logger().info('Keyboard interrupt, stopping report processing thread') self.stop() + except Exception as ex: get_logger().exception(ex) continue From 6727bb8d4307f01a50d998c1795345a1b1d3b410 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Tue, 10 Jul 2018 11:38:45 -0300 Subject: [PATCH 1458/1506] [ADD] API endpoint test --- server/models.py | 10 +++ test_cases/models/test_host.py | 123 ++++++++++++++++++++++++++++++++- 2 files changed, 132 insertions(+), 1 deletion(-) diff --git a/server/models.py b/server/models.py index 1699d0929a0..47fd434c247 100644 --- a/server/models.py +++ b/server/models.py @@ -264,6 +264,7 @@ class Host(Metadata): vulnerability_critical_count = query_expression() vulnerability_low_count = query_expression() vulnerability_unclassified_count = query_expression() + vulnerability_total_count = query_expression() @classmethod def query_with_count(cls, only_confirmed, host_ids, workspace_name): @@ -329,6 +330,15 @@ def query_with_count(cls, only_confirmed, host_ids, workspace_name): get_hosts_vulns = True ) ), + with_expression( + cls.vulnerability_total_count, + _make_vuln_count_property( + type_ = None, + only_confirmed = only_confirmed, + use_column_property = False, + get_hosts_vulns = True + ) + ), ) @property diff --git a/test_cases/models/test_host.py b/test_cases/models/test_host.py index 4979b7c925a..9a37f0630ea 100644 --- a/test_cases/models/test_host.py +++ b/test_cases/models/test_host.py @@ -7,8 +7,14 @@ import random import pytest from functools import partial -from server.models import Hostname +from server.models import Hostname, Host +from server.api.modules.hosts import HostsView + +from test_cases.test_api_workspaced_base import ( + ReadOnlyAPITests) +from test_cases import factories +from test_cases.factories import WorkspaceFactory @pytest.mark.parametrize( "with_host_vulns,with_service_vulns", [[True, False], @@ -120,3 +126,118 @@ def test_change_one(self, host, session): session.commit() assert len(host.hostnames) == 1 assert host.hostnames[0].name == 'y' + +HOST_TO_QUERY_AMOUNT = 3 +HOST_NOT_TO_QUERY_AMOUNT = 2 +SERVICE_BY_HOST = 3 +VULN_BY_HOST = 2 +VULN_BY_SERVICE = 1 + +class TestHostAPI(ReadOnlyAPITests): + model = Host + factory = factories.HostFactory + api_endpoint = 'hosts' + view_class = HostsView + + # This test the api endpoint for some of the host in the ws, with existing other host with vulns in the same and + # other ws + @pytest.mark.parametrize('querystring', ['countVulns/?hosts={}', + ]) + def test_vuln_count(self, + vulnerability_factory, + host_factory, + service_factory, + workspace_factory, + test_client, + session, + querystring): + + workspace1 = workspace_factory.create() + workspace2 = workspace_factory.create() + session.add(workspace1) + session.add(workspace2) + session.commit() + + hosts_to_query = host_factory.create_batch(HOST_TO_QUERY_AMOUNT, workspace=workspace1) + hosts_not_to_query = host_factory.create_batch(HOST_NOT_TO_QUERY_AMOUNT, workspace=workspace1) + hosts_not_to_query_w2 = host_factory.create_batch(HOST_NOT_TO_QUERY_AMOUNT, workspace=workspace2) + hosts = hosts_to_query + hosts_not_to_query + hosts_not_to_query_w2 + + services = [] + vulns = [] + + session.add_all(hosts) + + for host in hosts: + services += service_factory.create_batch(SERVICE_BY_HOST, host=host, workspace=host.workspace) + vulns += vulnerability_factory.create_batch(VULN_BY_HOST, host=host, service=None, workspace=host.workspace) + + session.add_all(services) + + for service in services: + vulns += vulnerability_factory.create_batch(VULN_BY_SERVICE, service=service, host=None, + workspace=service.workspace) + + session.add_all(vulns) + session.commit() + + url = self.url(workspace=workspace1) + querystring.format(",".join(map(lambda x: str(x.id), hosts_to_query))) + res = test_client.get(url) + + assert res.status_code == 200 + + for index, host in enumerate(hosts_to_query): + assert res.json['res'][index]['total'] == VULN_BY_HOST + VULN_BY_SERVICE * SERVICE_BY_HOST + assert host.id in map(lambda x: x['host_id'], res.json['res']) + + # This test the api endpoint for some of the host in the ws, with existing other host in other ws and ask for the + # other hosts + @pytest.mark.parametrize('querystring', [ 'countVulns/?hosts={}', + ]) + def test_vuln_count_ignore_other_ws(self, + vulnerability_factory, + host_factory, + service_factory, + workspace_factory, + test_client, + session, + querystring): + + workspace1 = workspace_factory.create() + workspace2 = workspace_factory.create() + session.add(workspace1) + session.add(workspace2) + session.commit() + + hosts_to_query = host_factory.create_batch(HOST_TO_QUERY_AMOUNT, workspace=workspace1) + hosts_not_to_query_w2 = host_factory.create_batch(HOST_NOT_TO_QUERY_AMOUNT, workspace=workspace2) + hosts = hosts_to_query + hosts_not_to_query_w2 + + services = [] + vulns = [] + + session.add_all(hosts) + + for host in hosts: + services += service_factory.create_batch(SERVICE_BY_HOST, host=host, workspace=host.workspace) + vulns += vulnerability_factory.create_batch(VULN_BY_HOST, host=host, service=None, workspace=host.workspace) + + session.add_all(services) + + for service in services: + vulns += vulnerability_factory.create_batch(VULN_BY_SERVICE, service=service, host=None, workspace=service.workspace) + + session.add_all(vulns) + session.commit() + + url = self.url(workspace=workspace1) + querystring.format(",".join(map(lambda x: str(x.id), hosts))) + res = test_client.get(url) + + assert res.status_code == 200 + + for index, host in enumerate(hosts_to_query): + assert res.json['res'][index]['total'] == VULN_BY_HOST + VULN_BY_SERVICE * SERVICE_BY_HOST + assert host.id in map(lambda x: x['host_id'], res.json['res']) + + for host in hosts_not_to_query_w2: + assert host.id not in map(lambda x: x['host_id'], res.json['res']) \ No newline at end of file From e6b1b180c8e1f923798f5280ac3dc16916ab257e Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Tue, 10 Jul 2018 12:26:08 -0300 Subject: [PATCH 1459/1506] [MOD] Dict response to {'hosts':{'host_id1':{'info': 0, ...}, ...}} --- server/api/modules/hosts.py | 4 ++-- test_cases/models/test_host.py | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 49c5caef6f9..964313886ab 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -124,13 +124,13 @@ def count_vulns(self, workspace_name): host_ids = flask.request.args.get('hosts', '') host_id_list = host_ids.split(',') - res_dict = {'res':[]} + res_dict = {'hosts':{}} host_count_schema = HostCountSchema() host_count = Host.query_with_count(False, host_id_list, workspace_name) for host in host_count.all(): - res_dict["res"].append(host_count_schema.dump(host).data) + res_dict["hosts"][host.id] = host_count_schema.dump(host).data # return counts.data return res_dict diff --git a/test_cases/models/test_host.py b/test_cases/models/test_host.py index 9a37f0630ea..395cef4dd8a 100644 --- a/test_cases/models/test_host.py +++ b/test_cases/models/test_host.py @@ -186,9 +186,9 @@ def test_vuln_count(self, assert res.status_code == 200 - for index, host in enumerate(hosts_to_query): - assert res.json['res'][index]['total'] == VULN_BY_HOST + VULN_BY_SERVICE * SERVICE_BY_HOST - assert host.id in map(lambda x: x['host_id'], res.json['res']) + for host in hosts_to_query: + assert res.json['hosts'][str(host.id)]['total'] == VULN_BY_HOST + VULN_BY_SERVICE * SERVICE_BY_HOST + assert str(host.id) in res.json['hosts'] # This test the api endpoint for some of the host in the ws, with existing other host in other ws and ask for the # other hosts @@ -235,9 +235,9 @@ def test_vuln_count_ignore_other_ws(self, assert res.status_code == 200 - for index, host in enumerate(hosts_to_query): - assert res.json['res'][index]['total'] == VULN_BY_HOST + VULN_BY_SERVICE * SERVICE_BY_HOST - assert host.id in map(lambda x: x['host_id'], res.json['res']) + for host in hosts_to_query: + assert res.json['hosts'][str(host.id)]['total'] == VULN_BY_HOST + VULN_BY_SERVICE * SERVICE_BY_HOST + assert str(host.id) in res.json['hosts'] for host in hosts_not_to_query_w2: - assert host.id not in map(lambda x: x['host_id'], res.json['res']) \ No newline at end of file + assert str(host.id) not in res.json['hosts'] From 96e073626e0e39c42303ca3a7af7b92c27c4e8ff Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Tue, 10 Jul 2018 16:12:50 -0300 Subject: [PATCH 1460/1506] FIX #4933, print and abort with message when configuration files are not found --- server/api/modules/upload_reports.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/server/api/modules/upload_reports.py b/server/api/modules/upload_reports.py index 8017d0bef5c..3193bcdfb07 100644 --- a/server/api/modules/upload_reports.py +++ b/server/api/modules/upload_reports.py @@ -49,7 +49,6 @@ upload_api = Blueprint('upload_reports', __name__) - class RawReportProcessor(Thread): def __init__(self): @@ -58,7 +57,14 @@ def __init__(self): setupPlugins() self.pending_actions = Queue() - plugin_manager = PluginManager(os.path.join(CONF.getConfigPath(), "plugins")) + try: + plugin_manager = PluginManager(os.path.join(CONF.getConfigPath(), "plugins")) + except AttributeError: + get_logger().warning( + "Upload reports in WEB-UI not configurated, run Faraday client and try again...") + self._stop = True + return + mappers_manager = MapperManager() self.model_controller = ModelController(mappers_manager, self.pending_actions) self.model_controller.start() @@ -130,9 +136,14 @@ def file_upload(workspace=None): random_prefix = ''.join(random.choice(chars) for x in range(12)) raw_report_filename = '{0}{1}'.format(random_prefix, secure_filename(report_file.filename)) - file_path = os.path.join( - CONF.getConfigPath(), - 'uploaded_reports/{0}'.format(raw_report_filename)) + try: + file_path = os.path.join( + CONF.getConfigPath(), + 'uploaded_reports/{0}'.format(raw_report_filename)) + except AttributeError: + get_logger().warning( + "Upload reports in WEB-UI not configurated, run Faraday client and try again...") + abort(make_response(jsonify(message="Upload reports not configurated: Run faraday client and start Faraday server again"), 500)) with open(file_path, 'w') as output: output.write(report_file.read()) From 28949fde5bffded09de3919249561185f4dc8649 Mon Sep 17 00:00:00 2001 From: Ezequiel Tavella Date: Tue, 10 Jul 2018 18:10:46 -0300 Subject: [PATCH 1461/1506] FIX #4950, references are overwrite from vulnerability template --- .../www/scripts/statusReport/controllers/statusReport.js | 8 -------- 1 file changed, 8 deletions(-) diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index b75d5d7c869..3f8566eb38e 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -968,14 +968,6 @@ angular.module("faradayApp") }); modal.result.then(function(data) { $scope.getCurrentSelection().forEach(function(vuln) { - var references = vuln.refs.concat([]); - data.refs.forEach(function(ref) { - if(vuln.refs.indexOf(ref) == -1){ - references.push(ref); - } - }); - data.refs = references; - vulnsManager.updateVuln(vuln, data).then(function(vulns){ }, function(errorMsg){ // TODO: show errors somehow From c147648eceb42cdf455e8afe05a9b1e13f34adfb Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Thu, 12 Jul 2018 11:28:21 -0300 Subject: [PATCH 1462/1506] [ADD] API request endpoint (/ws/ws_name/hosts/countVulns) returns all hosts available. Test added too --- server/api/modules/hosts.py | 7 +++++-- server/models.py | 8 +++++--- test_cases/models/test_host.py | 5 +++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/server/api/modules/hosts.py b/server/api/modules/hosts.py index 964313886ab..6ad59ca3e24 100644 --- a/server/api/modules/hosts.py +++ b/server/api/modules/hosts.py @@ -121,8 +121,11 @@ def service_list(self, workspace_name, host_id): @route('/countVulns/') def count_vulns(self, workspace_name): - host_ids = flask.request.args.get('hosts', '') - host_id_list = host_ids.split(',') + host_ids = flask.request.args.get('hosts', None) + if host_ids: + host_id_list = host_ids.split(',') + else: + host_id_list = None res_dict = {'hosts':{}} diff --git a/server/models.py b/server/models.py index 47fd434c247..1ab2ad373b8 100644 --- a/server/models.py +++ b/server/models.py @@ -268,7 +268,9 @@ class Host(Metadata): @classmethod def query_with_count(cls, only_confirmed, host_ids, workspace_name): - query = cls.query.join(Workspace).filter(cls.id.in_(host_ids)).filter(Workspace.name == workspace_name) + query = cls.query.join(Workspace).filter(Workspace.name == workspace_name) + if host_ids: + query = query.filter(cls.id.in_(host_ids)) return query.options( with_expression( cls.vulnerability_info_count, @@ -292,13 +294,13 @@ def query_with_count(cls, only_confirmed, host_ids, workspace_name): ), with_expression( cls.vulnerability_high_count, - _make_vuln_count_property( + _make_vuln_count_property( type_ = None, only_confirmed = only_confirmed, use_column_property = False, extra_query = "vulnerability.severity='high'", get_hosts_vulns = True - ) + ) ), with_expression( cls.vulnerability_critical_count, diff --git a/test_cases/models/test_host.py b/test_cases/models/test_host.py index 395cef4dd8a..544dd3ff50b 100644 --- a/test_cases/models/test_host.py +++ b/test_cases/models/test_host.py @@ -191,8 +191,8 @@ def test_vuln_count(self, assert str(host.id) in res.json['hosts'] # This test the api endpoint for some of the host in the ws, with existing other host in other ws and ask for the - # other hosts - @pytest.mark.parametrize('querystring', [ 'countVulns/?hosts={}', + # other hosts and test the api endpoint for all of the host in the ws, retrieving all host when none is required + @pytest.mark.parametrize('querystring', [ 'countVulns/?hosts={}', 'countVulns/', ]) def test_vuln_count_ignore_other_ws(self, vulnerability_factory, @@ -234,6 +234,7 @@ def test_vuln_count_ignore_other_ws(self, res = test_client.get(url) assert res.status_code == 200 + assert len(res.json['hosts']) == HOST_TO_QUERY_AMOUNT for host in hosts_to_query: assert res.json['hosts'][str(host.id)]['total'] == VULN_BY_HOST + VULN_BY_SERVICE * SERVICE_BY_HOST From 1961225343129125e59e7aa43a9793d1c22def04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 12 Jul 2018 12:11:50 -0300 Subject: [PATCH 1463/1506] Fix group by severity (ugly hack) #TieItWithWire --- .../www/scripts/statusReport/controllers/statusReport.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 3f8566eb38e..54db036099f 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -487,6 +487,13 @@ angular.module("faradayApp") column.grouping = { groupPriority: 0 }; paginationOptions.sortColumn = colname; paginationOptions.sortDirection = 'asc'; + }else if (colname === 'sev' && $scope.propertyGroupBy === 'severity'){ + // Ugly ugly hack so I don't have to change the displayName of + // severity from "sev" to "severity" + console.log('columns', $scope.columns); + column.grouping = { groupPriority: 0 }; + paginationOptions.sortColumn = 'severity'; + paginationOptions.sortDirection = 'asc' } } }; From 8451bbeb694242aa75cd48aa2d3597a5ad33c375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Thu, 12 Jul 2018 12:32:22 -0300 Subject: [PATCH 1464/1506] Remove accidentally commited console.log --- server/www/scripts/statusReport/controllers/statusReport.js | 1 - 1 file changed, 1 deletion(-) diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js index 54db036099f..4ebcf150bc8 100644 --- a/server/www/scripts/statusReport/controllers/statusReport.js +++ b/server/www/scripts/statusReport/controllers/statusReport.js @@ -490,7 +490,6 @@ angular.module("faradayApp") }else if (colname === 'sev' && $scope.propertyGroupBy === 'severity'){ // Ugly ugly hack so I don't have to change the displayName of // severity from "sev" to "severity" - console.log('columns', $scope.columns); column.grouping = { groupPriority: 0 }; paginationOptions.sortColumn = 'severity'; paginationOptions.sortDirection = 'asc' From bc0a3086a222a9b062a6635eec0e0ffc55768380 Mon Sep 17 00:00:00 2001 From: Leonardo Lazzaro Date: Thu, 12 Jul 2018 13:13:53 -0300 Subject: [PATCH 1465/1506] [FIX] update error message --- gui/nogui/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui/nogui/application.py b/gui/nogui/application.py index a766bc5ce64..2e8386dd6b2 100644 --- a/gui/nogui/application.py +++ b/gui/nogui/application.py @@ -39,7 +39,7 @@ def run(self, args): "check configuration.") % workspace) getLogger(self).error( "You may try and go to ~/.faraday/config/user.xml " - "to set a valid couch_uri and last_workspace") + "to set a valid api_uri and last_workspace") getLogger(self).error(str(e)) return -1 workspace = ws.name From c588c38f10c565168eea75aeee95ec99bf0b39f9 Mon Sep 17 00:00:00 2001 From: Eric Horvat Date: Thu, 12 Jul 2018 16:33:22 -0300 Subject: [PATCH 1466/1506] [ADD] VERSION file test check --- test_cases/test_server_config.py | 48 ++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/test_cases/test_server_config.py b/test_cases/test_server_config.py index e6f9125d1c9..49b9e198e17 100644 --- a/test_cases/test_server_config.py +++ b/test_cases/test_server_config.py @@ -7,6 +7,8 @@ import random import string import mock +import os +import re from server.config import ( copy_default_config_to_local, @@ -32,3 +34,49 @@ def test_copy_default_config_to_local_does_not_exist(copyfile, makedirs): assert copy_default_config_to_local() is None assert not makedirs.called assert not copyfile.called + +VERSION_PATTERN = r""" + v? + (?: + (?:(?P[0-9]+)!)? # epoch + (?P[0-9]+(?:\.[0-9]+)*) # release segment + (?P
                                              # pre-release
+            [-_\.]?
+            (?P(a|b|c|rc|alpha|beta|pre|preview))
+            [-_\.]?
+            (?P[0-9]+)?
+        )?
+        (?P                                         # post release
+            (?:-(?P[0-9]+))
+            |
+            (?:
+                [-_\.]?
+                (?Ppost|rev|r)
+                [-_\.]?
+                (?P[0-9]+)?
+            )
+        )?
+        (?P                                          # dev release
+            [-_\.]?
+            (?Pdev)
+            [-_\.]?
+            (?P[0-9]+)?
+        )?
+    )
+    (?:\+(?P[a-z0-9]+(?:[-_\.][a-z0-9]+)*))?       # local version
+"""
+
+_regex = re.compile(
+    r"^\s*" + VERSION_PATTERN + r"\s*$",
+    re.VERBOSE | re.IGNORECASE,
+)
+
+def isPEP440(arg):
+    return not _regex.match(arg) is None
+
+
+def test_exists_and_content():
+    f = open(os.path.join(os.getcwd(),"..","VERSION"),"r")
+    line1 = f.readline().rstrip()
+    assert f.read() == ''
+    assert isPEP440(line1)

From 8ffcb2bcf40a54ed129956732b0f1606c3e4a5d5 Mon Sep 17 00:00:00 2001
From: Eric Horvat 
Date: Thu, 12 Jul 2018 19:17:34 -0300
Subject: [PATCH 1467/1506] [FIX] Try to throw encode error, encoding in
 cleaning function

---
 server/utils/invalid_chars.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/server/utils/invalid_chars.py b/server/utils/invalid_chars.py
index db0278c2f9c..ade9472c395 100644
--- a/server/utils/invalid_chars.py
+++ b/server/utils/invalid_chars.py
@@ -66,6 +66,9 @@ def clean_char(char):
             if char == rep:
                 #print ord(char), char.encode('ascii', 'ignore')
                 return new_char
+        #get here if pass all controls, so try to encode or throw UnicodeEncodeError
+        char.encode()
+
         return char
     except UnicodeEncodeError:
         # Ugly hack triggered when importing some strange objects

From 604bd1dfd072c9c76f699a26c936e6c82c94410f Mon Sep 17 00:00:00 2001
From: Leonardo Lazzaro 
Date: Fri, 13 Jul 2018 12:56:46 -0300
Subject: [PATCH 1468/1506] Save user.xml if new admin was created

---
 server/commands/initdb.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/server/commands/initdb.py b/server/commands/initdb.py
index ef8fb96a85f..1504010a64b 100644
--- a/server/commands/initdb.py
+++ b/server/commands/initdb.py
@@ -123,13 +123,19 @@ def _create_admin_user(self, conn_string):
         if not already_created:
             if not os.path.isfile(FARADAY_USER_CONFIG_XML):
                 shutil.copy(FARADAY_BASE_CONFIG_XML, FARADAY_USER_CONFIG_XML)
-
+            self._save_user_xml(random_password)
             print("Admin user created with \n\n{red}username: {white}faraday \n"
                   "{red}password:{white} {"
                   "random_password} \n".format(random_password=random_password,
                                             white=Fore.WHITE, red=Fore.RED))
             print("{yellow}WARNING{white}: If you are going to execute couchdb importer you must use the couchdb password for faraday user.".format(white=Fore.WHITE, yellow=Fore.YELLOW))
 
+    def _save_user_xml(self, random_password):
+        conf = getInstanceConfiguration()
+        conf.setAPIUrl('http://localhost:5985')
+        conf.setAPIUsername('faraday')
+        conf.setAPIPassword(random_password)
+        conf.saveConfig()
 
     def _configure_existing_postgres_user(self):
         username = raw_input('Please enter the postgresql username: ')

From 5551e426bfb6810b0d341da8c01e90427eedff3b Mon Sep 17 00:00:00 2001
From: Leonardo Lazzaro 
Date: Fri, 13 Jul 2018 15:35:21 -0300
Subject: [PATCH 1469/1506] Sometimes faraday_postgresql user is created, but
 not present in server.ini

---
 server/commands/initdb.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/server/commands/initdb.py b/server/commands/initdb.py
index 1504010a64b..2fef9162196 100644
--- a/server/commands/initdb.py
+++ b/server/commands/initdb.py
@@ -184,6 +184,10 @@ def _configure_new_postgres_user(self, psql_log_file):
             print("{yellow}WARNING{white}: Role {username} already exists, skipping creation ".format(yellow=Fore.YELLOW, white=Fore.WHITE, username=username))
 
             try:
+                if not getattr(server.config, 'database', None):
+                    print('Manual configuration? \n faraday_postgresql was found in PostgreSQL, but no connection string was found in server.ini. ')
+                    print('Please configure [database] section with correct postgresql string. Ex. postgresql+psycopg2://faraday_postgresql:PASSWORD@localhost/faraday')
+                    sys.exit(1)
                 password = server.config.database.connection_string.split(':')[2].split('@')[0]
                 connection = psycopg2.connect(dbname='postgres',
                                               user=username,

From d05344116bb4b3ba4832ab9733cd60c5bb852a10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mat=C3=ADas=20Lang?= 
Date: Fri, 13 Jul 2018 15:42:31 -0300
Subject: [PATCH 1470/1506] Make importer single-threaded

It can be a bit slower but we don't have to deal with concurrency
problems. The script should be only run once so it is better to have
reliability than good performance!
---
 server/importer.py | 41 ++++++++++++-----------------------------
 1 file changed, 12 insertions(+), 29 deletions(-)

diff --git a/server/importer.py b/server/importer.py
index 001e3a892b1..ec5b4ebf586 100644
--- a/server/importer.py
+++ b/server/importer.py
@@ -12,7 +12,6 @@
 import json
 import logging
 import datetime
-import threading
 import multiprocessing
 
 
@@ -599,20 +598,18 @@ def update_from_document(self, document, workspace, level=None, couchdb_relation
                 yield service
 
 
-user_lock = threading.Lock()
 def get_or_create_user(session, username):
-    with user_lock:
-        rng = SystemRandom()
-        password =  "".join(
-            [rng.choice(string.ascii_letters + string.digits) for _ in
-                xrange(12)])
-        creator, created = get_or_create(session, User, username=username)
-        if created:
-            creator.active = False
-            creator.password = password
-        session.add(creator) # remove me
-        session.commit() # remove me
-        return creator
+    rng = SystemRandom()
+    password =  "".join(
+        [rng.choice(string.ascii_letters + string.digits) for _ in
+            xrange(12)])
+    creator, created = get_or_create(session, User, username=username)
+    if created:
+        creator.active = False
+        creator.password = password
+    session.add(creator) # remove me
+    session.commit() # remove me
+    return creator
 
 
 class VulnerabilityImporter(object):
@@ -1367,21 +1364,7 @@ def run(self):
                     logger.error(u"Unauthorized access to CouchDB. Make sure faraday-server's"\
                                  " configuration file has CouchDB admin's credentials set")
                     sys.exit(1)
-                thread = threading.Thread(target=self.import_workspace_into_database, args=(workspace_name, pbar))
-                thread.daemon = True
-                thread.start()
-                workspace_threads.append(thread)
-                if len(workspace_threads) > multiprocessing.cpu_count() * 2:
-                    for thread in workspace_threads:
-                        thread.join()
-                        pbar.update(1)
-                        workspace_threads.remove(thread)
-
-            logger.info('Waiting for treads to finish.')
-            for thread in workspace_threads:
-                thread.join()
-                pbar.update(1)
-                #self.import_workspace_into_database(workspace_name)
+                self.import_workspace_into_database(workspace_name, pbar)
 
     def get_objs(self, host, obj_type, level, workspace):
         if obj_type == 'Credential':

From aab9f730732db6997eb70a31f1ca84e558d8bfe0 Mon Sep 17 00:00:00 2001
From: Leonardo Lazzaro 
Date: Fri, 13 Jul 2018 16:12:09 -0300
Subject: [PATCH 1471/1506] Copy default is no user.xml was found

---
 server/commands/initdb.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/server/commands/initdb.py b/server/commands/initdb.py
index 2fef9162196..77d4b016756 100644
--- a/server/commands/initdb.py
+++ b/server/commands/initdb.py
@@ -18,9 +18,12 @@
 import sqlalchemy
 from sqlalchemy import create_engine
 
-from config.configuration import getInstanceConfiguration
-from faraday import FARADAY_USER_CONFIG_XML, FARADAY_BASE_CONFIG_XML, \
+from config.configuration import Configuration
+from faraday import (
+    FARADAY_USER_CONFIG_XML,
+    FARADAY_BASE_CONFIG_XML,
     FARADAY_BASE
+)
 
 try:
     # py2.7
@@ -131,7 +134,10 @@ def _create_admin_user(self, conn_string):
             print("{yellow}WARNING{white}: If you are going to execute couchdb importer you must use the couchdb password for faraday user.".format(white=Fore.WHITE, yellow=Fore.YELLOW))
 
     def _save_user_xml(self, random_password):
-        conf = getInstanceConfiguration()
+        user_xml = os.path.expanduser("~/.faraday/config/user.xml")
+        if not os.path.exists(user_xml):
+            shutil.copy(FARADAY_BASE_CONFIG_XML, user_xml)
+        conf = Configuration(user_xml)
         conf.setAPIUrl('http://localhost:5985')
         conf.setAPIUsername('faraday')
         conf.setAPIPassword(random_password)

From 7b3c65bb27f9a9288f8ae837d7517f7e5cc75b53 Mon Sep 17 00:00:00 2001
From: Leonardo Lazzaro 
Date: Fri, 13 Jul 2018 18:30:13 -0300
Subject: [PATCH 1472/1506] If credentials are set and login was not forced
 avoid asking input to user

---
 faraday.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/faraday.py b/faraday.py
index ed80e1165ed..7f30b9d52a1 100755
--- a/faraday.py
+++ b/faraday.py
@@ -451,16 +451,20 @@ def try_login_user(server_uri, api_username, api_password):
         sys.exit(-2)
 
 
-def doLoginLoop():
-    """ 
+def doLoginLoop(force_login=False):
+    """
     Sets the username and passwords from the command line.
-    If --login flag is set then username and password is set 
+    If --login flag is set then username and password is set
     """
 
     try:
 
         CONF = getInstanceConfiguration()
         old_server_url = CONF.getAPIUrl()
+        api_username = CONF.getAPIUsername()
+        api_password = CONF.getAPIPassword()
+        if old_server_url and api_username and api_password and not force_login:
+            return
 
         if old_server_url is None:
             new_server_url = raw_input(
@@ -511,9 +515,9 @@ def login(forced_login):
     server_uri = CONF.getServerURI()
     api_username = CONF.getAPIUsername()
     api_password = CONF.getAPIPassword()
-    
+
     if forced_login:
-        doLoginLoop()
+        doLoginLoop(forced_login)
         return
 
     if server_uri and api_username and api_password:
@@ -524,7 +528,7 @@ def login(forced_login):
             CONF.setDBSessionCookies(session_cookie)
             logger.info('Login successful: {0}'.format(api_username))
             return
-    
+
     doLoginLoop()
 
 

From 81380a899f255f656d119256d3c386e5a1f57cba Mon Sep 17 00:00:00 2001
From: Ezequiel Tavella 
Date: Mon, 16 Jul 2018 16:29:48 -0300
Subject: [PATCH 1473/1506] Fix issue with nginx reverse proxy, faraday dont
 know about https session, so redirect,  cant be used

---
 server/api/modules/upload_reports.py                        | 3 +--
 server/www/scripts/services/controllers/serviceModalEdit.js | 2 --
 server/www/scripts/statusReport/controllers/statusReport.js | 5 +----
 3 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/server/api/modules/upload_reports.py b/server/api/modules/upload_reports.py
index 3193bcdfb07..90adf5b507e 100644
--- a/server/api/modules/upload_reports.py
+++ b/server/api/modules/upload_reports.py
@@ -14,7 +14,6 @@
 import model.api
 
 from flask import (
-    redirect,
     request,
     abort,
     jsonify,
@@ -149,4 +148,4 @@ def file_upload(workspace=None):
             output.write(report_file.read())
 
     UPLOAD_REPORTS_QUEUE.put((workspace, file_path, request.cookies))
-    return redirect('/#/dashboard/ws/' + workspace)
+    return make_response(jsonify(message="ok"), 200)
diff --git a/server/www/scripts/services/controllers/serviceModalEdit.js b/server/www/scripts/services/controllers/serviceModalEdit.js
index 3e1c8cdda67..2a64833e0a1 100644
--- a/server/www/scripts/services/controllers/serviceModalEdit.js
+++ b/server/www/scripts/services/controllers/serviceModalEdit.js
@@ -35,9 +35,7 @@ angular.module('faradayApp')
             timestamp = date.getTime()/1000.0;
 
             for (var i = 0; i < $scope.servicesSelected.length; i++) {
-                console.log($scope.servicesSelected[i]._id);
                 updateAll.push(servicesManager.getService($scope.servicesSelected[i]._id, ws, false).then(function(serviceObj){
-                    console.log(serviceObj._id);
                     $scope.data['_id'] = serviceObj._id;
                     return servicesManager.updateService(serviceObj, $scope.data, $routeParams.wsId);
                 }));
diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js
index 4ebcf150bc8..2a694354931 100644
--- a/server/www/scripts/statusReport/controllers/statusReport.js
+++ b/server/www/scripts/statusReport/controllers/statusReport.js
@@ -1122,15 +1122,12 @@ angular.module("faradayApp")
                 withCredentials: false,
                 headers: {'Content-Type': undefined},
                 responseType: "arraybuffer",
-                params: {
-                  fd
-                }
             }).then(
                 function(d) {
                     $location.path("/dashboard/ws/" + $routeParams.wsId);
                 },
                 function(d){
-                    commonsFact.showMessage("Error uploading report");
+                        commonsFact.showMessage("Error uploading report");
                 }
             );
         };

From eb77f78e641f4c5d5d2f89990e148eecd5dbec37 Mon Sep 17 00:00:00 2001
From: Ezequiel Tavella 
Date: Mon, 16 Jul 2018 17:22:01 -0300
Subject: [PATCH 1474/1506] Fix issue with bad url in licenses

---
 server/www/scripts/licenses/providers/license.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/www/scripts/licenses/providers/license.js b/server/www/scripts/licenses/providers/license.js
index 60f6185af27..a9e33643d4c 100644
--- a/server/www/scripts/licenses/providers/license.js
+++ b/server/www/scripts/licenses/providers/license.js
@@ -57,7 +57,7 @@ angular.module('faradayApp')
                 var deferred = $q.defer(),
                 self = this;
 
-                var url = APIURL + "licenses/" + self._id;
+                var url = APIURL + "licenses/" + self._id + "/";
 
                 $http.delete(url)
                     .then(function(resp) {
@@ -71,7 +71,7 @@ angular.module('faradayApp')
             update: function(data) {
                 var deferred = $q.defer(),
                 self = this;
-                var url = APIURL + "licenses/" + self._id;
+                var url = APIURL + "licenses/" + self._id + "/";
 
                 $http.put(url, data)
                     .then(function(res) {

From e77328be3c3459f1b14a5599e758b371958a3442 Mon Sep 17 00:00:00 2001
From: Leonardo Lazzaro 
Date: Tue, 17 Jul 2018 12:26:07 -0300
Subject: [PATCH 1475/1506] Check if postgresql conn string is invalid

---
 faraday-server.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/faraday-server.py b/faraday-server.py
index 2ede16436c8..66d7750a681 100755
--- a/faraday-server.py
+++ b/faraday-server.py
@@ -94,6 +94,11 @@ def check_postgresql():
         try:
             if not db.session.query(Workspace).count():
                 logger.warn('No workspaces found. Remeber to execute couchdb importer')
+        except sqlalchemy.exc.ArgumentError:
+            logger.error(
+                '\n\b{RED}Please check you postgresql connection string in server.ini at .faraday on your ohme directory.{WHITE} \n'.format(RED=Fore.RED, WHITE=Fore.WHITE)
+            )
+            sys.exit(1)
         except sqlalchemy.exc.OperationalError:
             logger.error(
                     '\n\n{RED}Could not connect to postgresql.\n{WHITE}Please check: \n{YELLOW}  * if database is running \n  * configuration settings are correct. \n\n{WHITE}For first time installations execute{WHITE}: \n\n {GREEN} python manage.py initdb\n\n'.format(GREEN=Fore.GREEN, YELLOW=Fore.YELLOW, WHITE=Fore.WHITE, RED=Fore.RED))

From 8759bb2bdf12aaaccbf74a37e25747dfdf26c7d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mat=C3=ADas=20Lang?= 
Date: Tue, 17 Jul 2018 12:48:38 -0300
Subject: [PATCH 1476/1506] Change default page size to 100 in the status
 report

---
 server/www/scripts/statusReport/controllers/statusReport.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/www/scripts/statusReport/controllers/statusReport.js b/server/www/scripts/statusReport/controllers/statusReport.js
index 4ebcf150bc8..8e1da89513a 100644
--- a/server/www/scripts/statusReport/controllers/statusReport.js
+++ b/server/www/scripts/statusReport/controllers/statusReport.js
@@ -38,7 +38,7 @@ angular.module("faradayApp")
         var searchFilter = {};
         var paginationOptions = {
             page: 1,
-            pageSize: 10,
+            pageSize: 100,
             defaultPageSizes: [10, 50, 75, 100],
             sortColumn: null,
             sortDirection: null

From 077f913268ca28b6727b113d5fa1fd3106b27e63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mat=C3=ADas=20Lang?= 
Date: Tue, 17 Jul 2018 14:31:16 -0300
Subject: [PATCH 1477/1506] Set session cookie expiration time to 12 hours

---
 server/app.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/server/app.py b/server/app.py
index 801e49af1cf..efe9ce1d6e4 100644
--- a/server/app.py
+++ b/server/app.py
@@ -5,6 +5,7 @@
 
 import os
 import string
+import datetime
 from os.path import join, expanduser
 from random import SystemRandom
 
@@ -181,6 +182,7 @@ def create_app(db_connection_string=None, testing=None):
         # 'sha512_crypt',
         'plaintext',  # TODO: remove it
     ]
+    app.config['PERMANENT_SESSION_LIFETIME'] = datetime.timedelta(hours=12)
     try:
         storage_path = server.config.storage.path
     except AttributeError:

From dcaf4634cbafb0c65ec6d538661327a561c904f4 Mon Sep 17 00:00:00 2001
From: Eric Horvat 
Date: Tue, 17 Jul 2018 15:25:51 -0300
Subject: [PATCH 1478/1506] [FIX] Distinct within count vulns

---
 server/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/models.py b/server/models.py
index 9606edc05df..0f6f6d5033a 100644
--- a/server/models.py
+++ b/server/models.py
@@ -1226,7 +1226,7 @@ def _make_vuln_count_property(type_=None, only_confirmed=False,
             isouter=True
         )
 
-    query = (select([func.count(text('vulnerability.id'))]).
+    query = (select([func.count(text('distinct(vulnerability.id)'))]).
              select_from(from_clause)
              )
     if get_hosts_vulns:

From 5ac71f6954c3de103294c5e89fcd17e455366dba Mon Sep 17 00:00:00 2001
From: Ezequiel Tavella 
Date: Tue, 17 Jul 2018 18:27:28 -0300
Subject: [PATCH 1479/1506] Fix issue with security headers in twisted server
 and iframes in webui

---
 server/web.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/web.py b/server/web.py
index 3ac73cad144..fd167c1bd90 100644
--- a/server/web.py
+++ b/server/web.py
@@ -53,8 +53,8 @@ def render(self, request):
         ret = super(FileWithoutDirectoryListing, self).render(request)
         if self.type == 'text/html':
             request.responseHeaders.addRawHeader('Content-Security-Policy',
-                                                 'frame-ancestors \'none\'')
-            request.responseHeaders.addRawHeader('X-Frame-Options', 'DENY')
+                                                 'frame-ancestors \'self\'')
+            request.responseHeaders.addRawHeader('X-Frame-Options', 'SAMEORIGIN')
         return ret
 
 

From 41b380b15b1f7ebaead7d09e0de5f4d15d5c6caa Mon Sep 17 00:00:00 2001
From: Leonardo Lazzaro 
Date: Wed, 18 Jul 2018 14:05:32 -0300
Subject: [PATCH 1480/1506] [ADD] closure compile to jenkins

---
 Jenkinsfile | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index b11fd00567b..90b1dc374e3 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -90,6 +90,20 @@ node (label: "master"){
 
         }
     }
+
+    stage ("Run Closure Compiler") {
+        try {
+            sh """
+                java -jar /home/faraday/closure-compiler-v20180610.jar $WORKSPACE/server/www/scripts
+            """
+        }
+        catch (err) {
+            currentBuild.result = 'FAILURE'
+        }
+        finally {
+            notifyBuild(currentBuild.result, "PostgreSQL Build")
+        }
+    }
 }
 
 def notifyBuild(String buildStatus = 'STARTED', String extraMessage = '') {

From 0809a183629dc493abc1ad6db999eaf4454c26fe Mon Sep 17 00:00:00 2001
From: Leonardo Lazzaro 
Date: Wed, 18 Jul 2018 16:54:44 -0300
Subject: [PATCH 1481/1506] [FIX] change build status string

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 90b1dc374e3..25d7bc82576 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -101,7 +101,7 @@ node (label: "master"){
             currentBuild.result = 'FAILURE'
         }
         finally {
-            notifyBuild(currentBuild.result, "PostgreSQL Build")
+            notifyBuild(currentBuild.result, "Closure compiler")
         }
     }
 }

From 35e6e93a20dca3ee648f98475056d3bc8b570cff Mon Sep 17 00:00:00 2001
From: Eric Horvat 
Date: Thu, 19 Jul 2018 12:08:17 -0300
Subject: [PATCH 1482/1506] [ADD] overflow for ws name in header

---
 server/www/header.css | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/www/header.css b/server/www/header.css
index 1fd74720b31..80cd9278db5 100644
--- a/server/www/header.css
+++ b/server/www/header.css
@@ -111,6 +111,7 @@
 
 .workspace-switcher-title .title{
     width: 90%;
+    overflow: hidden;
 }
 
 .workspace-switcher-title .caret-container{

From 76f173f163f6ce4c94e886d5efdd27c2c5a78c62 Mon Sep 17 00:00:00 2001
From: Leonardo Lazzaro 
Date: Fri, 20 Jul 2018 11:48:56 -0300
Subject: [PATCH 1483/1506] Change logger error to prints. manage is not
 showing logger errors

---
 server/importer.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/importer.py b/server/importer.py
index ec5b4ebf586..d831d53e9c0 100644
--- a/server/importer.py
+++ b/server/importer.py
@@ -1328,14 +1328,14 @@ def _open_couchdb_conn(self):
             workspaces_list = couchdb_server_conn.list_workspaces()
 
         except RequestError:
-            logger.error(u"CouchDB is not running at {}. Check faraday-server's"\
+            print(u"CouchDB is not running at {}. Check faraday-server's"\
                 " configuration and make sure CouchDB is running".format(
                 server.couchdb.get_couchdb_url()))
             logger.error(u'Please start CouchDB and re-execute the importer with: \n\n --> python manage.py import_from_couchdb <--')
             sys.exit(1)
 
         except Unauthorized:
-            logger.error(u"Unauthorized access to CouchDB. Make sure faraday-server's"\
+            print(u"Unauthorized access to CouchDB. Make sure faraday-server's"\
                 " configuration file has CouchDB admin's credentials set")
             sys.exit(1)
 

From f1418ce51ee27f34a1efd1383c1453236eebecd5 Mon Sep 17 00:00:00 2001
From: Leonardo Lazzaro 
Date: Fri, 20 Jul 2018 11:49:09 -0300
Subject: [PATCH 1484/1506] use tqdm for each workspace

---
 server/importer.py | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/server/importer.py b/server/importer.py
index d831d53e9c0..2a7f241e57a 100644
--- a/server/importer.py
+++ b/server/importer.py
@@ -1354,17 +1354,14 @@ def run(self):
         users_import.run()
 
         logger.info('Importing workspaces. Using {0} threads'.format(multiprocessing.cpu_count() * 2))
-        workspace_threads = []
-        with tqdm(total=len(workspaces_list) * 18,
-                  unit='B', unit_scale=True, unit_divisor=1024) as pbar:
-            for workspace_name in workspaces_list:
-                logger.debug(u'Setting up workspace {}'.format(workspace_name))
-
-                if not server.couchdb.server_has_access_to(workspace_name):
-                    logger.error(u"Unauthorized access to CouchDB. Make sure faraday-server's"\
-                                 " configuration file has CouchDB admin's credentials set")
-                    sys.exit(1)
-                self.import_workspace_into_database(workspace_name, pbar)
+        for workspace_name in tqdm(workspaces_list):
+            logger.debug(u'Setting up workspace {}'.format(workspace_name))
+
+            if not server.couchdb.server_has_access_to(workspace_name):
+                logger.error(u"Unauthorized access to CouchDB. Make sure faraday-server's"\
+                             " configuration file has CouchDB admin's credentials set")
+                sys.exit(1)
+            self.import_workspace_into_database(workspace_name)
 
     def get_objs(self, host, obj_type, level, workspace):
         if obj_type == 'Credential':
@@ -1452,7 +1449,7 @@ def import_level_objects(self, couch_url, faraday_importer, couchdb_relational_m
                     couchdb_relational_map_by_type[couchdb_id].append({'type': obj_type, 'id': new_obj.id})
                     couchdb_relational_map[couchdb_id].append(new_obj.id)
 
-    def import_workspace_into_database(self, workspace_name, pbar):
+    def import_workspace_into_database(self, workspace_name):
         with app.app_context():
 
             faraday_importer = FaradayEntityImporter(workspace_name)
@@ -1488,7 +1485,6 @@ def import_workspace_into_database(self, workspace_name, pbar):
                 except Exception as ex:
                     logger.exception(ex)
                     raise
-                pbar.update(1)
             update_command_tools(workspace, command_tool_map,
                                  couchdb_relational_map_by_type)
             session.commit()

From c25e77a3b0d586a783732fd7225ba99734e0d400 Mon Sep 17 00:00:00 2001
From: Leonardo Lazzaro 
Date: Fri, 20 Jul 2018 12:01:07 -0300
Subject: [PATCH 1485/1506] Move tqdm and print a message to let the user know
 about the progress

---
 server/importer.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/server/importer.py b/server/importer.py
index 2a7f241e57a..9e9530ab886 100644
--- a/server/importer.py
+++ b/server/importer.py
@@ -1354,7 +1354,7 @@ def run(self):
         users_import.run()
 
         logger.info('Importing workspaces. Using {0} threads'.format(multiprocessing.cpu_count() * 2))
-        for workspace_name in tqdm(workspaces_list):
+        for workspace_name in workspaces_list:
             logger.debug(u'Setting up workspace {}'.format(workspace_name))
 
             if not server.couchdb.server_has_access_to(workspace_name):
@@ -1430,7 +1430,8 @@ def verify_import_data(self, couchdb_relational_map, couchdb_relational_map_by_t
     def import_level_objects(self, couch_url, faraday_importer, couchdb_relational_map_by_type, couchdb_relational_map, command_tool_map, level, obj_type, workspace):
         obj_importer = faraday_importer.get_importer_from_document(obj_type)()
         objs_dict = self.get_objs(couch_url, obj_type, level, workspace)
-        for raw_obj in (objs_dict.get('rows', [])):
+        print('Importing {0} from workspace {1}'.format(obj_type, workspace.name))
+        for raw_obj in tqdm(objs_dict.get('rows', [])):
             # we use no_autoflush since some queries triggers flush and some relationship are missing in the middle
             with session.no_autoflush:
                 raw_obj = raw_obj['value']

From 4765ecd8b25994f00a9f0f7d535d869cd6b83df8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mat=C3=ADas=20Lang?= 
Date: Fri, 20 Jul 2018 13:07:54 -0300
Subject: [PATCH 1486/1506] Fix evidence download URLs

---
 server/www/scripts/statusReport/controllers/modalEdit.js      | 2 +-
 .../statusReport/partials/ui-grid/columns/evidencecolumn.html | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/server/www/scripts/statusReport/controllers/modalEdit.js b/server/www/scripts/statusReport/controllers/modalEdit.js
index f3d1b0386d0..b5a80f1e307 100644
--- a/server/www/scripts/statusReport/controllers/modalEdit.js
+++ b/server/www/scripts/statusReport/controllers/modalEdit.js
@@ -163,7 +163,7 @@ angular.module('faradayApp')
         vm.openEvidence = function(name) {
             var currentEvidence = vm.data._attachments[name];
             if (!currentEvidence.newfile)
-                window.open(vm.baseurl + $routeParams.wsId + '/' + vm.data._id + '/' + encodeURIComponent(name), '_blank');
+                window.open(vm.baseurl + '_api/v2/ws/' + $routeParams.wsId + '/vulns/' + vm.data._id + '/attachment/' + encodeURIComponent(name), '_blank');
         };
 
         vm.newPolicyViolation = function() {
diff --git a/server/www/scripts/statusReport/partials/ui-grid/columns/evidencecolumn.html b/server/www/scripts/statusReport/partials/ui-grid/columns/evidencecolumn.html
index 94a5951ec6c..774dae15339 100644
--- a/server/www/scripts/statusReport/partials/ui-grid/columns/evidencecolumn.html
+++ b/server/www/scripts/statusReport/partials/ui-grid/columns/evidencecolumn.html
@@ -1,4 +1,4 @@
 
    
-    {{key | decodeURIComponent}} 
+    {{key | decodeURIComponent}} 
 
    
-
    {{COL_FIELD CUSTOM_FILTERS}}
    
\ No newline at end of file
+
    {{COL_FIELD CUSTOM_FILTERS}}
    

From 34a96fc1587fbd92a93b656a95b8aba3987c5cdb Mon Sep 17 00:00:00 2001
From: Leonardo Lazzaro 
Date: Fri, 20 Jul 2018 14:16:55 -0300
Subject: [PATCH 1487/1506] add postgresql to install.sh

---
 install.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/install.sh b/install.sh
index baead324478..a2aee95a3b3 100755
--- a/install.sh
+++ b/install.sh
@@ -28,7 +28,7 @@ then
     apt-get update
 
     #Install community dependencies
-    for pkg in build-essential python-setuptools python-pip python-dev libpq-dev libffi-dev gir1.2-gtk-3.0 gir1.2-vte-2.91 python-gobject zsh curl python-psycopg2 ; do
+    for pkg in build-essential python-setuptools python-pip python-dev libpq-dev libffi-dev gir1.2-gtk-3.0 gir1.2-vte-2.91 python-gobject zsh curl python-psycopg2 postgresql; do
         apt-get install -y $pkg
     done
 fi

From ce1abaf68844ddf91c60f74437796b1e00fb75ff Mon Sep 17 00:00:00 2001
From: Ezequiel Tavella 
Date: Fri, 20 Jul 2018 16:25:17 -0300
Subject: [PATCH 1488/1506] Fix style in checkbox select all

---
 server/www/scripts/credentials/partials/list.html | 8 ++++++--
 server/www/scripts/vulndb/partials/vulndb.html    | 2 +-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/server/www/scripts/credentials/partials/list.html b/server/www/scripts/credentials/partials/list.html
index 2cd8b461af0..75270986f76 100644
--- a/server/www/scripts/credentials/partials/list.html
+++ b/server/www/scripts/credentials/partials/list.html
@@ -45,7 +45,9 @@
                 
-                            
+                            
@@ -66,7 +68,9 @@
                             selection-model-mode="multiple-additive"
                             selection-model-selected-class="multi-selected"
                             selection-model-on-change="selectedCredentials()">
-                            
+                            
diff --git a/server/www/scripts/vulndb/partials/vulndb.html b/server/www/scripts/vulndb/partials/vulndb.html
index 623b01c3bd0..305a45fa3f9 100644
--- a/server/www/scripts/vulndb/partials/vulndb.html
+++ b/server/www/scripts/vulndb/partials/vulndb.html
@@ -48,7 +48,7 @@
             
    
                     
                         
    
+                                
+                            
                             
                                 Target
                             
+                                
+                            
                              {{credential.target}} 
                             {{credential.name}}
                             {{credential.username}}
    
-                        
    
                 
                     
    
+                        
                           
                         
                         

From 48a27e35fd24b2358609020eba674ab1a26740e3 Mon Sep 17 00:00:00 2001
From: Leonardo Lazzaro 
Date: Fri, 20 Jul 2018 21:11:57 +0000
Subject: [PATCH 1489/1506] Update version

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 0964d6a2c11..6ffd5359c41 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-3.0rc1
+3.0rc2

From 899f3a850f7966936c6a4c0716f440b09d9dbc31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mat=C3=ADas=20Lang?= 
Date: Mon, 23 Jul 2018 13:43:17 -0300
Subject: [PATCH 1490/1506] Add missing imports to upload reports API

It was breaking the API endpoint
---
 server/api/modules/upload_reports.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/server/api/modules/upload_reports.py b/server/api/modules/upload_reports.py
index 61eb23c2537..0a18ac47236 100644
--- a/server/api/modules/upload_reports.py
+++ b/server/api/modules/upload_reports.py
@@ -16,6 +16,8 @@
 from flask import (
     request,
     abort,
+    make_response,
+    jsonify,
     Blueprint,
 )
 

From 7a1b1611e5002f1071989df657feb3e6bdc0e416 Mon Sep 17 00:00:00 2001
From: Ezequiel Tavella 
Date: Mon, 23 Jul 2018 14:41:57 -0300
Subject: [PATCH 1491/1506] Delete bad file cwe_all.csv

---
 data/cwe_all.csv | 2849 ----------------------------------------------
 1 file changed, 2849 deletions(-)
 delete mode 100644 data/cwe_all.csv

diff --git a/data/cwe_all.csv b/data/cwe_all.csv
deleted file mode 100644
index 672a83382d8..00000000000
--- a/data/cwe_all.csv
+++ /dev/null
@@ -1,2849 +0,0 @@
-cwe,name,description,resolution,exploitation,references
-CWE-119,EN-Improper Restriction of Operations within the Bounds of a Memory Buffer (Type: Class),"The software performs operations on a memory buffer, but it can read from or write to a memory location that is outside of the intended boundary of the buffer.
-Certain languages allow direct addressing of memory locations and do not automatically ensure that these locations are valid for the memory buffer that is being referenced. This can cause read or write operations to be performed on memory locations that may be associated with other variables, data structures, or internal program data.
-As a result, an attacker may be able to execute arbitrary code, alter the intended control flow, read sensitive information, or cause the system to crash.",,high,"Writing Secure Code: Chapter 5, ""Public Enemy #1: The Buffer Overrun"" Page 127; Chapter 14, ""Prevent I18N Buffer Overruns"" Page 441
-Using the Strsafe.h Functions: http://msdn.microsoft.com/en-us/library/ms647466.aspx
-Safe C String Library v1.0.3: http://www.zork.org/safestr/
-Address Space Layout Randomization in Windows Vista: http://blogs.msdn.com/michael_howard/archive/2006/05/26/address-space-layout-randomization-in-windows-vista.aspx
-Limiting buffer overflows with ExecShield: http://www.redhat.com/magazine/009jul05/features/execshield/
-PaX: http://en.wikipedia.org/wiki/PaX
-Understanding DEP as a mitigation technology part 1: http://blogs.technet.com/b/srd/archive/2009/06/12/understanding-dep-as-a-mitigation-technology-part-1.aspx
-The Art of Software Security Assessment: Chapter 5, ""Memory Corruption"", Page 167.
-The Art of Software Security Assessment: Chapter 5, ""Protection Mechanisms"", Page 189."
-CWE-123,EN-Write-what-where Condition (Type: Base),"Any condition where the attacker has the ability to write an arbitrary value to an arbitrary location, often as the result of a buffer overflow.
-A buffer overflow condition exists when a program attempts to put more data in a buffer than it can hold, or when a program attempts to put data in a memory area outside of the boundaries of a buffer. The simplest type of error, and the most common cause of buffer overflows, is the ""classic"" case in which the program copies the buffer without restricting how much is copied. Other variants exist, but the existence of a classic overflow strongly suggests that the programmer is not considering even the most basic of security protections.",,high,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89"
-CWE-129,EN-Improper Validation of Array Index (Type: Base),"The product uses untrusted input when calculating or using an array index, but the product does not validate or incorrectly validates the index to ensure the index references a valid position within the array.
-This typically occurs when the pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. This may result in exposure of sensitive information or possibly a crash.",,high,"Writing Secure Code: Chapter 5, ""Array Indexing Errors"" Page 144
-Top 25 Series - Rank 14 - Improper Validation of Array Index: http://blogs.sans.org/appsecstreetfighter/2010/03/12/top-25-series-rank-14-improper-validation-of-array-index/
-Address Space Layout Randomization in Windows Vista: http://blogs.msdn.com/michael_howard/archive/2006/05/26/address-space-layout-randomization-in-windows-vista.aspx
-PaX: http://en.wikipedia.org/wiki/PaX
-Understanding DEP as a mitigation technology part 1: http://blogs.technet.com/b/srd/archive/2009/06/12/understanding-dep-as-a-mitigation-technology-part-1.aspx
-Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html
-24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89"
-CWE-194,EN-Unexpected Sign Extension (Type: Base),"The software performs an operation on a number that causes it to be sign extended when it is transformed into a larger data type. When the original number is negative, this can produce unexpected values that lead to resultant weaknesses.
-This can happen in signed and unsigned cases.",,high,"C Language Issues for Application Security: http://www.informit.com/articles/article.aspx?p=686170&seqNum=6
-Integral Security: http://www.ddj.com/security/193501774"
-CWE-20,EN-Improper Input Validation (Type: Class),"The product does not validate or incorrectly validates input that can affect the control flow or data flow of a program.
-When software does not validate input properly, an attacker is able to craft the input in a form that is not expected by the rest of the application. This will lead to parts of the system receiving unintended input, which may result in altered control flow, arbitrary control of a resource, or arbitrary code execution.",,high,"Input Validation with ESAPI - Very Important: http://manicode.blogspot.com/2008/08/input-validation-with-esapi.html
-OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI
-Hacking Exposed Web Applications, Second Edition: Input Validation Attacks
-Input validation or output filtering, which is better?: http://jeremiahgrossman.blogspot.com/2007/01/input-validation-or-output-filtering.html
-The importance of input validation: http://searchsoftwarequality.techtarget.com/tip/0,289483,sid92_gci1214373,00.html
-Writing Secure Code: Chapter 10, ""All Input Is Evil!"" Page 341"
-CWE-200,EN-Information Exposure (Type: Class),"An information exposure is the intentional or unintentional disclosure of information to an actor that is not explicitly authorized to have access to that information.
-The information either
-is regarded as sensitive within the product's own functionality, such as a private message; or
-provides information about the product or its environment that could be useful in an attack but is normally not available to the attacker, such as the installation path of a product that is remotely accessible.
-Many information exposures are resultant (e.g. PHP script error revealing the full path of the program), but they can also be primary (e.g. timing discrepancies in cryptography). There are many different types of problems that involve information exposures. Their severity can range widely depending on the type of information that is revealed.",,high,Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/
-CWE-209,EN-Information Exposure Through an Error Message (Type: Base),"The software generates an error message that includes sensitive information about its environment, users, or associated data.
-The sensitive information may be valuable information on its own (such as a password), or it may be useful for launching other, more deadly attacks. If an attack fails, an attacker may use error information provided by the server to launch another more focused attack. For example, an attempt to exploit a path traversal weakness (CWE-22) might yield the full pathname of the installed application. In turn, this could be used to select the proper number of "".."" sequences to navigate to the targeted file. An attack using SQL injection (CWE-89) might not initially succeed, but an error message could reveal the malformed query, which would expose query logic and possibly even passwords or other sensitive information used within the query.",,high,"Information Leakage: http://www.webappsec.org/projects/threat/classes/information_leakage.shtml
-Secure Programming with Static Analysis: Section 9.2, page 326.
-Writing Secure Code: Chapter 16, ""General Good Practices."" Page 415
-24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183
-24 Deadly Sins of Software Security: ""Sin 12: Information Leakage."" Page 191
-Top 25 Series - Rank 16 - Information Exposure Through an Error Message: http://software-security.sans.org/blog/2010/03/17/top-25-series-rank-16-information-exposure-through-an-error-message
-The Art of Software Security Assessment: Chapter 3, ""Overly Verbose Error Messages"", Page 75."
-CWE-234,EN-Failure to Handle Missing Parameter (Type: Variant),"If too few arguments are sent to a function, the function will still pop the expected number of arguments from the stack. Potentially, a variable number of arguments could be exhausted in a function as well.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,high,
-CWE-242,EN-Use of Inherently Dangerous Function (Type: Base),"The program calls a function that can never be guaranteed to work safely.
-Certain functions behave in dangerous ways regardless of how they are used. Functions in this category were often implemented without taking security concerns into account. The gets() function is unsafe because it does not perform bounds checking on the size of its input. An attacker can easily send arbitrarily-sized input to gets() and overflow the destination buffer. Similarly, the >> operator is unsafe to use when reading into a statically-allocated character array because it does not perform bounds checking on the size of its input. An attacker can easily send arbitrarily-sized input to the >> operator and overflow the destination buffer.",,high,"Herb Schildt's C++ Programming Cookbook: Chapter 5. Working with I/O
-Writing Secure Code: Chapter 5, ""gets and fgets"" Page 163"
-CWE-243,EN-Creation of chroot Jail Without Changing Working Directory (Type: Variant),"The program uses the chroot() system call to create a jail, but does not change the working directory afterward. This does not prevent access to files outside of the jail.
-Improper use of chroot() may allow attackers to escape from the chroot jail. The chroot() function call does not change the process's current working directory, so relative paths may still refer to file system resources outside of the chroot jail after chroot() has been called.",,high,
-CWE-268,EN-Privilege Chaining (Type: Base),"Two distinct privileges, roles, capabilities, or rights can be combined in a way that allows an entity to perform unsafe actions that would not be allowed without that combination.
-Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,high,Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html
-CWE-271,EN-Privilege Dropping / Lowering Errors (Type: Class),"The software does not drop privileges before passing control of a resource to an actor that does not have those privileges.
-In some contexts, a system executing with elevated permissions will hand off a process/file/etc. to another process or user. If the privileges of an entity are not reduced, then elevated privileges are spread throughout a system and possibly to an attacker.",,high,"24 Deadly Sins of Software Security: ""Sin 16: Executing Code With Too Much Privilege."" Page 243
-The Art of Software Security Assessment: Chapter 9, ""Dropping Privileges Permanently"", Page 479."
-CWE-285,EN-Improper Authorization (Type: Class),"The software does not perform or incorrectly performs an authorization check when an actor attempts to access a resource or perform an action.
-Assuming a user with a given identity, authorization is the process of determining whether that user can access a given resource, based on the user's privileges and any permissions or other access-control specifications that apply to the resource.
-When access control checks are not applied consistently - or not at all - users are able to access data or perform actions that they should not be allowed to perform. This can lead to a wide range of problems, including information exposures, denial of service, and arbitrary code execution.",,high,"Role Based Access Control and Role Based Security: http://csrc.nist.gov/groups/SNS/rbac/
-Writing Secure Code: Chapter 4, ""Authorization"" Page 114; Chapter 6, ""Determining Appropriate Access Control"" Page 171
-Top 25 Series - Rank 5 - Improper Access Control (Authorization): http://blogs.sans.org/appsecstreetfighter/2010/03/04/top-25-series-rank-5-improper-access-control-authorization/
-OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI
-Authentication using JAAS: http://www.javaranch.com/journal/2008/04/authentication-using-JAAS.html
-The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Authorization"", Page 39.
-The Art of Software Security Assessment: Chapter 11, ""ACL Inheritance"", Page 649."
-CWE-291,EN-Reliance on IP Address for Authentication (Type: Variant),"The software uses an IP address for authentication.
-IP addresses can be easily spoofed. Attackers can forge the source IP address of the packets they send, but response packets will return to the forged IP address. To see the response packets, the attacker has to sniff the traffic between the victim machine and the forged IP address. In order to accomplish the required sniffing, attackers typically attempt to locate themselves on the same subnet as the victim machine. Attackers may be able to circumvent this requirement by using source routing, but source routing is disabled across much of the Internet today. In summary, IP address verification can be a useful part of an authentication scheme, but it should not be the single factor required for authentication.",,high,
-CWE-292,EN-DEPRECATED (Duplicate): Trusting Self-reported DNS Name (Type: Variant),"This entry has been deprecated because it was a duplicate of CWE-350. All content has been transferred to CWE-350.
-IP addresses can be easily spoofed. Attackers can forge the source IP address of the packets they send, but response packets will return to the forged IP address. To see the response packets, the attacker has to sniff the traffic between the victim machine and the forged IP address. In order to accomplish the required sniffing, attackers typically attempt to locate themselves on the same subnet as the victim machine. Attackers may be able to circumvent this requirement by using source routing, but source routing is disabled across much of the Internet today. In summary, IP address verification can be a useful part of an authentication scheme, but it should not be the single factor required for authentication.",,high,
-CWE-293,EN-Using Referer Field for Authentication (Type: Variant),"The referer field in HTTP requests can be easily modified and, as such, is not a valid means of message integrity checking.
-IP addresses can be easily spoofed. Attackers can forge the source IP address of the packets they send, but response packets will return to the forged IP address. To see the response packets, the attacker has to sniff the traffic between the victim machine and the forged IP address. In order to accomplish the required sniffing, attackers typically attempt to locate themselves on the same subnet as the victim machine. Attackers may be able to circumvent this requirement by using source routing, but source routing is disabled across much of the Internet today. In summary, IP address verification can be a useful part of an authentication scheme, but it should not be the single factor required for authentication.",,high,"The Art of Software Security Assessment: Chapter 17, ""Referer Request Header"", Page 1030."
-CWE-294,EN-Authentication Bypass by Capture-replay (Type: Base),"A capture-replay flaw exists when the design of the software makes it possible for a malicious user to sniff network traffic and bypass authentication by replaying it to the server in question to the same effect as the original message (or with minor changes).
-Capture-replay attacks are common and can be difficult to defeat without cryptography. They are a subset of network injection attacks that rely on observing previously-sent valid commands, then changing them slightly if necessary and resending the same commands to the server.",,high,
-CWE-297,EN-Improper Validation of Certificate with Host Mismatch (Type: Variant),"The software communicates with a host that provides a certificate, but the software does not properly ensure that the certificate is actually associated with that host.
-Even if a certificate is well-formed, signed, and follows the chain of trust, it may simply be a valid certificate for a different site than the site that the software is interacting with. If the certificate's host-specific data is not properly checked - such as the Common Name (CN) in the Subject or the Subject Alternative Name (SAN) extension of an X.509 certificate - it may be possible for a redirection or spoofing attack to allow a malicious host with a valid certificate to provide data, impersonating a trusted host. In order to ensure data integrity, the certificate must be valid and it must pertain to the site that is being accessed.
-Even if the software attempts to check the hostname, it is still possible to incorrectly check the hostname. For example, attackers could create a certificate with a name that begins with a trusted name followed by a NUL byte, which could cause some string-based comparisons to only examine the portion that contains the trusted name.",,high,"The Most Dangerous Code in the World: Validating SSL Certificates in Non-Browser Software: http://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
-Why Eve and Mallory Love Android: An Analysis of Android SSL (In)Security: http://www2.dcsec.uni-hannover.de/files/android/p50-fahl.pdf
-Secure programming with the OpenSSL API, Part 2: Secure handshake: http://www.ibm.com/developerworks/library/l-openssl2/index.html
-An Introduction to OpenSSL Programming (Part I): http://www.rtfm.com/openssl-examples/part1.pdf
-24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347"
-CWE-308,EN-Use of Single-factor Authentication (Type: Base),"The use of single-factor authentication can lead to unnecessary risk of compromise when compared with the benefits of a dual-factor authentication scheme.
-While the use of multiple authentication schemes is simply piling on more complexity on top of authentication, it is inestimably valuable to have such measures of redundancy. The use of weak, reused, and common passwords is rampant on the internet. Without the added protection of multiple authentication schemes, a single mistake can result in the compromise of an account. For this reason, if multiple schemes are possible and also easy to use, they should be implemented and required.",,high,
-CWE-321,EN-Use of Hard-coded Cryptographic Key (Type: Base),"The use of a hard-coded cryptographic key significantly increases the possibility that encrypted data may be recovered.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The '...' manipulation is useful for bypassing some path traversal protection schemes. On some Windows systems, it is equivalent to ""..\.."" and might bypass checks that assume only two dots are valid. Incomplete filtering, such as removal of ""./"" sequences, can ultimately produce valid "".."" sequences due to a collapse into unsafe value (CWE-182).",,high,
-CWE-322,EN-Key Exchange without Entity Authentication (Type: Base),"The software performs a key exchange with an actor without verifying the identity of that actor.
-Performing a key exchange will preserve the integrity of the information sent between two entities, but this will not guarantee that the entities are who they claim they are. This may enable a set of ""man-in-the-middle"" attacks. Typically, this involves a victim client that contacts a malicious server that is impersonating a trusted server. If the client skips authentication or ignores an authentication failure, the malicious server may request authentication information from the user. The malicious server can then use this authentication information to log in to the trusted server using the victim's credentials, sniff traffic between the victim and trusted server, etc.",,high,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347
-The Art of Software Security Assessment: Chapter 2, ""Untrustworthy Credentials"", Page 37."
-CWE-323,"EN-Reusing a Nonce, Key Pair in Encryption (Type: Base)","Nonces should be used for the present occasion and only once.
-Performing a key exchange will preserve the integrity of the information sent between two entities, but this will not guarantee that the entities are who they claim they are. This may enable a set of ""man-in-the-middle"" attacks. Typically, this involves a victim client that contacts a malicious server that is impersonating a trusted server. If the client skips authentication or ignores an authentication failure, the malicious server may request authentication information from the user. The malicious server can then use this authentication information to log in to the trusted server using the victim's credentials, sniff traffic between the victim and trusted server, etc.",,high,
-CWE-360,EN-Trust of System Event Data (Type: Base),"Security based on event locations are insecure and can be spoofed.
-Events are a messaging system which may provide control data to programs listening for events. Events often do not have any type of authentication framework to allow them to be verified from a trusted source. Any application, in Windows, on a given desktop can send a message to any window on the same desktop. There is no authentication framework for these messages. Therefore, any message can be used to manipulate any process on the desktop if the process does not check the validity and safeness of those messages.",,high,
-CWE-378,EN-Creation of Temporary File With Insecure Permissions (Type: Base),"Opening temporary files without appropriate measures or controls can leave the file, its contents and any function that it impacts vulnerable to attack.
-If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,high,
-CWE-416,EN-Use After Free (Type: Base),"Referencing memory after it has been freed can cause a program to crash, use unexpected values, or execute code.
-The use of previously-freed memory can have any number of adverse consequences, ranging from the corruption of valid data to the execution of arbitrary code, depending on the instantiation and timing of the flaw. The simplest way data corruption may occur involves the system's reuse of the freed memory. Use-after-free errors have two common and sometimes overlapping causes:
-Error conditions and other exceptional circumstances.
-Confusion over which part of the program is responsible for freeing the memory.
-In this scenario, the memory in question is allocated to another pointer validly at some point after it has been freed. The original pointer to the freed memory is used again and points to somewhere within the new allocation. As the data is changed, it corrupts the validly used memory; this induces undefined behavior in the process.
-If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,high,"24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143"
-CWE-457,EN-Use of Uninitialized Variable (Type: Variant),"The code uses a variable that has not been initialized, leading to unpredictable or unintended results.
-In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,high,"Exploiting Uninitialized Data: http://www.felinemenace.org/~mercy/papers/UBehavior/UBehavior.zip
-MS08-014 : The Case of the Uninitialized Stack Variable Vulnerability: http://blogs.technet.com/swi/archive/2008/03/11/the-case-of-the-uninitialized-stack-variable-vulnerability.aspx
-24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143
-The Art of Software Security Assessment: Chapter 7, ""Variable Initialization"", Page 312."
-CWE-467,EN-Use of sizeof() on a Pointer Type (Type: Variant),"The code calls sizeof() on a malloced pointer type, which always returns the wordsize/8. This can produce an unexpected result if the programmer intended to determine how much memory has been allocated.
-Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,high,EXP01-A. Do not take the sizeof a pointer to determine the size of a type: https://www.securecoding.cert.org/confluence/display/seccode/EXP01-A.+Do+not+take+the+sizeof+a+pointer+to+determine+the+size+of+a+type
-CWE-486,EN-Comparison of Classes by Name (Type: Variant),"The program compares classes by name, which can cause it to use the wrong class when multiple classes can have the same name.
-If the decision to trust the methods and data of an object is based on the name of a class, it is possible for malicious users to send objects of the same name as trusted classes and thereby gain the trust afforded to known classes and types.",,high,
-CWE-493,EN-Critical Public Variable Without Final Modifier (Type: Variant),"The product has a critical public variable that is not final, which allows the variable to be modified to contain unexpected values.
-If a field is non-final and public, it can be changed once the value is set by any function that has access to the class which contains the field. This could lead to a vulnerability if other parts of the program make assumptions about the contents of that field.",,high,
-CWE-499,EN-Serializable Class Containing Sensitive Data (Type: Variant),"The code contains a class with sensitive data, but the class does not explicitly deny serialization. The data can be accessed by serializing the class through another class.
-Serializable classes are effectively open classes since data cannot be hidden in them. Classes that do not explicitly deny serialization can be serialized by any other class, which can then in turn use the data stored inside it.",,high,
-CWE-500,EN-Public Static Field Not Marked Final (Type: Variant),"An object contains a public static field that is not marked final, which might allow it to be modified in unexpected ways.
-Public static variables can be read without an accessor and changed without a mutator by any classes in the application.",,high,
-CWE-515,EN-Covert Storage Channel (Type: Base),"A covert storage channel transfers information through the setting of bits by one program and the reading of those bits by another. What distinguishes this case from that of ordinary operation is that the bits are used to convey encoded information.
-Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,high,
-CWE-639,EN-Authorization Bypass Through User-Controlled Key (Type: Base),"The system's authorization functionality does not prevent one user from gaining access to another user's data or record by modifying the key value identifying the data.
-Retrieval of a user record occurs in the system based on some key value that is under user control. The key would typically identify a user related record stored in the system and would be used to lookup that record for presentation to the user. It is likely that an attacker would have to be an authenticated user in the system. However, the authorization process would not properly check the data access operation to ensure that the authenticated user performing the operation has sufficient entitlements to perform the requested data access, hence bypassing any other authorization checks present in the system. One manifestation of this weakness would be if a system used sequential or otherwise easily guessable session ids that would allow one user to easily switch to another user's session and read/modify their data.",,high,
-CWE-640,EN-Weak Password Recovery Mechanism for Forgotten Password (Type: Base),"The software contains a mechanism for users to recover or change their passwords without knowing the original password, but the mechanism is weak.
-It is common for an application to have a mechanism that provides a means for a user to gain access to their account in the event they forget their password. Very often the password recovery mechanism is weak, which has the effect of making it more likely that it would be possible for a person other than the legitimate system user to gain access to that user's account.
-This weakness may be that the security question is too easy to guess or find an answer to (e.g. because it is too common). Or there might be an implementation weakness in the password recovery mechanism code that may for instance trick the system into e-mailing the new password to an e-mail account other than that of the user. There might be no throttling done on the rate of password resets so that a legitimate user can be denied service by an attacker if an attacker tries to recover their password in a rapid succession. The system may send the original password to the user rather than generating a new temporary password. In summary, password recovery functionality, if not carefully designed and implemented can often become the system's weakest link that can be misused in a way that would allow an attacker to gain unauthorized access to the system. Weak password recovery schemes completely undermine a strong password authentication scheme.",,high,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279"
-CWE-642,EN-External Control of Critical State Data (Type: Class),"The software stores security-critical state information about its users, or the software itself, in a location that is accessible to unauthorized actors.
-If an attacker can modify the state information without detection, then it could be used to perform unauthorized actions or access unexpected resources, since the application programmer does not expect that the state can be changed.
-State information can be stored in various locations such as a cookie, in a hidden web form field, input parameter or argument, an environment variable, a database record, within a settings file, etc. All of these locations have the potential to be modified by an attacker. When this state information is used to control security or determine resource usage, then it may create a vulnerability. For example, an application may perform authentication, then save the state in an ""authenticated=true"" cookie. An attacker may simply create this cookie in order to bypass the authentication.",,high,"Top 10 2007-Insecure Direct Object Reference: http://www.owasp.org/index.php/Top_10_2007-A4
-HMAC: http://en.wikipedia.org/wiki/Hmac
-24 Deadly Sins of Software Security: ""Sin 4: Use of Magic URLs, Predictable Cookies, and Hidden Form Fields."" Page 75"
-CWE-643,EN-Improper Neutralization of Data within XPath Expressions (XPath Injection) (Type: Base),"The software uses external input to dynamically construct an XPath expression used to retrieve data from an XML database, but it does not neutralize or incorrectly neutralizes that input. This allows an attacker to control the structure of the query.
-The net effect is that the attacker will have control over the information selected from the XML database and may use that ability to control application flow, modify logic, retrieve unauthorized data, or bypass important checks (e.g. authentication).",,high,"XPath Injection: http://www.webappsec.org/projects/threat/classes/xpath_injection.shtml
-The Art of Software Security Assessment: Chapter 17, ""XPath Injection"", Page 1070."
-CWE-644,EN-Improper Neutralization of HTTP Headers for Scripting Syntax (Type: Variant),"The application does not neutralize or incorrectly neutralizes web scripting syntax in HTTP headers that can be used by web browser components that can process raw headers, such as Flash.
-An attacker may be able to conduct cross-site scripting and other attacks against users who have these components enabled.
-If an application does not neutralize user controlled data being placed in the header of an HTTP response coming from the server, the header may contain a script that will get executed in the client's browser context, potentially resulting in a cross site scripting vulnerability or possibly an HTTP response splitting attack. It is important to carefully control data that is being placed both in HTTP response header and in the HTTP response body to ensure that no scripting syntax is present, taking various encodings into account.",,high,
-CWE-645,EN-Overly Restrictive Account Lockout Mechanism (Type: Base),"The software contains an account lockout protection mechanism, but the mechanism is too restrictive and can be triggered too easily. This allows attackers to deny service to legitimate users by causing their accounts to be locked out.
-Account lockout is a security feature often present in applications as a countermeasure to the brute force attack on the password based authentication mechanism of the system. After a certain number of failed login attempts, the users' account may be disabled for a certain period of time or until it is unlocked by an administrator. Other security events may also possibly trigger account lockout. However, an attacker may use this very security feature to deny service to legitimate system users. It is therefore important to ensure that the account lockout security mechanism is not overly restrictive.",,high,
-CWE-646,EN-Reliance on File Name or Extension of Externally-Supplied File (Type: Variant),"The software allows a file to be uploaded, but it relies on the file name or extension of the file to determine the appropriate behaviors. This could be used by attackers to cause the file to be misclassified and processed in a dangerous fashion.
-An application might use the file name or extension of of a user-supplied file to determine the proper course of action, such as selecting the correct process to which control should be passed, deciding what data should be made available, or what resources should be allocated. If the attacker can cause the code to misclassify the supplied file, then the wrong action could occur. For example, an attacker could supply a file that ends in a "".php.gif"" extension that appears to be a GIF image, but would be processed as PHP code. In extreme cases, code execution is possible, but the attacker could also cause exhaustion of resources, denial of service, exposure of debug or system data (including application source code), or being bound to a particular server side process. This weakness may be due to a vulnerability in any of the technologies used by the web and application servers, due to misconfiguration, or resultant from another flaw in the application itself.",,high,
-CWE-647,EN-Use of Non-Canonical URL Paths for Authorization Decisions (Type: Variant),"The software defines policy namespaces and makes authorization decisions based on the assumption that a URL is canonical. This can allow a non-canonical URL to bypass the authorization.
-If an application defines policy namespaces and makes authorization decisions based on the URL, but it does not require or convert to a canonical URL before making the authorization decision, then it opens the application to attack. For example, if the application only wants to allow access to http://www.example.com/mypage, then the attacker might be able to bypass this restriction using equivalent URLs such as:
-http://WWW.EXAMPLE.COM/mypage
-http://www.example.com/%6Dypage (alternate encoding)
-http://192.168.1.1/mypage (IP address)
-http://www.example.com/mypage/ (trailing /)
-http://www.example.com:80/mypage
-Therefore it is important to specify access control policy that is based on the path information in some canonical form with all alternate encodings rejected (which can be accomplished by a default deny rule).",,high,
-CWE-649,EN-Reliance on Obfuscation or Encryption of Security-Relevant Inputs without Integrity Checking (Type: Base),"The software uses obfuscation or encryption of inputs that should not be mutable by an external actor, but the software does not use integrity checks to detect if those inputs have been modified.
-When an application relies on obfuscation or incorrectly applied / weak encryption to protect client-controllable tokens or parameters, that may have an effect on the user state, system state, or some decision made on the server. Without protecting the tokens/parameters for integrity, the application is vulnerable to an attack where an adversary blindly traverses the space of possible values of the said token/parameter in order to attempt to gain an advantage. The goal of the attacker is to find another admissible value that will somehow elevate his or her privileges in the system, disclose information or change the behavior of the system in some way beneficial to the attacker. If the application does not protect these critical tokens/parameters for integrity, it will not be able to determine that these values have been tampered with. Measures that are used to protect data for confidentiality should not be relied upon to provide the integrity service.",,high,
-CWE-650,EN-Trusting HTTP Permission Methods on the Server Side (Type: Variant),"The server contains a protection mechanism that assumes that any URI that is accessed using HTTP GET will not cause a state change to the associated resource. This might allow attackers to bypass intended access restrictions and conduct resource modification and deletion attacks, since some applications allow GET to modify state.
-An application may disallow the HTTP requests to perform DELETE, PUT and POST operations on the resource representation, believing that it will be enough to prevent unintended resource alterations. Even though the HTTP GET specification requires that GET requests should not have side effects, there is nothing in the HTTP protocol itself that prevents the HTTP GET method from performing more than just query of the data. For instance, it is a common practice with REST based Web Services to have HTTP GET requests modifying resources on the server side. Whenever that happens however, the access control needs to be properly enforced in the application. No assumptions should be made that only HTTP DELETE, PUT, and POST methods have the power to alter the representation of the resource being accessed in the request.",,high,
-CWE-652,EN-Improper Neutralization of Data within XQuery Expressions (XQuery Injection) (Type: Base),"The software uses external input to dynamically construct an XQuery expression used to retrieve data from an XML database, but it does not neutralize or incorrectly neutralizes that input. This allows an attacker to control the structure of the query.
-The net effect is that the attacker will have control over the information selected from the XML database and may use that ability to control application flow, modify logic, retrieve unauthorized data, or bypass important checks (e.g. authentication).",,high,
-CWE-676,EN-Use of Potentially Dangerous Function (Type: Base),"The program invokes a potentially dangerous function that could introduce a vulnerability if it is used incorrectly, but the function can also be used safely.
-Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,high,"Security Development Lifecycle (SDL) Banned Function Calls: http://msdn.microsoft.com/en-us/library/bb288454.aspx
-Writing Secure Code: Chapter 5, ""Safe String Handling"" Page 156, 160
-The Art of Software Security Assessment: Chapter 8, ""C String Handling"", Page 388."
-CWE-682,EN-Incorrect Calculation (Type: Class),"The software performs a calculation that generates incorrect or unintended results that are later used in security-critical decisions or resource management..
-When software performs a security-critical calculation incorrectly, it might lead to incorrect resource allocations, incorrect privilege assignments, or failed comparisons among other things. Many of the direct results of an incorrect calculation can lead to even larger problems such as failed protection mechanisms or even arbitrary code execution.",,high,"SafeInt: http://safeint.codeplex.com/
-24 Deadly Sins of Software Security: ""Sin 7: Integer Overflows."" Page 119
-The Art of Software Security Assessment: Chapter 6, ""Signed Integer Boundaries"", Page 220."
-CWE-78,EN-Improper Neutralization of Special Elements used in an OS Command (OS Command Injection) (Type: Base),"The software constructs all or part of an OS command using externally-influenced input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could modify the intended OS command when it is sent to a downstream component..
-This could allow attackers to execute unexpected, dangerous commands directly on the operating system. This weakness can lead to a vulnerability in environments in which the attacker does not have direct access to the operating system, such as in web applications. Alternately, if the weakness occurs in a privileged program, it could allow the attacker to specify commands that normally would not be accessible, or to call alternate commands with privileges that the attacker does not have. The problem is exacerbated if the compromised process does not follow the principle of least privilege, because the attacker-controlled commands may run with special system privileges that increases the amount of damage.
-There are at least two subtypes of OS command injection:
-The application intends to execute a single, fixed program that is under its own control. It intends to use externally-supplied inputs as arguments to that program. For example, the program might use system(""nslookup [HOSTNAME]"") to run nslookup and allow the user to supply a HOSTNAME, which is used as an argument. Attackers cannot prevent nslookup from executing. However, if the program does not remove command separators from the HOSTNAME argument, attackers could place the separators into the arguments, which allows them to execute their own program after nslookup has finished executing.
-The application accepts an input that it uses to fully select which program to run, as well as which commands to use. The application simply redirects this entire command to the operating system. For example, the program might use ""exec([COMMAND])"" to execute the [COMMAND] that was supplied by the user. If the COMMAND is under attacker control, then the attacker can execute arbitrary commands or programs. If the command is being executed using functions like exec() and CreateProcess(), the attacker might not be able to combine multiple commands together in the same line.
-From a weakness standpoint, these variants represent distinct programmer errors. In the first variant, the programmer clearly intends that input from untrusted parties will be part of the arguments in the command to be executed. In the second variant, the programmer does not intend for the command to be accessible to any untrusted party, but the programmer probably has not accounted for alternate ways in which malicious attackers can provide input.",,high,"Exploiting Software: How to Break Code
-Meta-Character Vulnerabilities: http://www.cs.purdue.edu/homes/cs390s/slides/week09.pdf
-OS Commanding: http://projects.webappsec.org/OS-Commanding
-The World Wide Web Security FAQ: http://www.w3.org/Security/Faq/wwwsf4.html
-Security Issues in Perl Scripts: http://www.cgisecurity.com/lib/sips.html
-24 Deadly Sins of Software Security: ""Sin 10: Command Injection."" Page 171
-Top 25 Series - Rank 9 - OS Command Injection: http://blogs.sans.org/appsecstreetfighter/2010/02/24/top-25-series-rank-9-os-command-injection/
-OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI
-Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html
-The Art of Software Security Assessment: Chapter 8, ""Shell Metacharacters"", Page 425."
-CWE-784,EN-Reliance on Cookies without Validation and Integrity Checking in a Security Decision (Type: Variant),"The application uses a protection mechanism that relies on the existence or values of a cookie, but it does not properly ensure that the cookie is valid for the associated user.
-Attackers can easily modify cookies, within the browser or by implementing the client-side code outside of the browser. Attackers can bypass protection mechanisms such as authorization and authentication by modifying the cookie to contain an expected value.",,high,"Unforgivable Vulnerabilities: http://cve.mitre.org/docs/docs-2007/unforgivable.pdf
-Writing Secure Code: Chapter 13, ""Sensitive Data in Cookies and Fields"" Page 435"
-CWE-862,EN-Missing Authorization (Type: Class),"The software does not perform an authorization check when an actor attempts to access a resource or perform an action.
-Assuming a user with a given identity, authorization is the process of determining whether that user can access a given resource, based on the user's privileges and any permissions or other access-control specifications that apply to the resource.
-When access control checks are not applied, users are able to access data or perform actions that they should not be allowed to perform. This can lead to a wide range of problems, including information exposures, denial of service, and arbitrary code execution.",,high,"Role Based Access Control and Role Based Security: http://csrc.nist.gov/groups/SNS/rbac/
-Writing Secure Code: Chapter 4, ""Authorization"" Page 114; Chapter 6, ""Determining Appropriate Access Control"" Page 171
-Top 25 Series - Rank 5 - Improper Access Control (Authorization): http://blogs.sans.org/appsecstreetfighter/2010/03/04/top-25-series-rank-5-improper-access-control-authorization/
-OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI
-Authentication using JAAS: http://www.javaranch.com/journal/2008/04/authentication-using-JAAS.html
-The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Authorization"", Page 39."
-CWE-863,EN-Incorrect Authorization (Type: Class),"The software performs an authorization check when an actor attempts to access a resource or perform an action, but it does not correctly perform the check. This allows attackers to bypass intended access restrictions.
-Assuming a user with a given identity, authorization is the process of determining whether that user can access a given resource, based on the user's privileges and any permissions or other access-control specifications that apply to the resource.
-When access control checks are incorrectly applied, users are able to access data or perform actions that they should not be allowed to perform. This can lead to a wide range of problems, including information exposures, denial of service, and arbitrary code execution.",,high,"Role Based Access Control and Role Based Security: http://csrc.nist.gov/groups/SNS/rbac/
-Writing Secure Code: Chapter 4, ""Authorization"" Page 114; Chapter 6, ""Determining Appropriate Access Control"" Page 171
-Top 25 Series - Rank 5 - Improper Access Control (Authorization): http://blogs.sans.org/appsecstreetfighter/2010/03/04/top-25-series-rank-5-improper-access-control-authorization/
-Authentication using JAAS: http://www.javaranch.com/journal/2008/04/authentication-using-JAAS.html
-OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI
-The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Authorization"", Page 39."
-CWE-99,EN-Improper Control of Resource Identifiers (Resource Injection) (Type: Base),"The software receives input from an upstream component, but it does not restrict or incorrectly restricts the input before it is used as an identifier for a resource that may be outside the intended sphere of control.
-This may enable an attacker to access or modify otherwise protected system resources.",,high,
-CWE-120,EN-Buffer Copy without Checking Size of Input (Classic Buffer Overflow) (Type: Base),"The program copies an input buffer to an output buffer without verifying that the size of the input buffer is less than the size of the output buffer, leading to a buffer overflow.
-A buffer overflow condition exists when a program attempts to put more data in a buffer than it can hold, or when a program attempts to put data in a memory area outside of the boundaries of a buffer. The simplest type of error, and the most common cause of buffer overflows, is the ""classic"" case in which the program copies the buffer without restricting how much is copied. Other variants exist, but the existence of a classic overflow strongly suggests that the programmer is not considering even the most basic of security protections.",,high,"Writing Secure Code: Chapter 5, ""Public Enemy #1: The Buffer Overrun"" Page 127
-24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89
-Using the Strsafe.h Functions: http://msdn.microsoft.com/en-us/library/ms647466.aspx
-Safe C String Library v1.0.3: http://www.zork.org/safestr/
-Address Space Layout Randomization in Windows Vista: http://blogs.msdn.com/michael_howard/archive/2006/05/26/address-space-layout-randomization-in-windows-vista.aspx
-Limiting buffer overflows with ExecShield: http://www.redhat.com/magazine/009jul05/features/execshield/
-PaX: http://en.wikipedia.org/wiki/PaX
-Top 25 Series - Rank 3 - Classic Buffer Overflow: http://software-security.sans.org/blog/2010/03/02/top-25-series-rank-3-classic-buffer-overflow/
-Understanding DEP as a mitigation technology part 1: http://blogs.technet.com/b/srd/archive/2009/06/12/understanding-dep-as-a-mitigation-technology-part-1.aspx
-Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html
-The Art of Software Security Assessment: Chapter 3, ""Nonexecutable Stack"", Page 76.
-The Art of Software Security Assessment: Chapter 5, ""Protection Mechanisms"", Page 189.
-The Art of Software Security Assessment: Chapter 8, ""C String Handling"", Page 388."
-CWE-122,EN-Heap-based Buffer Overflow (Type: Variant),"A heap overflow condition is a buffer overflow, where the buffer that can be overwritten is allocated in the heap portion of memory, generally meaning that the buffer was allocated using a routine such as malloc().
-A buffer overflow condition exists when a program attempts to put more data in a buffer than it can hold, or when a program attempts to put data in a memory area outside of the boundaries of a buffer. The simplest type of error, and the most common cause of buffer overflows, is the ""classic"" case in which the program copies the buffer without restricting how much is copied. Other variants exist, but the existence of a classic overflow strongly suggests that the programmer is not considering even the most basic of security protections.",,high,"Writing Secure Code: Chapter 5, ""Heap Overruns"" Page 138
-24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89
-The Art of Software Security Assessment: Chapter 3, ""Nonexecutable Stack"", Page 76.
-The Art of Software Security Assessment: Chapter 5, ""Protection Mechanisms"", Page 189."
-CWE-131,EN-Incorrect Calculation of Buffer Size (Type: Base),"The software does not correctly calculate the size to be used when allocating a buffer, which could lead to a buffer overflow.
-If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,high,"SafeInt: http://safeint.codeplex.com/
-Top 25 Series - Rank 18 - Incorrect Calculation of Buffer Size: http://software-security.sans.org/blog/2010/03/19/top-25-series-rank-18-incorrect-calculation-of-buffer-size
-Address Space Layout Randomization in Windows Vista: http://blogs.msdn.com/michael_howard/archive/2006/05/26/address-space-layout-randomization-in-windows-vista.aspx
-Understanding DEP as a mitigation technology part 1: http://blogs.technet.com/b/srd/archive/2009/06/12/understanding-dep-as-a-mitigation-technology-part-1.aspx
-PaX: http://en.wikipedia.org/wiki/PaX
-Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html
-Writing Secure Code: Chapter 20, ""Integer Overflows"" Page 620
-24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89
-The Art of Software Security Assessment: Chapter 8, ""Incrementing Pointers Incorrectly"", Page 401."
-CWE-22,EN-Improper Limitation of a Pathname to a Restricted Directory (Path Traversal) (Type: Class),"The software uses external input to construct a pathname that is intended to identify a file or directory that is located underneath a restricted parent directory, but the software does not properly neutralize special elements within the pathname that can cause the pathname to resolve to a location that is outside of the restricted directory.
-Many file operations are intended to take place within a restricted directory. By using special elements such as "".."" and ""/"" separators, attackers can escape outside of the restricted location to access files or directories that are elsewhere on the system. One of the most common special elements is the ""../"" sequence, which in most modern operating systems is interpreted as the parent directory of the current location. This is referred to as relative path traversal. Path traversal also covers the use of absolute pathnames such as ""/usr/local/bin"", which may also be useful in accessing unexpected files. This is referred to as absolute path traversal.
-In many programming languages, the injection of a null byte (the 0 or NUL) may allow an attacker to truncate a generated filename to widen the scope of attack. For example, the software may add "".txt"" to any pathname, thus limiting the attacker to text files, but a null injection may effectively remove this restriction.",,high,"Writing Secure Code: Chapter 11, ""Directory Traversal and Using Parent Paths (..)"" Page 370
-OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI
-Testing for Path Traversal (OWASP-AZ-001): http://www.owasp.org/index.php/Testing_for_Path_Traversal_(OWASP-AZ-001)
-Top 25 Series - Rank 7 - Path Traversal: http://blogs.sans.org/appsecstreetfighter/2010/03/09/top-25-series-rank-7-path-traversal/
-Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html
-The Art of Software Security Assessment: Chapter 9, ""Filenames and Paths"", Page 503."
-CWE-311,EN-Missing Encryption of Sensitive Data (Type: Base),"The software does not encrypt sensitive or critical information before storage or transmission.
-The lack of proper data encryption passes up the guarantees of confidentiality, integrity, and accountability that properly implemented encryption conveys.",,high,"Writing Secure Code: Chapter 9, ""Protecting Secret Data"" Page 299
-24 Deadly Sins of Software Security: ""Sin 17: Failure to Protect Stored Data."" Page 253
-Top 25 Series - Rank 10 - Missing Encryption of Sensitive Data: http://blogs.sans.org/appsecstreetfighter/2010/02/26/top-25-series-rank-10-missing-encryption-of-sensitive-data/
-The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Encryption"", Page 43.
-SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf"
-CWE-464,EN-Addition of Data Structure Sentinel (Type: Base),"The accidental addition of a data-structure sentinel can cause serious programming logic problems.
-Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,high,
-CWE-67,EN-Improper Handling of Windows Device Names (Type: Variant),"The software constructs pathnames from user input, but it does not handle or incorrectly handles a pathname containing a Windows device name such as AUX or CON. This typically leads to denial of service or an information exposure when the application attempts to process the pathname as a regular file.
-Not properly handling virtual filenames (e.g. AUX, CON, PRN, COM1, LPT1) can result in different types of vulnerabilities. In some cases an attacker can request a device via injection of a virtual filename in a URL, which may cause an error that leads to a denial of service or an error page that reveals sensitive information. A software system that allows device names to bypass filtering runs the risk of an attacker injecting malicious code in a file with the name of a device.",,high,"Writing Secure Code
-The Art of Software Security Assessment: Chapter 11, ""Device Files"", Page 666."
-CWE-73,EN-External Control of File Name or Path (Type: Class),"The software allows user input to control or influence paths or file names that are used in filesystem operations.
-This could allow an attacker to access or modify system files or other files that are critical to the application.
-Path manipulation errors occur when the following two conditions are met:
-1. An attacker can specify a path used in an operation on the filesystem.
-2. By specifying the resource, the attacker gains a capability that would not otherwise be permitted.
-For example, the program may give the attacker the ability to overwrite the specified file or run with a configuration controlled by the attacker.",,high,OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI
-CWE-76,EN-Improper Neutralization of Equivalent Special Elements (Type: Base),"The software properly neutralizes certain special elements, but it improperly neutralizes equivalent special elements.
-The software may have a fixed list of special characters it believes is complete. However, there may be alternate encodings, or representations that also have the same meaning. For example, the software may filter out a leading slash (/) to prevent absolute path names, but does not account for a tilde (~) followed by a user name, which on some *nix systems could be expanded to an absolute pathname. Alternately, the software might filter a dangerous ""-e"" command-line switch when calling an external program, but it might not account for ""--exec"" or other switches that have the same semantics.",,high,
-CWE-79,EN-Improper Neutralization of Input During Web Page Generation (Cross-site Scripting) (Type: Base),"The software does not neutralize or incorrectly neutralizes user-controllable input before it is placed in output that is used as a web page that is served to other users.
-Cross-site scripting (XSS) vulnerabilities occur when:
-1. Untrusted data enters a web application, typically from a web request.
-2. The web application dynamically generates a web page that contains this untrusted data.
-3. During page generation, the application does not prevent the data from containing content that is executable by a web browser, such as JavaScript, HTML tags, HTML attributes, mouse events, Flash, ActiveX, etc.
-4. A victim visits the generated web page through a web browser, which contains malicious script that was injected using the untrusted data.
-5. Since the script comes from a web page that was sent by the web server, the victim's web browser executes the malicious script in the context of the web server's domain.
-6. This effectively violates the intention of the web browser's same-origin policy, which states that scripts in one domain should not be able to access resources or run code in a different domain.
-There are three main kinds of XSS:
-The server reads data directly from the HTTP request and reflects it back in the HTTP response. Reflected XSS exploits occur when an attacker causes a victim to supply dangerous content to a vulnerable web application, which is then reflected back to the victim and executed by the web browser. The most common mechanism for delivering malicious content is to include it as a parameter in a URL that is posted publicly or e-mailed directly to the victim. URLs constructed in this manner constitute the core of many phishing schemes, whereby an attacker convinces a victim to visit a URL that refers to a vulnerable site. After the site reflects the attacker's content back to the victim, the content is executed by the victim's browser.
-The application stores dangerous data in a database, message forum, visitor log, or other trusted data store. At a later time, the dangerous data is subsequently read back into the application and included in dynamic content. From an attacker's perspective, the optimal place to inject malicious content is in an area that is displayed to either many users or particularly interesting users. Interesting users typically have elevated privileges in the application or interact with sensitive data that is valuable to the attacker. If one of these users executes malicious content, the attacker may be able to perform privileged operations on behalf of the user or gain access to sensitive data belonging to the user. For example, the attacker might inject XSS into a log message, which might not be handled properly when an administrator views the logs.
-In DOM-based XSS, the client performs the injection of XSS into the page; in the other types, the server performs the injection. DOM-based XSS generally involves server-controlled, trusted script that is sent to the client, such as Javascript that performs sanity checks on a form before the user submits it. If the server-supplied script processes user-supplied data and then injects it back into the web page (such as with dynamic HTML), then DOM-based XSS is possible.
-Once the malicious script is injected, the attacker can perform a variety of malicious activities. The attacker could transfer private information, such as cookies that may include session information, from the victim's machine to the attacker. The attacker could send malicious requests to a web site on behalf of the victim, which could be especially dangerous to the site if the victim has administrator privileges to manage that site. Phishing attacks could be used to emulate trusted web sites and trick the victim into entering a password, allowing the attacker to compromise the victim's account on that web site. Finally, the script could exploit a vulnerability in the web browser itself possibly taking over the victim's machine, sometimes referred to as ""drive-by hacking.""
-In many cases, the attack can be launched without the victim even being aware of it. Even with careful users, attackers frequently use a variety of methods to encode the malicious portion of the attack, such as URL encoding or Unicode, so the request looks less suspicious.",,high,"XSS Attacks
-24 Deadly Sins of Software Security: ""Sin 2: Web-Server Related Vulnerabilities (XSS, XSRF, and Response Splitting)."" Page 31
-24 Deadly Sins of Software Security: ""Sin 3: Web-Client Related Vulnerabilities (XSS)."" Page 63
-Cross-site scripting: http://en.wikipedia.org/wiki/Cross-site_scripting
-Writing Secure Code: Chapter 13, ""Web-Specific Input Issues"" Page 413
-XSS (Cross Site Scripting) Cheat Sheet: http://ha.ckers.org/xss.html
-Mitigating Cross-site Scripting With HTTP-only Cookies: http://msdn.microsoft.com/en-us/library/ms533046.aspx
-Anti-XSS 3.0 Beta and CAT.NET Community Technology Preview now Live!: http://blogs.msdn.com/cisg/archive/2008/12/15/anti-xss-3-0-beta-and-cat-net-community-technology-preview-now-live.aspx
-OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI
-XSS Defense HOWTO: http://blog.modsecurity.org/2008/07/do-you-know-how.html
-Web Application Firewall: http://www.owasp.org/index.php/Web_Application_Firewall
-Web Application Firewall Evaluation Criteria: http://www.webappsec.org/projects/wafec/v1/wasc-wafec-v1.0.html
-Firefox Implements httpOnly And is Vulnerable to XMLHTTPRequest
-XMLHttpRequest allows reading HTTPOnly cookies: https://bugzilla.mozilla.org/show_bug.cgi?id=380418
-Apache Wicket: http://wicket.apache.org/
-XSS (Cross Site Scripting) Prevention Cheat Sheet: http://www.owasp.org/index.php/XSS_(Cross_Site_Scripting)_Prevention_Cheat_Sheet
-DOM based XSS Prevention Cheat Sheet: http://www.owasp.org/index.php/DOM_based_XSS_Prevention_Cheat_Sheet
-Top 25 series - Rank 1 - Cross Site Scripting: http://blogs.sans.org/appsecstreetfighter/2010/02/22/top-25-series-rank-1-cross-site-scripting/
-The Art of Software Security Assessment: Chapter 17, ""Cross Site Scripting"", Page 1071."
-CWE-80,EN-Improper Neutralization of Script-Related HTML Tags in a Web Page (Basic XSS) (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special characters such as ""<"", "">"", and ""&"" that could be interpreted as web-scripting elements when they are sent to a downstream component that processes web pages.
-This may allow such characters to be treated as control characters, which are executed client-side in the context of the user's session. Although this can be classified as an injection problem, the more pertinent issue is the improper conversion of such special characters to respective context-appropriate entities before displaying them to the user.",,high,
-CWE-98,EN-Improper Control of Filename for Include/Require Statement in PHP Program (PHP Remote File Inclusion) (Type: Base),"The PHP application receives input from an upstream component, but it does not restrict or incorrectly restricts the input before its usage in ""require,"" ""include,"" or similar functions.
-In certain versions and configurations of PHP, this can allow an attacker to specify a URL to a remote location from which the software will obtain the code to execute. In other cases in association with path traversal, the attacker can specify a local file that may contain executable statements that can be parsed by PHP.",,high,"Testing for Path Traversal (OWASP-AZ-001): http://www.owasp.org/index.php/Testing_for_Path_Traversal_(OWASP-AZ-001)
-Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html
-A Study in Scarlet: http://www.cgisecurity.com/lib/studyinscarlet.txt
-Suhosin: http://www.hardened-php.net/suhosin/
-Top 25 Series - Rank 13 - PHP File Inclusion: http://blogs.sans.org/appsecstreetfighter/2010/03/11/top-25-series-rank-13-php-file-inclusion/"
-CWE-188,EN-Reliance on Data/Memory Layout (Type: Base),"The software makes invalid assumptions about how protocol data or memory is organized at a lower level, resulting in unintended program behavior.
-For example, an attacker might succeed in authentication by providing a small password that matches the associated portion of the larger, correct password.",,low,"The Art of Software Security Assessment: Chapter 6, ""Structure Padding"", Page 284."
-CWE-197,EN-Numeric Truncation Error (Type: Base),"Truncation errors occur when a primitive is cast to a primitive of a smaller size and data is lost in the conversion.
-When a primitive is cast to a smaller primitive, the high order bits of the large value are lost in the conversion, potentially resulting in an unexpected value that is not equal to the original value. This value may be required as an index into a buffer, a loop iterator, or simply necessary state data. In any case, the value cannot be trusted and the system will be in an undefined state. While this method may be employed viably to isolate the low bits of a value, this usage is rare, and truncation usually implies that an implementation error has occurred.",,low,"The Art of Software Security Assessment: Chapter 6, ""Truncation"", Page 259."
-CWE-252,EN-Unchecked Return Value (Type: Base),"The software does not check the return value from a method or function, which can prevent it from detecting unexpected states and conditions.
-Two common programmer assumptions are ""this function call can never fail"" and ""it doesn't matter if this function call fails"". If an attacker can force the function to fail or otherwise return a value that is not expected, then the subsequent program logic could lead to a vulnerability, because the software is not in a state that the programmer assumes. For example, if the program calls a function to drop privileges but does not check the return code to ensure that privileges were successfully dropped, then the program will continue to operate with the higher privileges.",,low,"The Art of Software Security Assessment: Chapter 7, ""Program Building Blocks"" Page 341.
-Writing Secure Code: Chapter 20, ""Checking Returns"" Page 624
-24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183
-ERR10-CPP. Check for error conditions: https://www.securecoding.cert.org/confluence/display/cplusplus/ERR10-CPP.+Check+for+error+conditions"
-CWE-253,EN-Incorrect Check of Function Return Value (Type: Base),"The software incorrectly checks a return value from a function, which prevents the software from detecting errors or exceptional conditions.
-Two common programmer assumptions are ""this function call can never fail"" and ""it doesn't matter if this function call fails"". If an attacker can force the function to fail or otherwise return a value that is not expected, then the subsequent program logic could lead to a vulnerability, because the software is not in a state that the programmer assumes. For example, if the program calls a function to drop privileges but does not check the return code to ensure that privileges were successfully dropped, then the program will continue to operate with the higher privileges.",,low,"The Art of Software Security Assessment: Chapter 7, ""Return Value Testing and Interpretation"", Page 340."
-CWE-296,EN-Improper Following of a Certificates Chain of Trust (Type: Base),"The software does not follow, or incorrectly follows, the chain of trust for a certificate back to a trusted root certificate, resulting in incorrect trust of any resource that is associated with that certificate.
-If a system does not follow the chain of trust of a certificate to a root server, the certificate loses all usefulness as a metric of trust. Essentially, the trust gained from a certificate is derived from a chain of trust -- with a reputable trusted entity at the end of that list. The end user must trust that reputable source, and this reputable source must vouch for the resource in question through the medium of the certificate.
-In some cases, this trust traverses several entities who vouch for one another. The entity trusted by the end user is at one end of this trust chain, while the certificate-wielding resource is at the other end of the chain. If the user receives a certificate at the end of one of these trust chains and then proceeds to check only that the first link in the chain, no real trust has been derived, since the entire chain must be traversed back to a trusted source to verify the certificate.
-There are several ways in which the chain of trust might be broken, including but not limited to:
-Any certificate in the chain is self-signed, unless it the root.
-Not every intermediate certificate is checked, starting from the original certificate all the way up to the root certificate.
-An intermediate, CA-signed certificate does not have the expected Basic Constraints or other important extensions.
-The root certificate has been compromised or authorized to the wrong party.",,low,"The Most Dangerous Code in the World: Validating SSL Certificates in Non-Browser Software: http://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
-24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347"
-CWE-298,EN-Improper Validation of Certificate Expiration (Type: Variant),"A certificate expiration is not validated or is incorrectly validated, so trust may be assigned to certificates that have been abandoned due to age.
-When the expiration of a certificate is not taken into account, no trust has necessarily been conveyed through it. Therefore, the validity of the certificate cannot be verified and all benefit of the certificate is lost.",,low,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347"
-CWE-324,EN-Use of a Key Past its Expiration Date (Type: Base),"The product uses a cryptographic key or password past its expiration date, which diminishes its safety significantly by increasing the timing window for cracking attacks against that key.
-While the expiration of keys does not necessarily ensure that they are compromised, it is a significant concern that keys which remain in use for prolonged periods of time have a decreasing probability of integrity. For this reason, it is important to replace keys within a period of time proportional to their strength.",,low,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347"
-CWE-379,EN-Creation of Temporary File in Directory with Incorrect Permissions (Type: Base),"The software creates a temporary file in a directory whose permissions allow unintended actors to determine the file's existence or otherwise access that file.
-On some operating systems, the fact that the temporary file exists may be apparent to any user with sufficient privileges to access that directory. Since the file is visible, the application that is using the temporary file could be known. If one has access to list the processes on the system, the attacker has gained information about what the user is doing at that time. By correlating this with the applications the user is running, an attacker could potentially discover what a user's actions are. From this, higher levels of security could be breached.",,low,"The Art of Software Security Assessment: Chapter 9, ""Temporary Files"", Page 538."
-CWE-462,EN-Duplicate Key in Associative List (Alist) (Type: Base),"Duplicate keys in associative lists can lead to non-unique keys being mistaken for an error.
-A duplicate key entry -- if the alist is designed properly -- could be used as a constant time replace function. However, duplicate key entries could be inserted by mistake. Because of this ambiguity, duplicate key entries in an association list are not recommended and should not be allowed.",,low,
-CWE-479,EN-Signal Handler Use of a Non-reentrant Function (Type: Variant),"The program defines a signal handler that calls a non-reentrant function.
-Non-reentrant functions are functions that cannot safely be called, interrupted, and then recalled before the first call has finished without resulting in memory corruption. This can lead to an unexpected system state an unpredictable results with a variety of potential consequences depending on context, including denial of service and code execution.
-Many functions are not reentrant, but some of them can result in the corruption of memory if they are used in a signal handler. The function call syslog() is an example of this. In order to perform its functionality, it allocates a small amount of memory as ""scratch space."" If syslog() is suspended by a signal call and the signal handler calls syslog(), the memory used by both of these functions enters an undefined, and possibly, exploitable state. Implementations of malloc() and free() manage metadata in global structures in order to track which memory is allocated versus which memory is available, but they are non-reentrant. Simultaneous calls to these functions can cause corruption of the metadata.",,low,"The Art of Software Security Assessment: Chapter 13, ""Signal Vulnerabilities"", Page 791."
-CWE-480,EN-Use of Incorrect Operator (Type: Base),"The programmer accidentally uses the wrong operator, which changes the application logic in security-relevant ways.
-Non-reentrant functions are functions that cannot safely be called, interrupted, and then recalled before the first call has finished without resulting in memory corruption. This can lead to an unexpected system state an unpredictable results with a variety of potential consequences depending on context, including denial of service and code execution.
-Many functions are not reentrant, but some of them can result in the corruption of memory if they are used in a signal handler. The function call syslog() is an example of this. In order to perform its functionality, it allocates a small amount of memory as ""scratch space."" If syslog() is suspended by a signal call and the signal handler calls syslog(), the memory used by both of these functions enters an undefined, and possibly, exploitable state. Implementations of malloc() and free() manage metadata in global structures in order to track which memory is allocated versus which memory is available, but they are non-reentrant. Simultaneous calls to these functions can cause corruption of the metadata.",,low,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289."
-CWE-481,EN-Assigning instead of Comparing (Type: Variant),"The code uses an operator for assignment when the intention was to perform a comparison.
-In many languages the compare statement is very close in appearance to the assignment statement and are often confused. This bug is generally the result of a typo and usually causes obvious problems with program execution. If the comparison is in an if statement, the if statement will usually evaluate the value of the right-hand side of the predicate.",,low,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289."
-CWE-482,EN-Comparing instead of Assigning (Type: Variant),"The code uses an operator for comparison when the intention was to perform an assignment.
-In many languages, the compare statement is very close in appearance to the assignment statement; they are often confused.",,low,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289."
-CWE-483,EN-Incorrect Block Delimitation (Type: Variant),"The code does not explicitly delimit a block that is intended to contain 2 or more statements, creating a logic error.
-In some languages, braces (or other delimiters) are optional for blocks. When the delimiter is omitted, it is possible to insert a logic error in which a statement is thought to be in a block but is not. In some cases, the logic error can have security implications.",,low,
-CWE-641,EN-Improper Restriction of Names for Files and Other Resources (Type: Base),"The application constructs the name of a file or other resource using input from an upstream component, but does not restrict or incorrectly restricts the resulting name.
-This may produce resultant weaknesses. For instance, if the names of these resources contain scripting characters, it is possible that a script may get executed in the client's browser if the application ever displays the name of the resource on a dynamically generated web page. Alternately, if the resources are consumed by some application parser, a specially crafted name can exploit some vulnerability internal to the parser, potentially resulting in execution of arbitrary code on the server machine. The problems will vary based on the context of usage of such malformed resource names and whether vulnerabilities are present in or assumptions are made by the targeted technology that would make code execution possible.",,low,
-CWE-648,EN-Incorrect Use of Privileged APIs (Type: Base),"The application does not conform to the API requirements for a function call that requires extra privileges. This could allow attackers to gain privileges by causing the function to be called incorrectly.
-When an application contains certain functions that perform operations requiring an elevated level of privilege, the caller of a privileged API must be careful to:
-ensure that assumptions made by the APIs are valid, such as validity of arguments
-account for known weaknesses in the design/implementation of the API
-call the API from a safe context
-If the caller of the API does not follow these requirements, then it may allow a malicious user or process to elevate their privilege, hijack the process, or steal sensitive data.
-For instance, it is important to know if privileged APIs do not shed their privileges before returning to the caller or if the privileged function might make certain assumptions about the data, context or state information passed to it by the caller. It is important to always know when and how privileged APIs can be called in order to ensure that their elevated level of privilege cannot be exploited.",,low,
-CWE-762,EN-Mismatched Memory Management Routines (Type: Variant),"The application attempts to return a memory resource to the system, but it calls a release function that is not compatible with the function that was originally used to allocate that resource.
-This weakness can be generally described as mismatching memory management routines, such as:
-The memory was allocated on the stack (automatically), but it was deallocated using the memory management routine free() (CWE-590), which is intended for explicitly allocated heap memory.
-The memory was allocated explicitly using one set of memory management functions, and deallocated using a different set. For example, memory might be allocated with malloc() in C++ instead of the new operator, and then deallocated with the delete operator.
-When the memory management functions are mismatched, the consequences may be as severe as code execution, memory corruption, or program crash. Consequences and ease of exploit will vary depending on the implementation of the routines and the object being managed.",,low,"boost C++ Library Smart Pointers: http://www.boost.org/doc/libs/1_38_0/libs/smart_ptr/smart_ptr.htm
-Valgrind: http://valgrind.org/"
-CWE-783,EN-Operator Precedence Logic Error (Type: Variant),"The program uses an expression in which operator precedence causes incorrect logic to be used.
-While often just a bug, operator precedence logic errors can have serious consequences if they are used in security-critical code, such as making an authentication decision.",,low,"EXP00-C. Use parentheses for precedence of operation: https://www.securecoding.cert.org/confluence/display/seccode/EXP00-C.+Use+parentheses+for+precedence+of+operation
-The Art of Software Security Assessment: Chapter 6, ""Precedence"", Page 287."
-CWE-789,EN-Uncontrolled Memory Allocation (Type: Variant),"The product allocates memory based on an untrusted size value, but it does not validate or incorrectly validates the size, allowing arbitrary amounts of memory to be allocated.
-This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. These problems may be resultant from missing sentinel values (CWE-463) or trusting a user-influenced input length variable.",,low,"The Art of Software Security Assessment: Chapter 10, ""Resource Limits"", Page 574."
-CWE-333,EN-Improper Handling of Insufficient Entropy in TRNG (Type: Variant),"True random number generators (TRNG) generally have a limited source of entropy and therefore can fail or block.
-The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,medium,
-CWE-367,EN-Time-of-check Time-of-use (TOCTOU) Race Condition (Type: Base),"The software checks the state of a resource before using that resource, but the resource's state can change between the check and the use in a way that invalidates the results of the check. This can cause the software to perform invalid actions when the resource is in an unexpected state.
-This weakness can be security-relevant when an attacker can influence the state of the resource between check and use. This can happen with shared resources such as files, memory, or even variables in multithreaded programs.",,medium,"Portably Solving File TOCTTOU Races with Hardness Amplification: http://www.usenix.org/events/fast08/tech/tsafrir.html
-24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205
-The Art of Software Security Assessment: Chapter 9, ""TOCTOU"", Page 527."
-CWE-404,EN-Improper Resource Shutdown or Release (Type: Base),"The program does not release or incorrectly releases a resource before it is made available for re-use.
-When a resource is created or allocated, the developer is responsible for properly releasing the resource as well as accounting for all potential paths of expiration or invalidation, such as a set period of time or revocation.",,medium,"24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143"
-CWE-407,EN-Algorithmic Complexity (Type: Base),"An algorithm in a product has an inefficient worst-case computational complexity that may be detrimental to system performance and can be triggered by an attacker, typically using crafted manipulations that ensure that the worst case is being reached.
-In the absence of a policy to restrict asymmetric resource consumption, the application or system cannot distinguish between legitimate transmissions and traffic intended to serve as an amplifying attack on target systems. Systems can often be configured to restrict the amount of traffic sent out on behalf of a client, based on the client's origin or access level. This is usually defined in a resource allocation policy. In the absence of a mechanism to keep track of transmissions, the system or application can be easily abused to transmit asymmetrically greater traffic than the request or client should be permitted to.",,medium,Algorithmic Complexity Attacks: http://www.cs.rice.edu/~scrosby/hash/CrosbyWallach_UsenixSec2003/index.html
-CWE-415,EN-Double Free (Type: Variant),"The product calls free() twice on the same memory address, potentially leading to modification of unexpected memory locations.
-When a program calls free() twice with the same argument, the program's memory management data structures become corrupted. This corruption can cause the program to crash or, in some circumstances, cause two later calls to malloc() to return the same pointer. If malloc() returns the same value twice and the program later gives the attacker control over the data that is written into this doubly-allocated memory, the program becomes vulnerable to a buffer overflow attack.",,medium,"24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143
-The Art of Software Security Assessment: Chapter 7, ""Double Frees"", Page 379."
-CWE-59,EN-Improper Link Resolution Before File Access (Link Following) (Type: Base),"The software attempts to access a file based on the filename, but it does not properly prevent that filename from identifying a link or shortcut that resolves to an unintended resource.
-Some functions that offer security features supported by the OS are not available on all versions of the OS in common use. Likewise, functions are often deprecated or made obsolete for security reasons and should not be used.",,medium,"The Art of Software Security Assessment: Chapter 9, ""Symbolic Link Attacks"", Page 518."
-CWE-601,EN-URL Redirection to Untrusted Site (Open Redirect) (Type: Variant),"A web application accepts a user-controlled input that specifies a link to an external site, and uses that link in a Redirect. This simplifies phishing attacks.
-An http parameter may contain a URL value and could cause the web application to redirect the request to the specified URL. By modifying the URL value to a malicious site, an attacker may successfully launch a phishing scam and steal user credentials. Because the server name in the modified link is identical to the original site, phishing attempts have a more trustworthy appearance.",,medium,"Exploitable Redirects on the Web: Identification, Prevalence, and Defense: http://www.cs.indiana.edu/cgi-pub/cshue/research/woot08.pdf
-Open redirect vulnerabilities: definition and prevention: http://www.net-security.org/dl/insecure/INSECURE-Mag-17.pdf
-Top 25 Series - Rank 23 - Open Redirect: http://software-security.sans.org/blog/2010/03/25/top-25-series-rank-23-open-redirect
-OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI"
-CWE-749,EN-Exposed Dangerous Method or Function (Type: Base),"The software provides an Applications Programming Interface (API) or similar interface for interaction with external actors, but the interface includes a dangerous method or function that is not properly restricted.
-This weakness can lead to a wide variety of resultant weaknesses, depending on the behavior of the exposed method. It can apply to any number of technologies and approaches, such as ActiveX controls, Java functions, IOCTLs, and so on.
-The exposure can occur in a few different ways:
-1) The function/method was never intended to be exposed to outside actors.
-2) The function/method was only intended to be accessible to a limited set of actors, such as Internet-based access from a single web site.",,medium,"No description: http://msdn.microsoft.com/workshop/components/activex/safety.asp
-No description: http://msdn.microsoft.com/workshop/components/activex/security.asp"
-CWE-755,EN-Improper Handling of Exceptional Conditions (Type: Class),"The software does not handle or incorrectly handles an exceptional condition.
-The programmer may assume that certain events or conditions will never occur or do not need to be worried about, such as low memory conditions, lack of access to resources due to restrictive permissions, or misbehaving clients or components. However, attackers may intentionally trigger these unusual conditions, thus violating the programmer's assumptions, possibly introducing instability, incorrect behavior, or a vulnerability.
-Note that this entry is not exclusively about the use of exceptions and exception handling, which are mechanisms for both checking and handling unusual or unexpected conditions.",,medium,
-CWE-766,EN-Critical Variable Declared Public (Type: Variant),"The software declares a critical variable or field to be public when intended security policy requires it to be private.
-When software is operating in a concurrent environment and repeatedly unlocks a critical resource, the consequences will vary based on the type of lock, the lock's implementation, and the resource being protected. In some situations such as with semaphores, the resources are pooled and extra calls to unlock will increase the count for the number of available resources, likely resulting in a crash or unpredictable behavior when the system nears capacity.",,medium,
-CWE-767,EN-Access to Critical Private Variable via Public Method (Type: Variant),"The software defines a public method that reads or modifies a private variable.
-If an attacker modifies the variable to contain unexpected values, this could violate assumptions from other parts of the code. Additionally, if an attacker can read the private variable, it may expose sensitive information or make it easier to launch further attacks.",,medium,
-CWE-776,EN-Improper Restriction of Recursive Entity References in DTDs (XML Entity Expansion) (Type: Variant),"The software uses XML documents and allows their structure to be defined with a Document Type Definition (DTD), but it does not properly control the number of recursive definitions of entities.
-If the DTD contains a large number of nested or recursive entities, this can lead to explosive growth of data when parsed, causing a denial of service.",,medium,"Multiple vendors XML parser (and SOAP/WebServices server) Denial of Service attack using DTD: http://www.securityfocus.com/archive/1/303509
-XML security: Preventing XML bombs: http://searchsoftwarequality.techtarget.com/expert/KnowledgebaseAnswer/0,289625,sid92_gci1168442,00.html?asrc=SS_CLA_302%20%20558&psrc=CLT_92#
-Dismantling an XML-Bomb: http://blog.didierstevens.com/2008/09/23/dismantling-an-xml-bomb/
-XML Entity Expansion: http://projects.webappsec.org/XML-Entity-Expansion
-Tip: Configure SAX parsers for secure processing: http://www.ibm.com/developerworks/xml/library/x-tipcfsx.html
-XML Denial of Service Attacks and Defenses: http://msdn.microsoft.com/en-us/magazine/ee335713.aspx
-Preventing Entity Expansion Attacks in JAXB: http://blog.bdoughan.com/2011/03/preventing-entity-expansion-attacks-in.html"
-CWE-777,EN-Regular Expression without Anchors (Type: Variant),"The software uses a regular expression to perform neutralization, but the regular expression is not anchored and may allow malicious or malformed data to slip through.
-When performing tasks such as whitelist validation, data is examined and possibly modified to ensure that it is well-formed and adheres to a list of safe values. If the regular expression is not anchored, malicious or malformed data may be included before or after any string matching the regular expression. The type of malicious data that is allowed will depend on the context of the application and which anchors are omitted from the regular expression.",,medium,
-CWE-779,EN-Logging of Excessive Data (Type: Base),"The software logs too much information, making log files hard to process and possibly hindering recovery efforts or forensic analysis after an attack.
-While logging is a good practice in general, and very high levels of logging are appropriate for debugging stages of development, too much logging in a production environment might hinder a system administrator's ability to detect anomalous conditions. This can provide cover for an attacker while attempting to penetrate a system, clutter the audit trail for forensic analysis, or make it more difficult to debug problems in a production environment.",,medium,
-CWE-781,EN-Improper Address Validation in IOCTL with METHOD_NEITHER I/O Control Code (Type: Variant),"The software defines an IOCTL that uses METHOD_NEITHER for I/O, but it does not validate or incorrectly validates the addresses that are provided.
-When an IOCTL uses the METHOD_NEITHER option for I/O control, it is the responsibility of the IOCTL to validate the addresses that have been supplied to it. If validation is missing or incorrect, attackers can supply arbitrary memory addresses, leading to code execution or a denial of service.",,medium,"Exploiting Common Flaws in Drivers: http://reversemode.com/index.php?option=com_content&task=view&id=38&Itemid=1
-Remote and Local Exploitation of Network Drivers: https://www.blackhat.com/presentations/bh-usa-07/Bulygin/Presentation/bh-usa-07-bulygin.pdf
-Windows driver vulnerabilities: the METHOD_NEITHER odyssey: http://www.net-security.org/dl/insecure/INSECURE-Mag-18.pdf
-Buffer Descriptions for I/O Control Codes: http://msdn.microsoft.com/en-us/library/ms795857.aspx
-Using Neither Buffered Nor Direct I/O: http://msdn.microsoft.com/en-us/library/cc264614.aspx
-Securing Device Objects: http://msdn.microsoft.com/en-us/library/ms794722.aspx
-No description: http://www.piotrbania.com/all/articles/ewdd.pdf"
-CWE-782,EN-Exposed IOCTL with Insufficient Access Control (Type: Variant),"The software implements an IOCTL with functionality that should be restricted, but it does not properly enforce access control for the IOCTL.
-When an IOCTL contains privileged functionality and is exposed unnecessarily, attackers may be able to access this functionality by invoking the IOCTL. Even if the functionality is benign, if the programmer has assumed that the IOCTL would only be accessed by a trusted process, there may be little or no validation of the incoming data, exposing weaknesses that would never be reachable if the attacker cannot call the IOCTL directly.
-The implementations of IOCTLs will differ between operating system types and versions, so the methods of attack and prevention may vary widely.",,medium,Securing Device Objects: http://msdn.microsoft.com/en-us/library/ms794722.aspx
-CWE-117,EN-Improper Output Neutralization for Logs (Type: Base),"The software does not neutralize or incorrectly neutralizes output that is written to logs.
-This can allow an attacker to forge log entries or inject malicious content into logs.
-Log forging vulnerabilities occur when:
-Data enters an application from an untrusted source.
-The data is written to an application or system log file.",,medium,"Exploiting Software: How to Break Code
-The night the log was forged: http://doc.novsu.ac.ru/oreilly/tcpip/puis/ch10_05.htm
-OWASP TOP 10: http://www.owasp.org/index.php/Top_10_2007"
-CWE-124,EN-Buffer Underwrite (Buffer Underflow) (Type: Base),"The software writes to a buffer using an index or pointer that references a memory location prior to the beginning of the buffer.
-This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used.",,medium,"Buffer UNDERFLOWS: What do you know about it?: http://seclists.org/vuln-dev/2004/Jan/0022.html
-24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89"
-CWE-128,EN-Wrap-around Error (Type: Base),"Wrap around errors occur whenever a value is incremented past the maximum value for its type and therefore ""wraps around"" to a very small, negative, or undefined value.
-This typically occurs when the pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. This may result in exposure of sensitive information or possibly a crash.",,medium,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89
-The Art of Software Security Assessment: Chapter 6, ""Signed Integer Boundaries"", Page 220."
-CWE-170,EN-Improper Null Termination (Type: Base),"The software does not terminate or incorrectly terminates a string or array with a null character or equivalent terminator.
-Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,medium,
-CWE-190,EN-Integer Overflow or Wraparound (Type: Base),"The software performs a calculation that can produce an integer overflow or wraparound, when the logic assumes that the resulting value will always be larger than the original value. This can introduce other weaknesses when the calculation is used for resource management or execution control.
-An integer overflow or wraparound occurs when an integer value is incremented to a value that is too large to store in the associated representation. When this occurs, the value may wrap to become a very small or negative number. While this may be intended behavior in circumstances that rely on wrapping, it can have security consequences if the wrap is unexpected. This is especially the case if the integer overflow can be triggered using user-supplied inputs. This becomes security-critical when the result is used to control looping, make a security decision, or determine the offset or size in behaviors such as memory allocation, copying, concatenation, etc.",,medium,"An overview of common programming security vulnerabilities and possible solutions: http://fort-knox.org/thesis.pdf
-Basic Integer Overflows: http://www.phrack.org/issues.html?issue=60&id=10#article
-Writing Secure Code: Chapter 20, ""Integer Overflows"" Page 620
-24 Deadly Sins of Software Security: ""Sin 7: Integer Overflows."" Page 119
-SafeInt: http://safeint.codeplex.com/
-Top 25 Series - Rank 17 - Integer Overflow Or Wraparound: http://software-security.sans.org/blog/2010/03/18/top-25-series-rank-17-integer-overflow-or-wraparound
-The Art of Software Security Assessment: Chapter 6, ""Signed Integer Boundaries"", Page 220."
-CWE-196,EN-Unsigned to Signed Conversion Error (Type: Variant),"An unsigned-to-signed conversion error takes place when a large unsigned primitive is used as a signed value.
-It is dangerous to rely on implicit casts between signed and unsigned numbers because the result can take on an unexpected value and violate assumptions made by the program.",,medium,"The Art of Software Security Assessment: Chapter 6, ""Type Conversions"", Page 223."
-CWE-202,EN-Exposure of Sensitive Data Through Data Queries (Type: Variant),"When trying to keep information confidential, an attacker can often infer some of the information by using statistics.
-In situations where data should not be tied to individual users, but a large number of users should be able to make queries that ""scrub"" the identity of users, it may be possible to get information about a user -- e.g., by specifying search terms that are known to be unique to that user.",,medium,
-CWE-250,EN-Execution with Unnecessary Privileges (Type: Class),"The software performs an operation at a privilege level that is higher than the minimum level required, which creates new weaknesses or amplifies the consequences of other weaknesses.
-New weaknesses can be exposed because running with extra privileges, such as root or Administrator, can disable the normal security checks being performed by the operating system or surrounding environment. Other pre-existing weaknesses can turn into security vulnerabilities if they occur while operating at raised privileges.
-Privilege management functions can behave in some less-than-obvious ways, and they have different quirks on different platforms. These inconsistencies are particularly pronounced if you are transitioning from one non-root user to another. Signal handlers and spawned processes run at the privilege of the owning process, so if a process is running as root when a signal fires or a sub-process is executed, the signal handler or sub-process will operate with root privileges.",,medium,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/
-Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html
-Writing Secure Code: Chapter 7, ""Running with Least Privilege"" Page 207
-Federal Desktop Core Configuration: http://nvd.nist.gov/fdcc/index.cfm
-24 Deadly Sins of Software Security: ""Sin 16: Executing Code With Too Much Privilege."" Page 243
-The Art of Software Security Assessment: Chapter 9, ""Privilege Vulnerabilities"", Page 477."
-CWE-269,EN-Improper Privilege Management (Type: Base),"The software does not properly assign, modify, track, or check privileges for an actor, creating an unintended sphere of control for that actor.
-Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,medium,"24 Deadly Sins of Software Security: ""Sin 16: Executing Code With Too Much Privilege."" Page 243
-The Art of Software Security Assessment: Chapter 9, ""Dropping Privileges Permanently"", Page 479."
-CWE-273,EN-Improper Check for Dropped Privileges (Type: Base),"The software attempts to drop privileges but does not check or incorrectly checks to see if the drop succeeded.
-If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,medium,
-CWE-276,EN-Incorrect Default Permissions (Type: Variant),"The software, upon installation, sets incorrect permissions for an object that exposes it to an unintended actor.
-If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,medium,"The Art of Software Security Assessment: Chapter 3, ""Insecure Defaults"", Page 69."
-CWE-299,EN-Improper Check for Certificate Revocation (Type: Variant),"The software does not check or incorrectly checks the revocation status of a certificate, which may cause it to use a certificate that has been compromised.
-An improper check for certificate revocation is a far more serious flaw than related certificate failures. This is because the use of any revoked certificate is almost certainly malicious. The most common reason for certificate revocation is compromise of the system in question, with the result that no legitimate servers will be using a revoked certificate, unless they are sorely out of sync.",,medium,"24 Deadly Sins of Software Security: ""Sin 23: Improper Use of PKI, Especially SSL."" Page 347"
-CWE-301,EN-Reflection Attack in an Authentication Protocol (Type: Variant),"Simple authentication protocols are subject to reflection attacks if a malicious user can use the target machine to impersonate a trusted user.
-A mutual authentication protocol requires each party to respond to a random challenge by the other party by encrypting it with a pre-shared key. Often, however, such protocols employ the same pre-shared key for communication with a number of different entities. A malicious user or an attacker can easily compromise this protocol without possessing the correct key by employing a reflection attack on the protocol.",,medium,"The Art of Software Security Assessment: Chapter 2, ""Insufficient Validation"", Page 38."
-CWE-329,EN-Not Using a Random IV with CBC Mode (Type: Variant),"Not using a random initialization Vector (IV) with Cipher Block Chaining (CBC) Mode causes algorithms to be susceptible to dictionary attacks.
-This weakness is especially dangerous when the hash is used in security algorithms that require the one-way property to hold. For example, if an authentication system takes an incoming password and generates a hash, then compares the hash to another hash that it has stored in its authentication database, then the ability to create a collision could allow an attacker to provide an alternate password that produces the same target hash, bypassing authentication.",,medium,"The Art of Software Security Assessment: Chapter 2, ""Initialization Vectors"", Page 42."
-CWE-332,EN-Insufficient Entropy in PRNG (Type: Variant),"The lack of entropy available for, or used by, a Pseudo-Random Number Generator (PRNG) can be a stability and security threat.
-When software generates predictable values in a context requiring unpredictability, it may be possible for an attacker to guess the next value that will be generated, and use this guess to impersonate another user or access sensitive information.",,medium,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf
-CWE-338,EN-Use of Cryptographically Weak PRNG (Type: Base),"The product uses a Pseudo-Random Number Generator (PRNG) in a security context, but the PRNG is not cryptographically strong.
-The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,medium,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299"
-CWE-353,EN-Missing Support for Integrity Check (Type: Base),"The software uses a transmission protocol that does not include a mechanism for verifying the integrity of the data during transmission, such as a checksum.
-If integrity check values or ""checksums"" are omitted from a protocol, there is no way of determining if data has been corrupted in transmission. The lack of checksum functionality in a protocol removes the first application-level check of data that can be used. The end-to-end philosophy of checks states that integrity checks should be performed at the lowest level that they can be completely implemented. Excluding further sanity checks and input validation performed by applications, the protocol's checksum is the most important level of checksum, since it can be performed more completely than at any previous level and takes into account entire messages, as opposed to single packets.",,medium,"24 Deadly Sins of Software Security: ""Sin 15: Not Updating Easily."" Page 231"
-CWE-354,EN-Improper Validation of Integrity Check Value (Type: Base),"The software does not validate or incorrectly validates the integrity check values or ""checksums"" of a message. This may prevent it from detecting if the data has been modified or corrupted in transmission.
-Improper validation of checksums before use results in an unnecessary risk that can easily be mitigated. The protocol specification describes the algorithm used for calculating the checksum. It is then a simple matter of implementing the calculation and verifying that the calculated checksum and the received checksum match. Improper verification of the calculated checksum and the received checksum can lead to far greater consequences.",,medium,
-CWE-362,EN-Concurrent Execution using Shared Resource with Improper Synchronization (Race Condition) (Type: Class),"The program contains a code sequence that can run concurrently with other code, and the code sequence requires temporary, exclusive access to a shared resource, but a timing window exists in which the shared resource can be modified by another code sequence that is operating concurrently.
-This can have security implications when the expected synchronization is in security-critical code, such as recording whether a user is authenticated or modifying important state information that should not be influenced by an outsider.
-A race condition occurs within concurrent environments, and is effectively a property of a code sequence. Depending on the context, a code sequence may be in the form of a function call, a small number of instructions, a series of program invocations, etc.
-A race condition violates these properties, which are closely related:
-Exclusivity - the code sequence is given exclusive access to the shared resource, i.e., no other code sequence can modify properties of the shared resource before the original sequence has completed execution.
-Atomicity - the code sequence is behaviorally atomic, i.e., no other thread or process can concurrently execute the same sequence of instructions (or a subset) against the same resource.
-A race condition exists when an ""interfering code sequence"" can still access the shared resource, violating exclusivity. Programmers may assume that certain code sequences execute too quickly to be affected by an interfering code sequence; when they are not, this violates atomicity. For example, the single ""x++"" statement may appear atomic at the code layer, but it is actually non-atomic at the instruction layer, since it involves a read (the original value of x), followed by a computation (x+1), followed by a write (save the result to x).
-The interfering code sequence could be ""trusted"" or ""untrusted."" A trusted interfering code sequence occurs within the program; it cannot be modified by the attacker, and it can only be invoked indirectly. An untrusted interfering code sequence can be authored directly by the attacker, and typically it is external to the vulnerable program.",,medium,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205
-volatile - Multithreaded Programmer's Best Friend: http://www.ddj.com/cpp/184403766
-Thread-safe webapps using Spring: http://www.javalobby.org/articles/thread-safe/index.jsp
-Prevent race conditions: http://www.ibm.com/developerworks/library/l-sprace.html
-Race Conditions, Files, and Security Flaws; or the Tortoise and the Hare Redux: http://www.cs.ucdavis.edu/research/tech-reports/1995/CSE-95-9.pdf
-Secure Programming for Linux and Unix HOWTO: http://www.dwheeler.com/secure-programs/Secure-Programs-HOWTO/avoid-race.html
-Discovering and Exploiting Named Pipe Security Flaws for Fun and Profit: http://www.blakewatts.com/namedpipepaper.html
-On Race Vulnerabilities in Web Applications: http://security.dico.unimi.it/~roberto/pubs/dimva08-web.pdf
-Avoiding Race Conditions and Insecure File Operations: http://developer.apple.com/documentation/Security/Conceptual/SecureCodingGuide/Articles/RaceConditions.html
-Top 25 Series - Rank 25 - Race Conditions: http://blogs.sans.org/appsecstreetfighter/2010/03/26/top-25-series-rank-25-race-conditions/
-Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html"
-CWE-364,EN-Signal Handler Race Condition (Type: Base),"The software uses a signal handler that introduces a race condition.
-Race conditions frequently occur in signal handlers, since signal handlers support asynchronous actions. These race conditions have a variety of root causes and symptoms. Attackers may be able to exploit a signal handler race condition to cause the software state to be corrupted, possibly leading to a denial of service or even code execution.
-These issues occur when non-reentrant functions, or state-sensitive actions occur in the signal handler, where they may be called at any time. These behaviors can violate assumptions being made by the ""regular"" code that is interrupted, or by other signal handlers that may also be invoked. If these functions are called at an inopportune moment - such as while a non-reentrant function is already running - memory corruption could occur that may be exploitable for code execution. Another signal race condition commonly found occurs when free is called within a signal handler, resulting in a double free and therefore a write-what-where condition. Even if a given pointer is set to NULL after it has been freed, a race condition still exists between the time the memory was freed and the pointer was set to NULL. This is especially problematic if the same signal handler has been set for more than one signal -- since it means that the signal handler itself may be reentered.
-There are several known behaviors related to signal handlers that have received the label of ""signal handler race condition"":
-Shared state (e.g. global data or static variables) that are accessible to both a signal handler and ""regular"" code
-Shared state between a signal handler and other signal handlers
-Use of non-reentrant functionality within a signal handler - which generally implies that shared state is being used. For example, malloc() and free() are non-reentrant because they may use global or static data structures for managing memory, and they are indirectly used by innocent-seeming functions such as syslog(); these functions could be exploited for memory corruption and, possibly, code execution.
-Association of the same signal handler function with multiple signals - which might imply shared state, since the same code and resources are accessed. For example, this can be a source of double-free and use-after-free weaknesses.
-Use of setjmp and longjmp, or other mechanisms that prevent a signal handler from returning control back to the original functionality
-While not technically a race condition, some signal handlers are designed to be called at most once, and being called more than once can introduce security problems, even when there are not any concurrent calls to the signal handler. This can be a source of double-free and use-after-free weaknesses.
-Signal handler vulnerabilities are often classified based on the absence of a specific protection mechanism, although this style of classification is discouraged in CWE because programmers often have a choice of several different mechanisms for addressing the weakness. Such protection mechanisms may preserve exclusivity of access to the shared resource, and behavioral atomicity for the relevant code:
-Avoiding shared state
-Using synchronization in the signal handler
-Using synchronization in the regular code
-Disabling or masking other signals, which provides atomicity (which effectively ensures exclusivity)",,medium,"Delivering Signals for Fun and Profit: http://lcamtuf.coredump.cx/signals.txt
-Race Condition: Signal Handling: http://www.fortify.com/vulncat/en/vulncat/cpp/race_condition_signal_handling.html
-24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205
-The Art of Software Security Assessment: Chapter 13, ""Signal Vulnerabilities"", Page 791."
-CWE-365,EN-Race Condition in Switch (Type: Base),"The code contains a switch statement in which the switched variable can be modified while the switch is still executing, resulting in unexpected behavior.
-Race conditions frequently occur in signal handlers, since signal handlers support asynchronous actions. These race conditions have a variety of root causes and symptoms. Attackers may be able to exploit a signal handler race condition to cause the software state to be corrupted, possibly leading to a denial of service or even code execution.
-These issues occur when non-reentrant functions, or state-sensitive actions occur in the signal handler, where they may be called at any time. These behaviors can violate assumptions being made by the ""regular"" code that is interrupted, or by other signal handlers that may also be invoked. If these functions are called at an inopportune moment - such as while a non-reentrant function is already running - memory corruption could occur that may be exploitable for code execution. Another signal race condition commonly found occurs when free is called within a signal handler, resulting in a double free and therefore a write-what-where condition. Even if a given pointer is set to NULL after it has been freed, a race condition still exists between the time the memory was freed and the pointer was set to NULL. This is especially problematic if the same signal handler has been set for more than one signal -- since it means that the signal handler itself may be reentered.
-There are several known behaviors related to signal handlers that have received the label of ""signal handler race condition"":
-Shared state (e.g. global data or static variables) that are accessible to both a signal handler and ""regular"" code
-Shared state between a signal handler and other signal handlers
-Use of non-reentrant functionality within a signal handler - which generally implies that shared state is being used. For example, malloc() and free() are non-reentrant because they may use global or static data structures for managing memory, and they are indirectly used by innocent-seeming functions such as syslog(); these functions could be exploited for memory corruption and, possibly, code execution.
-Association of the same signal handler function with multiple signals - which might imply shared state, since the same code and resources are accessed. For example, this can be a source of double-free and use-after-free weaknesses.
-Use of setjmp and longjmp, or other mechanisms that prevent a signal handler from returning control back to the original functionality
-While not technically a race condition, some signal handlers are designed to be called at most once, and being called more than once can introduce security problems, even when there are not any concurrent calls to the signal handler. This can be a source of double-free and use-after-free weaknesses.
-Signal handler vulnerabilities are often classified based on the absence of a specific protection mechanism, although this style of classification is discouraged in CWE because programmers often have a choice of several different mechanisms for addressing the weakness. Such protection mechanisms may preserve exclusivity of access to the shared resource, and behavioral atomicity for the relevant code:
-Avoiding shared state
-Using synchronization in the signal handler
-Using synchronization in the regular code
-Disabling or masking other signals, which provides atomicity (which effectively ensures exclusivity)",,medium,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205"
-CWE-366,EN-Race Condition within a Thread (Type: Base),"If two threads of execution use a resource simultaneously, there exists the possibility that resources may be used while invalid, in turn making the state of execution undefined.
-Race conditions frequently occur in signal handlers, since signal handlers support asynchronous actions. These race conditions have a variety of root causes and symptoms. Attackers may be able to exploit a signal handler race condition to cause the software state to be corrupted, possibly leading to a denial of service or even code execution.
-These issues occur when non-reentrant functions, or state-sensitive actions occur in the signal handler, where they may be called at any time. These behaviors can violate assumptions being made by the ""regular"" code that is interrupted, or by other signal handlers that may also be invoked. If these functions are called at an inopportune moment - such as while a non-reentrant function is already running - memory corruption could occur that may be exploitable for code execution. Another signal race condition commonly found occurs when free is called within a signal handler, resulting in a double free and therefore a write-what-where condition. Even if a given pointer is set to NULL after it has been freed, a race condition still exists between the time the memory was freed and the pointer was set to NULL. This is especially problematic if the same signal handler has been set for more than one signal -- since it means that the signal handler itself may be reentered.
-There are several known behaviors related to signal handlers that have received the label of ""signal handler race condition"":
-Shared state (e.g. global data or static variables) that are accessible to both a signal handler and ""regular"" code
-Shared state between a signal handler and other signal handlers
-Use of non-reentrant functionality within a signal handler - which generally implies that shared state is being used. For example, malloc() and free() are non-reentrant because they may use global or static data structures for managing memory, and they are indirectly used by innocent-seeming functions such as syslog(); these functions could be exploited for memory corruption and, possibly, code execution.
-Association of the same signal handler function with multiple signals - which might imply shared state, since the same code and resources are accessed. For example, this can be a source of double-free and use-after-free weaknesses.
-Use of setjmp and longjmp, or other mechanisms that prevent a signal handler from returning control back to the original functionality
-While not technically a race condition, some signal handlers are designed to be called at most once, and being called more than once can introduce security problems, even when there are not any concurrent calls to the signal handler. This can be a source of double-free and use-after-free weaknesses.
-Signal handler vulnerabilities are often classified based on the absence of a specific protection mechanism, although this style of classification is discouraged in CWE because programmers often have a choice of several different mechanisms for addressing the weakness. Such protection mechanisms may preserve exclusivity of access to the shared resource, and behavioral atomicity for the relevant code:
-Avoiding shared state
-Using synchronization in the signal handler
-Using synchronization in the regular code
-Disabling or masking other signals, which provides atomicity (which effectively ensures exclusivity)",,medium,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205
-The Art of Software Security Assessment: Chapter 13, ""Race Conditions"", Page 759."
-CWE-369,EN-Divide By Zero (Type: Base),"The product divides a value by zero.
-This weakness typically occurs when an unexpected value is provided to the product, or if an error occurs that is not properly detected. It frequently occurs in calculations involving physical dimensions such as size, length, width, and height.",,medium,"No description: http://www.cprogramming.com/tutorial/exceptions.html
-No description: http://msdn.microsoft.com/en-us/library/ms173160(VS.80).aspx"
-CWE-370,EN-Missing Check for Certificate Revocation after Initial Check (Type: Base),"The software does not check the revocation status of a certificate after its initial revocation check, which can cause the software to perform privileged actions even after the certificate is revoked at a later time.
-If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,medium,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205"
-CWE-374,EN-Passing Mutable Objects to an Untrusted Method (Type: Base),"Sending non-cloned mutable data as an argument may result in that data being altered or deleted by the called function, thereby putting the calling function into an undefined state.
-If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,medium,"Does Java pass by reference or pass by value?: http://www.javaworld.com/javaworld/javaqa/2000-05/03-qa-0526-pass.html
-Java: The Complete Reference, J2SE 5th Edition"
-CWE-375,EN-Returning a Mutable Object to an Untrusted Caller (Type: Base),"Sending non-cloned mutable data as a return value may result in that data being altered or deleted by the calling function, thereby putting the class in an undefined state.
-If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,medium,
-CWE-385,EN-Covert Timing Channel (Type: Base),"Covert timing channels convey information by modulating some aspect of system behavior over time, so that the program receiving the information can observe system behavior and infer protected information.
-In some instances, knowing when data is transmitted between parties can provide a malicious user with privileged information. Also, externally monitoring the timing of operations can potentially reveal sensitive data. For example, a cryptographic operation can expose its internal state if the time it takes to perform the operation varies, based on the state.
-Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,medium,
-CWE-390,EN-Detection of Error Condition Without Action (Type: Class),"The software detects a specific error, but takes no actions to handle the error.
-In some instances, knowing when data is transmitted between parties can provide a malicious user with privileged information. Also, externally monitoring the timing of operations can potentially reveal sensitive data. For example, a cryptographic operation can expose its internal state if the time it takes to perform the operation varies, based on the state.
-Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,medium,"24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183"
-CWE-391,EN-Unchecked Error Condition (Type: Base),"Ignoring exceptions and other error conditions may allow an attacker to induce unexpected behavior unnoticed.
-In some instances, knowing when data is transmitted between parties can provide a malicious user with privileged information. Also, externally monitoring the timing of operations can potentially reveal sensitive data. For example, a cryptographic operation can expose its internal state if the time it takes to perform the operation varies, based on the state.
-Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,medium,
-CWE-401,EN-Improper Release of Memory Before Removing Last Reference (Memory Leak) (Type: Base),"The software does not sufficiently track and release allocated memory after it has been used, which slowly consumes remaining memory.
-This is often triggered by improper handling of malformed data or unexpectedly interrupted sessions.",,medium,How to Break Software Security
-CWE-460,EN-Improper Cleanup on Thrown Exception (Type: Variant),"The product does not clean up its state or incorrectly cleans up its state when an exception is thrown, leading to unexpected state or control flow.
-In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,medium,
-CWE-468,EN-Incorrect Pointer Scaling (Type: Base),"In C and C++, one may often accidentally refer to the wrong memory due to the semantics of when math operations are implicitly scaled.
-Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,medium,"The Art of Software Security Assessment: Chapter 6, ""Pointer Arithmetic"", Page 277."
-CWE-469,EN-Use of Pointer Subtraction to Determine Size (Type: Base),"The application subtracts one pointer from another in order to determine size, but this calculation can be incorrect if the pointers do not exist in the same memory chunk.
-Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,medium,
-CWE-476,EN-NULL Pointer Dereference (Type: Base),"A NULL pointer dereference occurs when the application dereferences a pointer that it expects to be valid, but is NULL, typically causing a crash or exit.
-NULL pointer dereference issues can occur through a number of flaws, including race conditions, and simple programming omissions.",,medium,
-CWE-484,EN-Omitted Break Statement in Switch (Type: Base),"The program omits a break statement within a switch or similar construct, causing code associated with multiple conditions to execute. This can cause problems when the programmer only intended to execute code associated with one condition.
-This can lead to critical code executing in situations where it should not.",,medium,"The Art of Software Security Assessment: Chapter 7, ""Switch Statements"", Page 337."
-CWE-487,EN-Reliance on Package-level Scope (Type: Variant),"Java packages are not inherently closed; therefore, relying on them for code security is not a good practice.
-If the decision to trust the methods and data of an object is based on the name of a class, it is possible for malicious users to send objects of the same name as trusted classes and thereby gain the trust afforded to known classes and types.",,medium,
-CWE-492,EN-Use of Inner Class Containing Sensitive Data (Type: Variant),"Inner classes are translated into classes that are accessible at package scope and may expose code that the programmer intended to keep private to attackers.
-Data can ""bleed"" from one session to another through member variables of singleton objects, such as Servlets, and objects from a shared pool.
-In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,medium,
-CWE-494,EN-Download of Code Without Integrity Check (Type: Base),"The product downloads source code or an executable from a remote location and executes the code without sufficiently verifying the origin and integrity of the code.
-An attacker can execute malicious code by compromising the host server, performing DNS spoofing, or modifying the code in transit.",,medium,"Introduction to Code Signing: http://msdn.microsoft.com/en-us/library/ms537361(VS.85).aspx
-Authenticode: http://msdn.microsoft.com/en-us/library/ms537359(v=VS.85).aspx
-Code Signing Guide: http://developer.apple.com/documentation/Security/Conceptual/CodeSigningGuide/Introduction/chapter_1_section_1.html
-Secure Software Updates: Disappointments and New Challenges: http://prisms.cs.umass.edu/~kevinfu/papers/secureupdates-hotsec06.pdf
-24 Deadly Sins of Software Security: ""Sin 18: The Sins of Mobile Code."" Page 267
-Top 25 Series - Rank 20 - Download of Code Without Integrity Check: http://blogs.sans.org/appsecstreetfighter/2010/04/05/top-25-series-rank-20-download-code-integrity-check/
-Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html"
-CWE-498,EN-Cloneable Class Containing Sensitive Information (Type: Variant),"The code contains a class with sensitive data, but the class is cloneable. The data can then be accessed by cloning the class.
-Cloneable classes are effectively open classes, since data cannot be hidden in them. Classes that do not explicitly deny cloning can be cloned by any other class without running the constructor.",,medium,
-CWE-502,EN-Deserialization of Untrusted Data (Type: Variant),"The application deserializes untrusted data without sufficiently verifying that the resulting data will be valid.
-It is often convenient to serialize objects for communication or to save them for later use. However, deserialized data or code can often be modified without using the provided accessor functions if it does not use cryptography to protect itself. Furthermore, any cryptography would still be client-side security -- which is a dangerous security assumption.
-Data that is untrusted can not be trusted to be well-formed.",,medium,"Unserializing user-supplied data, a bad idea: http://heine.familiedeelstra.com/security/unserialize
-Why Python Pickle is Insecure: http://nadiana.com/python-pickle-insecure"
-CWE-532,EN-Information Exposure Through Log Files (Type: Variant),"Information written to log files can be of a sensitive nature and give valuable guidance to an attacker or expose sensitive user information.
-While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,medium,
-CWE-602,EN-Client-Side Enforcement of Server-Side Security (Type: Base),"The software is composed of a server that relies on the client to implement a mechanism that is intended to protect the server.
-When the server relies on protection mechanisms placed on the client side, an attacker can modify the client-side behavior to bypass the protection mechanisms resulting in potentially unexpected interactions between the client and server. The consequences will vary, depending on what the mechanisms are trying to protect.",,medium,"Writing Secure Code: Chapter 23, ""Client-Side Security Is an Oxymoron"" Page 687"
-CWE-665,EN-Improper Initialization (Type: Base),"The software does not initialize or incorrectly initializes a resource, which might leave the resource in an unexpected state when it is accessed or used.
-This can have security implications when the associated resource is expected to have certain properties or values, such as a variable that determines whether a user has been authenticated or not.",,medium,"Exploiting Uninitialized Data: http://www.felinemenace.org/~mercy/papers/UBehavior/UBehavior.zip
-MS08-014 : The Case of the Uninitialized Stack Variable Vulnerability: http://blogs.technet.com/swi/archive/2008/03/11/the-case-of-the-uninitialized-stack-variable-vulnerability.aspx
-The Art of Software Security Assessment: Chapter 7, ""Variable Initialization"", Page 312."
-CWE-754,EN-Improper Check for Unusual or Exceptional Conditions (Type: Class),"The software does not check or improperly checks for unusual or exceptional conditions that are not expected to occur frequently during day to day operation of the software.
-The programmer may assume that certain events or conditions will never occur or do not need to be worried about, such as low memory conditions, lack of access to resources due to restrictive permissions, or misbehaving clients or components. However, attackers may intentionally trigger these unusual conditions, thus violating the programmer's assumptions, possibly introducing instability, incorrect behavior, or a vulnerability.
-Note that this entry is not exclusively about the use of exceptions and exception handling, which are mechanisms for both checking and handling unusual or unexpected conditions.",,medium,"The Art of Software Security Assessment: Chapter 7, ""Program Building Blocks"" Page 341
-The Art of Software Security Assessment: Chapter 1, ""Exceptional Conditions,"" Page 22
-24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183
-Top 25 Series - Rank 15 - Improper Check for Unusual or Exceptional Conditions: http://blogs.sans.org/appsecstreetfighter/2010/03/15/top-25-series-rank-15-improper-check-for-unusual-or-exceptional-conditions/"
-CWE-778,EN-Insufficient Logging (Type: Base),"When a security-critical event occurs, the software either does not record the event or omits important details about the event when logging it.
-When security-critical events are not logged properly, such as a failed login attempt, this can make malicious behavior more difficult to detect and may hinder forensic analysis after an attack succeeds.",,medium,"The Art of Software Security Assessment: Chapter 2, ""Accountability"", Page 40."
-CWE-780,EN-Use of RSA Algorithm without OAEP (Type: Variant),"The software uses the RSA algorithm but does not incorporate Optimal Asymmetric Encryption Padding (OAEP), which might weaken the encryption.
-Padding schemes are often used with cryptographic algorithms to make the plaintext less predictable and complicate attack efforts. The OAEP scheme is often used with RSA to nullify the impact of predictable common text.",,medium,"RSA Problem: http://people.csail.mit.edu/rivest/RivestKaliski-RSAProblem.pdf
-Optimal Asymmetric Encryption Padding: http://en.wikipedia.org/wiki/Optimal_Asymmetric_Encryption_Padding"
-CWE-908,EN-Use of Uninitialized Resource (Type: Base),"The software uses a resource that has not been properly initialized.
-This can have security implications when the associated resource is expected to have certain properties or values.",,medium,Exploiting Uninitialized Data: http://www.felinemenace.org/~mercy/papers/UBehavior/UBehavior.zip
-CWE-909,EN-Missing Initialization of Resource (Type: Base),"The software does not initialize a critical resource.
-Many resources require initialization before they can be properly used. If a resource is not initialized, it could contain unpredictable or expired data, or it could be initialized to defaults that are invalid. This can have security implications when the resource is expected to have certain properties or values.",,medium,
-CWE-910,EN-Use of Expired File Descriptor (Type: Base),"The software uses or accesses a file descriptor after it has been closed.
-After a file descriptor for a particular file or device has been released, it can be reused. The code might not write to the original file, since the reused file descriptor might reference a different file or device.",,medium,
-CWE-911,EN-Improper Update of Reference Count (Type: Base),"The software uses a reference count to manage a resource, but it does not update or incorrectly updates the reference count.
-Reference counts can be used when tracking how many objects contain a reference to a particular resource, such as in memory management or garbage collection. When the reference count reaches zero, the resource can be de-allocated or reused because there are no more objects that use it. If the reference count accidentally reaches zero, then the resource might be released too soon, even though it is still in use. If all objects no longer use the resource, but the reference count is not zero, then the resource might not ever be released.",,medium,Windows Kernel Reference Count Vulnerabilities - Case Study: http://j00ru.vexillium.org/dump/zn_slides.pdf
-CWE-94,EN-Improper Control of Generation of Code (Code Injection) (Type: Class),"The software constructs all or part of a code segment using externally-influenced input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could modify the syntax or behavior of the intended code segment.
-When software allows a user's input to contain code syntax, it might be possible for an attacker to craft the code in such a way that it will alter the intended control flow of the software. Such an alteration could lead to arbitrary code execution.
-Injection problems encompass a wide variety of issues -- all mitigated in very different ways. For this reason, the most effective way to discuss these weaknesses is to note the distinct features which classify them as injection weaknesses. The most important issue to note is that all injection problems share one thing in common -- i.e., they allow for the injection of control plane data into the user-controlled data plane. This means that the execution of the process may be altered by sending code in through legitimate data channels, using no other mechanism. While buffer overflows, and many other flaws, involve the use of some further issue to gain execution, injection problems need only for the data to be parsed. The most classic instantiations of this category of weakness are SQL injection and format string vulnerabilities.",,medium,"24 Deadly Sins of Software Security: ""Sin 3: Web-Client Related Vulnerabilities (XSS)."" Page 63"
-CWE-95,EN-Improper Neutralization of Directives in Dynamically Evaluated Code (Eval Injection) (Type: Base),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes code syntax before using the input in a dynamic evaluation call (e.g. ""eval"").
-This may allow an attacker to execute arbitrary code, or at least modify what code can be executed.",,medium,"No description: http://www.rubycentral.com/book/taint.html
-The Art of Software Security Assessment: Chapter 18, ""Inline Evaluation"", Page 1095."
-CWE-287,EN-Improper Authentication (Type: Class),"When an actor claims to have a given identity, the software does not prove or insufficiently proves that the claim is correct.
-Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,high,"Weak Password Brings 'Happiness' to Twitter Hacker: http://www.wired.com/threatlevel/2009/01/professed-twitt/
-Top 10 2007-Broken Authentication and Session Management: http://www.owasp.org/index.php/Top_10_2007-A7
-Guide to Authentication: http://www.owasp.org/index.php/Guide_to_Authentication
-Authentication: http://msdn.microsoft.com/en-us/library/aa374735(VS.85).aspx
-Writing Secure Code: Chapter 4, ""Authentication"" Page 109"
-CWE-306,EN-Missing Authentication for Critical Function (Type: Variant),"The software does not perform any authentication for functionality that requires a provable user identity or consumes a significant amount of resources.
-Authentication techniques should follow the algorithms that define them exactly, otherwise authentication can be bypassed or more easily subjected to brute force attacks.",,high,"The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Authentication,"" Page 36
-Top 25 Series - Rank 19 - Missing Authentication for Critical Function: http://blogs.sans.org/appsecstreetfighter/2010/02/23/top-25-series-rank-19-missing-authentication-for-critical-function/
-OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI"
-CWE-319,EN-Cleartext Transmission of Sensitive Information (Type: Base),"The software transmits sensitive or security-critical data in cleartext in a communication channel that can be sniffed by unauthorized actors.
-Many communication channels can be ""sniffed"" by attackers during data transmission. For example, network traffic can often be sniffed by any attacker who has access to a network interface. This significantly lowers the difficulty of exploitation by attackers.",,high,"Top 10 2007-Insecure Communications: http://www.owasp.org/index.php/Top_10_2007-A9
-Writing Secure Code: Chapter 9, ""Protecting Secret Data"" Page 299
-24 Deadly Sins of Software Security: ""Sin 22: Failing to Protect Network Traffic."" Page 337
-Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/"
-CWE-327,EN-Use of a Broken or Risky Cryptographic Algorithm (Type: Base),"The use of a broken or risky cryptographic algorithm is an unnecessary risk that may result in the exposure of sensitive information.
-The use of a non-standard algorithm is dangerous because a determined attacker may be able to break the algorithm and compromise whatever data has been protected. Well-known techniques may exist to break the algorithm.",,high,"Applied Cryptography: http://www.schneier.com/book-applied.html
-Handbook of Applied Cryptography: http://www.cacr.math.uwaterloo.ca/hac/
-Avoiding bogus encryption products: Snake Oil FAQ: http://www.faqs.org/faqs/cryptography-faq/snake-oil/
-SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf
-Microsoft Scraps Old Encryption in New Code: http://www.eweek.com/c/a/Security/Microsoft-Scraps-Old-Encryption-in-New-Code/
-Writing Secure Code: Chapter 8, ""Cryptographic Foibles"" Page 259
-24 Deadly Sins of Software Security: ""Sin 21: Using the Wrong Cryptography."" Page 315
-Top 25 Series - Rank 24 - Use of a Broken or Risky Cryptographic Algorithm: http://blogs.sans.org/appsecstreetfighter/2010/03/25/top-25-series-rank-24-use-of-a-broken-or-risky-cryptographic-algorithm/
-The Art of Software Security Assessment: Chapter 2, ""Insufficient or Obsolete Encryption"", Page 44."
-CWE-330,EN-Use of Insufficiently Random Values (Type: Class),"The software may use insufficiently random numbers or values in a security context that depends on unpredictable numbers.
-When software generates predictable values in a context requiring unpredictability, it may be possible for an attacker to guess the next value that will be generated, and use this guess to impersonate another user or access sensitive information.",,high,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf
-Building Secure Software: How to Avoid Security Problems the Right Way
-Writing Secure Code: Chapter 8, ""Using Poor Random Numbers"" Page 259
-24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299"
-CWE-400,EN-Uncontrolled Resource Consumption (Resource Exhaustion) (Type: Base),"The software does not properly restrict the size or amount of resources that are requested or influenced by an actor, which can be used to consume more resources than intended.
-Limited resources include memory, file system storage, database connection pool entries, or CPU. If an attacker can trigger the allocation of these limited resources, but the number or size of the resources is not controlled, then the attacker could cause a denial of service that consumes all available resources. This would prevent valid users from accessing the software, and it could potentially have an impact on the surrounding environment. For example, a memory exhaustion attack against an application could slow down the application as well as its host operating system.
-Resource exhaustion problems have at least two common causes:
-Error conditions and other exceptional circumstances
-Confusion over which part of the program is responsible for releasing the resource",,high,"Detection and Prediction of Resource-Exhaustion Vulnerabilities: http://homepages.di.fc.ul.pt/~nuno/PAPERS/ISSRE08.pdf
-Resource exhaustion: http://cr.yp.to/docs/resources.html
-Resource exhaustion: http://homes.cerias.purdue.edu/~pmeunier/secprog/sanitized/class1/6.resource%20exhaustion.ppt
-Writing Secure Code: Chapter 17, ""Protecting Against Denial of Service Attacks"" Page 517"
-CWE-434,EN-Unrestricted Upload of File with Dangerous Type (Type: Base),"The software allows the attacker to upload or transfer files of dangerous types that can be automatically processed within the product's environment.
-If code is stored in a file with an extension such as "".inc"" or "".pl"", and the web server does not have a handler for that extension, then the server will likely send the contents of the file directly to the requester without the pre-processing that was expected. When that file contains sensitive information such as database credentials, this may allow the attacker to compromise the application or associated components.",,high,"Dynamic File Uploads, Security and You: http://shsc.info/FileUploadSecurity
-8 Basic Rules to Implement Secure File Uploads: http://blogs.sans.org/appsecstreetfighter/2009/12/28/8-basic-rules-to-implement-secure-file-uploads/
-Top 25 Series - Rank 8 - Unrestricted Upload of Dangerous File Type: http://blogs.sans.org/appsecstreetfighter/2010/02/25/top-25-series-rank-8-unrestricted-upload-of-dangerous-file-type/
-Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html
-The Art of Software Security Assessment: Chapter 17, ""File Uploading"", Page 1068."
-CWE-64,EN-Windows Shortcut Following (.LNK) (Type: Variant),"The software, when opening a file or directory, does not sufficiently handle when the file is a Windows shortcut (.LNK) whose target is outside of the intended control sphere. This could allow an attacker to cause the software to operate on unauthorized files.
-The shortcut (file with the .lnk extension) can permit an attacker to read/write a file that they originally did not have permissions to access.",,high,
-CWE-681,EN-Incorrect Conversion between Numeric Types (Type: Base),"When converting from one data type to another, such as long to integer, data can be omitted or translated in a way that produces unexpected values. If the resulting values are used in a sensitive context, then dangerous behaviors may occur.
-Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,high,"The Art of Software Security Assessment: Chapter 6, ""Type Conversions"", Page 223."
-CWE-732,EN-Incorrect Permission Assignment for Critical Resource (Type: Class),"The software specifies permissions for a security-critical resource in a way that allows that resource to be read or modified by unintended actors.
-When a resource is given a permissions setting that provides access to a wider range of actors than required, it could lead to the exposure of sensitive information, or the modification of that resource by unintended parties. This is especially dangerous when the resource is related to program configuration, execution or sensitive user data.",,high,"The Art of Software Security Assessment: Chapter 9, ""File Permissions."" Page 495.
-Building Secure Software: How to Avoid Security Problems the Right Way: Chapter 8, ""Access Control."" Page 194.
-Top 25 Series - Rank 21 - Incorrect Permission Assignment for Critical Response: http://software-security.sans.org/blog/2010/03/24/top-25-series-rank-21-incorrect-permission-assignment-for-critical-response
-Federal Desktop Core Configuration: http://nvd.nist.gov/fdcc/index.cfm"
-CWE-770,EN-Allocation of Resources Without Limits or Throttling (Type: Base),"The software allocates a reusable resource or group of resources on behalf of an actor without imposing any restrictions on how many resources can be allocated, in violation of the intended security policy for that actor.
-Command injection vulnerabilities typically occur when:
-1. Data enters the application from an untrusted source.
-2. The data is part of a string that is executed as a command by the application.
-3. By executing the command, the application gives an attacker a privilege or capability that the attacker would not otherwise have.",,high,"Real-Life Example of a 'Business Logic Defect' (Screen Shots!): http://h30501.www3.hp.com/t5/Following-the-White-Rabbit-A/Real-Life-Example-of-a-Business-Logic-Defect-Screen-Shots/ba-p/22581
-Detection and Prediction of Resource-Exhaustion Vulnerabilities: http://homepages.di.fc.ul.pt/~nuno/PAPERS/ISSRE08.pdf
-Resource exhaustion: http://cr.yp.to/docs/resources.html
-Resource exhaustion: http://homes.cerias.purdue.edu/~pmeunier/secprog/sanitized/class1/6.resource%20exhaustion.ppt
-Writing Secure Code: Chapter 17, ""Protecting Against Denial of Service Attacks"" Page 517
-Top 25 Series - Rank 22 - Allocation of Resources Without Limits or Throttling: http://blogs.sans.org/appsecstreetfighter/2010/03/23/top-25-series-rank-22-allocation-of-resources-without-limits-or-throttling/
-The Art of Software Security Assessment: Chapter 10, ""Resource Limits"", Page 574."
-CWE-771,EN-Missing Reference to Active Allocated Resource (Type: Base),"The software does not properly maintain a reference to a resource that has been allocated, which prevents the resource from being reclaimed.
-This does not necessarily apply in languages or frameworks that automatically perform garbage collection, since the removal of all references may act as a signal that the resource is ready to be reclaimed.",,high,
-CWE-772,EN-Missing Release of Resource after Effective Lifetime (Type: Base),"The software does not release a resource after its effective lifetime has ended, i.e., after the resource is no longer needed.
-When a resource is not released after use, it can allow attackers to cause a denial of service.",,high,
-CWE-773,EN-Missing Reference to Active File Descriptor or Handle (Type: Variant),"The software does not properly maintain references to a file descriptor or handle, which prevents that file descriptor/handle from being reclaimed.
-This can cause the software to consume all available file descriptors or handles, which can prevent other processes from performing critical file processing operations.",,high,
-CWE-774,EN-Allocation of File Descriptors or Handles Without Limits or Throttling (Type: Variant),"The software allocates file descriptors or handles on behalf of an actor without imposing any restrictions on how many descriptors can be allocated, in violation of the intended security policy for that actor.
-This can cause the software to consume all available file descriptors or handles, which can prevent other processes from performing critical file processing operations.",,high,"The Art of Software Security Assessment: Chapter 10, ""Resource Limits"", Page 574."
-CWE-775,EN-Missing Release of File Descriptor or Handle after Effective Lifetime (Type: Variant),"The software does not release a file descriptor or handle after its effective lifetime has ended, i.e., after the file descriptor/handle is no longer needed.
-When a file descriptor or handle is not released after use (typically by explicitly closing it), attackers can cause a denial of service by consuming all available file descriptors/handles, or otherwise preventing other system processes from obtaining their own file descriptors/handles.",,high,"The Art of Software Security Assessment: Chapter 10, ""File Descriptor Leaks"", Page 582."
-CWE-804,EN-Guessable CAPTCHA (Type: Base),"The software uses a CAPTCHA challenge, but the challenge can be guessed or automatically recognized by a non-human actor.
-An automated attacker could bypass the intended protection of the CAPTCHA challenge and perform actions at a higher frequency than humanly possible, such as launching spam attacks.
-There can be several different causes of a guessable CAPTCHA:
-An audio or visual image that does not have sufficient distortion from the unobfuscated source image.
-A question is generated that with a format that can be automatically recognized, such as a math question.
-A question for which the number of possible answers is limited, such as birth years or favorite sports teams.
-A general-knowledge or trivia question for which the answer can be accessed using a data base, such as country capitals or popular actors.
-Other data associated with the CAPTCHA may provide hints about its contents, such as an image whose filename contains the word that is used in the CAPTCHA.",,high,Insufficient Anti-automation: http://projects.webappsec.org/Insufficient+Anti-automation
-CWE-805,EN-Buffer Access with Incorrect Length Value (Type: Base),"The software uses a sequential operation to read or write a buffer, but it uses an incorrect length value that causes it to access memory that is outside of the bounds of the buffer.
-When the length value exceeds the size of the destination, a buffer overflow could occur.",,high,"Writing Secure Code: Chapter 6, ""Why ACLs Are Important"" Page 171
-Address Space Layout Randomization in Windows Vista: http://blogs.msdn.com/michael_howard/archive/2006/05/26/address-space-layout-randomization-in-windows-vista.aspx
-Limiting buffer overflows with ExecShield: http://www.redhat.com/magazine/009jul05/features/execshield/
-PaX: http://en.wikipedia.org/wiki/PaX
-Top 25 Series - Rank 12 - Buffer Access with Incorrect Length Value: http://blogs.sans.org/appsecstreetfighter/2010/03/11/top-25-series-rank-12-buffer-access-with-incorrect-length-value/
-Safe C String Library v1.0.3: http://www.zork.org/safestr/
-Using the Strsafe.h Functions: http://msdn.microsoft.com/en-us/library/ms647466.aspx
-Understanding DEP as a mitigation technology part 1: http://blogs.technet.com/b/srd/archive/2009/06/12/understanding-dep-as-a-mitigation-technology-part-1.aspx
-Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html"
-CWE-806,EN-Buffer Access Using Size of Source Buffer (Type: Variant),"The software uses the size of a source buffer when reading from or writing to a destination buffer, which may cause it to access memory that is outside of the bounds of the buffer.
-When the size of the destination is smaller than the size of the source, a buffer overflow could occur.",,high,"Using the Strsafe.h Functions: http://msdn.microsoft.com/en-us/library/ms647466.aspx
-Safe C String Library v1.0.3: http://www.zork.org/safestr/
-Address Space Layout Randomization in Windows Vista: http://blogs.msdn.com/michael_howard/archive/2006/05/26/address-space-layout-randomization-in-windows-vista.aspx
-Limiting buffer overflows with ExecShield: http://www.redhat.com/magazine/009jul05/features/execshield/
-PaX: http://en.wikipedia.org/wiki/PaX
-Understanding DEP as a mitigation technology part 1: http://blogs.technet.com/b/srd/archive/2009/06/12/understanding-dep-as-a-mitigation-technology-part-1.aspx"
-CWE-807,EN-Reliance on Untrusted Inputs in a Security Decision (Type: Base),"The application uses a protection mechanism that relies on the existence or values of an input, but the input can be modified by an untrusted actor in a way that bypasses the protection mechanism.
-Developers may assume that inputs such as cookies, environment variables, and hidden form fields cannot be modified. However, an attacker could change these inputs using customized clients or other attacks. This change might not be detected. When security decisions such as authentication and authorization are made based on the values of these inputs, attackers can bypass the security of the software.
-Without sufficient encryption, integrity checking, or other mechanism, any input that originates from an outsider cannot be trusted.",,high,"Top 25 Series - Rank 6 - Reliance on Untrusted Inputs in a Security Decision: http://blogs.sans.org/appsecstreetfighter/2010/03/05/top-25-series-rank-6-reliance-on-untrusted-inputs-in-a-security-decision/
-HMAC: http://en.wikipedia.org/wiki/Hmac
-Understanding ASP.NET View State: http://msdn.microsoft.com/en-us/library/ms972976.aspx
-OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI"
-CWE-93,EN-Improper Neutralization of CRLF Sequences (CRLF Injection) (Type: Base),"The software uses CRLF (carriage return line feeds) as a special element, e.g. to separate lines or records, but it does not neutralize or incorrectly neutralizes CRLF sequences from inputs.
-Since an implicit intent does not specify a particular application to receive the data, any application can process the intent by using an Intent Filter for that intent. This can allow untrusted applications to obtain sensitive data.",,high,CRLF Injection: http://marc.info/?l=bugtraq&m=102088154213630&w=2
-CWE-102,EN-Struts: Duplicate Validation Forms (Type: Variant),"The application uses multiple validation forms with the same name, which might cause the Struts Validator to validate a form that the programmer does not expect.
-If two validation forms have the same name, the Struts Validator arbitrarily chooses one of the forms to use for input validation and discards the other. This decision might not correspond to the programmer's expectations, possibly leading to resultant weaknesses. Moreover, it indicates that the validation logic is not up-to-date, and can indicate that other, more subtle validation errors are present.",,unclassified,
-CWE-103,EN-Struts: Incomplete validate() Method Definition (Type: Variant),"The application has a validator form that either does not define a validate() method, or defines a validate() method but does not call super.validate().
-If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,unclassified,
-CWE-104,EN-Struts: Form Bean Does Not Extend Validation Class (Type: Variant),"If a form bean does not extend an ActionForm subclass of the Validator framework, it can expose the application to other weaknesses related to insufficient input validation.
-If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,unclassified,
-CWE-105,EN-Struts: Form Field Without Validator (Type: Variant),"The application has a form field that is not validated by a corresponding validation form, which can introduce other weaknesses related to insufficient input validation.
-If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,unclassified,
-CWE-106,EN-Struts: Plug-in Framework not in Use (Type: Variant),"When an application does not use an input validation framework such as the Struts Validator, there is a greater risk of introducing weaknesses related to insufficient input validation.
-If you do not call super.validate(), the Validation Framework cannot check the contents of the form against a validation form. In other words, the validation framework will be disabled for the given form.",,unclassified,
-CWE-107,EN-Struts: Unused Validation Form (Type: Variant),"An unused validation form indicates that validation logic is not up-to-date.
-It is easy for developers to forget to update validation logic when they remove or rename action form mappings. One indication that validation logic is not being properly maintained is the presence of an unused validation form.",,unclassified,
-CWE-108,EN-Struts: Unvalidated Action Form (Type: Variant),"Every Action Form must have a corresponding validation form.
-If a Struts Action Form Mapping specifies a form, it must have a validation form defined under the Struts Validator.",,unclassified,
-CWE-109,EN-Struts: Validator Turned Off (Type: Variant),"Automatic filtering via a Struts bean has been turned off, which disables the Struts Validator and custom validation logic. This exposes the application to other weaknesses related to insufficient input validation.
-If a Struts Action Form Mapping specifies a form, it must have a validation form defined under the Struts Validator.",,unclassified,
-CWE-11,EN-ASP.NET Misconfiguration: Creating Debug Binary (Type: Variant),"Debugging messages help attackers learn about the system and plan a form of attack.
-ASP .NET applications can be configured to produce debug binaries. These binaries give detailed debugging messages and should not be used in production environments. Debug binaries are meant to be used in a development or testing environment and can pose a security risk if they are deployed to production.",,unclassified,
-CWE-110,EN-Struts: Validator Without Form Field (Type: Variant),"Validation fields that do not appear in forms they are associated with indicate that the validation logic is out of date.
-It is easy for developers to forget to update validation logic when they make changes to an ActionForm class. One indication that validation logic is not being properly maintained is inconsistencies between the action form and the validation form.",,unclassified,
-CWE-111,EN-Direct Use of Unsafe JNI (Type: Base),"When a Java application uses the Java Native Interface (JNI) to call code written in another programming language, it can expose the application to weaknesses in that code, even if those weaknesses cannot occur in Java.
-Many safety features that programmers may take for granted simply do not apply for native code, so you must carefully review all such code for potential problems. The languages used to implement native code may be more susceptible to buffer overflows and other attacks. Native code is unprotected by the security features enforced by the runtime environment, such as strong typing and array bounds checking.",,unclassified,"Fortify Descriptions: http://vulncat.fortifysoftware.com
-The Java(TM) Tutorial: The Java Native Interface: http://java.sun.com/docs/books/tutorial/native1.1/"
-CWE-112,EN-Missing XML Validation (Type: Base),"The software accepts XML from an untrusted source but does not validate the XML against the proper schema.
-Most successful attacks begin with a violation of the programmer's assumptions. By accepting an XML document without validating it against a DTD or XML schema, the programmer leaves a door open for attackers to provide unexpected, unreasonable, or malicious input.",,unclassified,
-CWE-113,EN-Improper Neutralization of CRLF Sequences in HTTP Headers (HTTP Response Splitting) (Type: Base),"The software receives data from an upstream component, but does not neutralize or incorrectly neutralizes CR and LF characters before the data is included in outgoing HTTP headers.
-Including unvalidated data in an HTTP header allows an attacker to specify the entirety of the HTTP response rendered by the browser. When an HTTP request contains unexpected CR (carriage return, also given by %0d or \r) and LF (line feed, also given by %0a or \n) characters the server may respond with an output stream that is interpreted as two different HTTP responses (instead of one). An attacker can control the second response and mount attacks such as cross-site scripting and cache poisoning attacks.
-HTTP response splitting weaknesses may be present when:
-Data enters a web application through an untrusted source, most frequently an HTTP request.
-The data is included in an HTTP response header sent to a web user without being validated for malicious characters.",,unclassified,"OWASP TOP 10: http://www.owasp.org/index.php/Top_10_2007
-24 Deadly Sins of Software Security: ""Sin 2: Web-Server Related Vulnerabilities (XSS, XSRF, and Response Splitting)."" Page 31"
-CWE-114,EN-Process Control (Type: Base),"Executing commands or loading libraries from an untrusted source or in an untrusted environment can cause an application to execute malicious commands (and payloads) on behalf of an attacker.
-Process control vulnerabilities take two forms: 1. An attacker can change the command that the program executes: the attacker explicitly controls what the command is. 2. An attacker can change the environment in which the command executes: the attacker implicitly controls what the command means. Process control vulnerabilities of the first type occur when either data enters the application from an untrusted source and the data is used as part of a string representing a command that is executed by the application. By executing the command, the application gives an attacker a privilege or capability that the attacker would not otherwise have.",,unclassified,
-CWE-115,EN-Misinterpretation of Input (Type: Base),"The software misinterprets an input, whether from an attacker or another product, in a security-relevant fashion.
-Process control vulnerabilities take two forms: 1. An attacker can change the command that the program executes: the attacker explicitly controls what the command is. 2. An attacker can change the environment in which the command executes: the attacker implicitly controls what the command means. Process control vulnerabilities of the first type occur when either data enters the application from an untrusted source and the data is used as part of a string representing a command that is executed by the application. By executing the command, the application gives an attacker a privilege or capability that the attacker would not otherwise have.",,unclassified,
-CWE-118,EN-Improper Access of Indexable Resource (Range Error) (Type: Class),"The software does not restrict or incorrectly restricts operations within the boundaries of a resource that is accessed using an index or pointer, such as memory or files.
-This can allow an attacker to forge log entries or inject malicious content into logs.
-Log forging vulnerabilities occur when:
-Data enters an application from an untrusted source.
-The data is written to an application or system log file.",,unclassified,
-CWE-12,EN-ASP.NET Misconfiguration: Missing Custom Error Page (Type: Variant),"An ASP .NET application must enable custom error pages in order to prevent attackers from mining information from the framework's built-in responses.
-Certain languages allow direct addressing of memory locations and do not automatically ensure that these locations are valid for the memory buffer that is being referenced. This can cause read or write operations to be performed on memory locations that may be associated with other variables, data structures, or internal program data.
-As a result, an attacker may be able to execute arbitrary code, alter the intended control flow, read sensitive information, or cause the system to crash.",,unclassified,"19 Deadly Sins of Software Security
-ASP.NET Misconfiguration: Missing Custom Error Handling: http://www.owasp.org/index.php/ASP.NET_Misconfiguration:_Missing_Custom_Error_Handling"
-CWE-125,EN-Out-of-bounds Read (Type: Base),"The software reads data past the end, or before the beginning, of the intended buffer.
-This typically occurs when the pointer or its index is incremented or decremented to a position beyond the bounds of the buffer or when pointer arithmetic results in a position outside of the valid memory location to name a few. This may result in corruption of sensitive information, a crash, or code execution among other things.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89"
-CWE-126,EN-Buffer Over-read (Type: Variant),"The software reads from a buffer using buffer access mechanisms such as indexes or pointers that reference memory locations after the targeted buffer.
-This typically occurs when the pointer or its index is incremented to a position beyond the bounds of the buffer or when pointer arithmetic results in a position outside of the valid memory location to name a few. This may result in exposure of sensitive information or possibly a crash.",,unclassified,
-CWE-127,EN-Buffer Under-read (Type: Variant),"The software reads from a buffer using buffer access mechanisms such as indexes or pointers that reference memory locations prior to the targeted buffer.
-This typically occurs when the pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. This may result in exposure of sensitive information or possibly a crash.",,unclassified,
-CWE-13,EN-ASP.NET Misconfiguration: Password in Configuration File (Type: Variant),"Storing a plaintext password in a configuration file allows anyone who can read the file access to the password-protected resource making them an easy target for attackers.
-This typically occurs when the pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. This may result in exposure of sensitive information or possibly a crash.",,unclassified,"How To: Encrypt Configuration Sections in ASP.NET 2.0 Using DPAPI: http://msdn.microsoft.com/en-us/library/ms998280.aspx
-How To: Encrypt Configuration Sections in ASP.NET 2.0 Using RSA: http://msdn.microsoft.com/en-us/library/ms998283.aspx
-.NET Framework Developer's Guide - Securing Connection Strings: http://msdn.microsoft.com/en-us/library/89211k9b(VS.80).aspx"
-CWE-130,EN-Improper Handling of Length Parameter Inconsistency (Type: Variant),"The software parses a formatted message or structure, but it does not handle or incorrectly handles a length field that is inconsistent with the actual length of the associated data.
-If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,unclassified,
-CWE-132,EN-DEPRECATED (Duplicate): Miscalculated Null Termination (Type: Base),"This entry has been deprecated because it was a duplicate of CWE-170. All content has been transferred to CWE-170.
-If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,unclassified,
-CWE-135,EN-Incorrect Calculation of Multi-Byte String Length (Type: Base),"The software does not correctly calculate the length of strings that can contain wide or multi-byte characters.
-If an attacker can manipulate the length parameter associated with an input such that it is inconsistent with the actual length of the input, this can be leveraged to cause the target application to behave in unexpected, and possibly, malicious ways. One of the possible motives for doing so is to pass in arbitrarily large input to the application. Another possible motivation is the modification of application state by including invalid data for subsequent properties of the application. Such weaknesses commonly lead to attacks such as buffer overflows and execution of arbitrary code.",,unclassified,"Writing Secure Code: Chapter 5, ""Unicode and ANSI Buffer Size Mismatches"" Page 153"
-CWE-138,EN-Improper Neutralization of Special Elements (Type: Class),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as control elements or syntactic markers when they are sent to a downstream component.
-Most languages and protocols have their own special elements such as characters and reserved words. These special elements can carry control implications. If software does not prevent external control or influence over the inclusion of such special elements, the control flow of the program may be altered from what was intended. For example, both Unix and Windows interpret the symbol < (""less than"") as meaning ""read input from a file"".",,unclassified,
-CWE-14,EN-Compiler Removal of Code to Clear Buffers (Type: Base),"Sensitive memory is cleared according to the source code, but compiler optimizations leave the memory untouched when it is not read from again, aka ""dead store removal.""
-This compiler optimization error occurs when:
-1. Secret data are stored in memory.
-2. The secret data are scrubbed from memory by overwriting its contents.
-3. The source code is compiled using an optimizing compiler, which identifies and removes the function that overwrites the contents as a dead store because the memory is not used subsequently.",,unclassified,"Writing Secure Code: Chapter 9, ""A Compiler Optimization Caveat"" Page 322
-When scrubbing secrets in memory doesn't work: http://cert.uni-stuttgart.de/archive/bugtraq/2002/11/msg00046.html
-Some Bad News and Some Good News: http://msdn.microsoft.com/library/default.asp?url=/library/en-us/dncode/html/secure10102002.asp
-GNU GCC: Optimizer Removes Code Necessary for Security: http://www.derkeiler.com/Mailing-Lists/securityfocus/bugtraq/2002-11/0257.html"
-CWE-140,EN-Improper Neutralization of Delimiters (Type: Base),"The software does not neutralize or incorrectly neutralizes delimiters.
-This compiler optimization error occurs when:
-1. Secret data are stored in memory.
-2. The secret data are scrubbed from memory by overwriting its contents.
-3. The source code is compiled using an optimizing compiler, which identifies and removes the function that overwrites the contents as a dead store because the memory is not used subsequently.",,unclassified,
-CWE-141,EN-Improper Neutralization of Parameter/Argument Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as parameter or argument delimiters when they are sent to a downstream component.
-As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408.
-The Art of Software Security Assessment: Chapter 10, ""IFS"", Page 604."
-CWE-142,EN-Improper Neutralization of Value Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as value delimiters when they are sent to a downstream component.
-As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408."
-CWE-143,EN-Improper Neutralization of Record Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as record delimiters when they are sent to a downstream component.
-As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408."
-CWE-144,EN-Improper Neutralization of Line Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as line delimiters when they are sent to a downstream component.
-As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408."
-CWE-145,EN-Improper Neutralization of Section Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as section delimiters when they are sent to a downstream component.
-As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.
-One example of a section delimiter is the boundary string in a multipart MIME message. In many cases, doubled line delimiters can serve as a section delimiter.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408."
-CWE-146,EN-Improper Neutralization of Expression/Command Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as expression or command delimiters when they are sent to a downstream component.
-As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Embedded Delimiters"", Page 408."
-CWE-147,EN-Improper Neutralization of Input Terminators (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as input terminators when they are sent to a downstream component.
-For example, a ""."" in SMTP signifies the end of mail message data, whereas a null character can be used for the end of a string.",,unclassified,
-CWE-148,EN-Improper Neutralization of Input Leaders (Type: Variant),"The application does not properly handle when a leading character or sequence (""leader"") is missing or malformed, or if multiple leaders are used when only one should be allowed.
-For example, a ""."" in SMTP signifies the end of mail message data, whereas a null character can be used for the end of a string.",,unclassified,
-CWE-149,EN-Improper Neutralization of Quoting Syntax (Type: Variant),"Quotes injected into an application can be used to compromise a system. As data are parsed, an injected/absent/duplicate/malformed use of quotes may cause the process to take unexpected actions.
-For example, a ""."" in SMTP signifies the end of mail message data, whereas a null character can be used for the end of a string.",,unclassified,
-CWE-15,EN-External Control of System or Configuration Setting (Type: Base),"One or more system settings or configuration elements can be externally controlled by a user.
-Allowing external control of system settings can disrupt service or cause an application to behave in unexpected, and potentially malicious ways.",,unclassified,
-CWE-150,"EN-Improper Neutralization of Escape, Meta, or Control Sequences (Type: Variant)","The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as escape, meta, or control character sequences when they are sent to a downstream component.
-As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,
-CWE-151,EN-Improper Neutralization of Comment Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as comment delimiters when they are sent to a downstream component.
-As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,
-CWE-152,EN-Improper Neutralization of Macro Symbols (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as macro symbols when they are sent to a downstream component.
-As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,
-CWE-153,EN-Improper Neutralization of Substitution Characters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as substitution characters when they are sent to a downstream component.
-As data is parsed, an injected/absent/malformed delimiter may cause the process to take unexpected actions.",,unclassified,
-CWE-154,EN-Improper Neutralization of Variable Name Delimiters (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as variable name delimiters when they are sent to a downstream component.
-As data is parsed, an injected delimiter may cause the process to take unexpected actions that result in an attack. Example: ""$"" for an environment variable.",,unclassified,
-CWE-155,EN-Improper Neutralization of Wildcards or Matching Symbols (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as wildcards or matching symbols when they are sent to a downstream component.
-As data is parsed, an injected element may cause the process to take unexpected actions.",,unclassified,
-CWE-156,EN-Improper Neutralization of Whitespace (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could be interpreted as whitespace when they are sent to a downstream component.
-This can include space, tab, etc.",,unclassified,
-CWE-157,EN-Failure to Sanitize Paired Delimiters (Type: Variant),"The software does not properly handle the characters that are used to mark the beginning and ending of a group of entities, such as parentheses, brackets, and braces.
-This can include space, tab, etc.",,unclassified,
-CWE-158,EN-Improper Neutralization of Null Byte or NUL Character (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes NUL characters or null bytes when they are sent to a downstream component.
-As data is parsed, an injected NUL character or null byte may cause the software to believe the input is terminated earlier than it actually is, or otherwise cause the input to be misinterpreted. This could then be used to inject potentially dangerous input that occurs after the null byte or otherwise bypass validation routines and other protection mechanisms.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""NUL Character Injection"", Page 411."
-CWE-159,EN-Failure to Sanitize Special Element (Type: Class),"Weaknesses in this attack-focused category do not properly filter and interpret special elements in user-controlled input which could cause adverse effect on the software behavior and integrity.
-As data is parsed, an injected NUL character or null byte may cause the software to believe the input is terminated earlier than it actually is, or otherwise cause the input to be misinterpreted. This could then be used to inject potentially dangerous input that occurs after the null byte or otherwise bypass validation routines and other protection mechanisms.",,unclassified,
-CWE-160,EN-Improper Neutralization of Leading Special Elements (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes leading special elements that could be interpreted in unexpected ways when they are sent to a downstream component.
-As data is parsed, improperly handled leading special elements may cause the process to take unexpected actions that result in an attack.",,unclassified,
-CWE-161,EN-Improper Neutralization of Multiple Leading Special Elements (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes multiple leading special elements that could be interpreted in unexpected ways when they are sent to a downstream component.
-As data is parsed, improperly handled multiple leading special elements may cause the process to take unexpected actions that result in an attack.",,unclassified,
-CWE-162,EN-Improper Neutralization of Trailing Special Elements (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes trailing special elements that could be interpreted in unexpected ways when they are sent to a downstream component.
-As data is parsed, improperly handled trailing special elements may cause the process to take unexpected actions that result in an attack.",,unclassified,
-CWE-163,EN-Improper Neutralization of Multiple Trailing Special Elements (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes multiple trailing special elements that could be interpreted in unexpected ways when they are sent to a downstream component.
-As data is parsed, improperly handled multiple trailing special elements may cause the process to take unexpected actions that result in an attack.",,unclassified,
-CWE-164,EN-Improper Neutralization of Internal Special Elements (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes internal special elements that could be interpreted in unexpected ways when they are sent to a downstream component.
-As data is parsed, improperly handled internal special elements may cause the process to take unexpected actions that result in an attack.",,unclassified,
-CWE-165,EN-Improper Neutralization of Multiple Internal Special Elements (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes multiple internal special elements that could be interpreted in unexpected ways when they are sent to a downstream component.
-As data is parsed, improperly handled multiple internal special elements may cause the process to take unexpected actions that result in an attack.",,unclassified,
-CWE-166,EN-Improper Handling of Missing Special Element (Type: Base),"The software receives input from an upstream component, but it does not handle or incorrectly handles when an expected special element is missing.
-As data is parsed, improperly handled multiple internal special elements may cause the process to take unexpected actions that result in an attack.",,unclassified,
-CWE-167,EN-Improper Handling of Additional Special Element (Type: Base),"The software receives input from an upstream component, but it does not handle or incorrectly handles when an additional unexpected special element is missing.
-As data is parsed, improperly handled multiple internal special elements may cause the process to take unexpected actions that result in an attack.",,unclassified,
-CWE-168,EN-Improper Handling of Inconsistent Special Elements (Type: Base),"The software does not handle when an inconsistency exists between two or more special characters or reserved words.
-An example of this problem would be if paired characters appear in the wrong order, or if the special characters are not properly nested.",,unclassified,
-CWE-172,EN-Encoding Error (Type: Class),"The software does not properly encode or decode the data, resulting in unexpected values.
-Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified,
-CWE-173,EN-Improper Handling of Alternate Encoding (Type: Variant),"The software does not properly handle when an input uses an alternate encoding that is valid for the control sphere to which the input is being sent.
-Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified,
-CWE-174,EN-Double Decoding of the Same Data (Type: Variant),"The software decodes the same input twice, which can limit the effectiveness of any protection mechanism that occurs in between the decoding operations.
-Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified,
-CWE-175,EN-Improper Handling of Mixed Encoding (Type: Variant),"The software does not properly handle when the same input uses several different (mixed) encodings.
-Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified,
-CWE-176,EN-Improper Handling of Unicode Encoding (Type: Variant),"The software does not properly handle when an input contains Unicode encoding.
-Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Character Sets and Unicode"", Page 446."
-CWE-177,EN-Improper Handling of URL Encoding (Hex Encoding) (Type: Variant),"The software does not properly handle when all or part of an input has been URL encoded.
-Null termination errors frequently occur in two different ways. An off-by-one error could cause a null to be written out of bounds, leading to an overflow. Or, a program could use a strncpy() function call incorrectly, which prevents a null terminator from being added at all. Other scenarios are possible.",,unclassified,
-CWE-178,EN-Improper Handling of Case Sensitivity (Type: Base),"The software does not properly account for differences in case sensitivity when accessing or determining the properties of a resource, leading to inconsistent results.
-Improperly handled case sensitive data can lead to several possible consequences, including:
-case-insensitive passwords reducing the size of the key space, making brute force attacks easier
-bypassing filters or access controls using alternate names
-multiple interpretation errors using alternate names.",,unclassified,
-CWE-179,EN-Incorrect Behavior Order: Early Validation (Type: Base),"The software validates input before applying protection mechanisms that modify the input, which could allow an attacker to bypass the validation via dangerous inputs that only arise after the modification.
-Software needs to validate data at the proper time, after data has been canonicalized and cleansed. Early validation is susceptible to various manipulations that result in dangerous inputs that are produced by canonicalization and cleansing.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Escaping Metacharacters"", Page 439."
-CWE-180,EN-Incorrect Behavior Order: Validate Before Canonicalize (Type: Base),"The software validates input before it is canonicalized, which prevents the software from detecting data that becomes invalid after the canonicalization step.
-This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,unclassified,
-CWE-181,EN-Incorrect Behavior Order: Validate Before Filter (Type: Base),"The software validates data before it has been filtered, which prevents the software from detecting data that becomes invalid after the filtering step.
-This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,unclassified,
-CWE-182,EN-Collapse of Data into Unsafe Value (Type: Base),"The software filters data in a way that causes it to be reduced or ""collapsed"" into an unsafe value that violates an expected security property.
-This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Character Stripping Vulnerabilities"", Page 437."
-CWE-183,EN-Permissive Whitelist (Type: Base),"An application uses a ""whitelist"" of acceptable values, but the whitelist includes at least one unsafe value, leading to resultant weaknesses.
-This can be used by an attacker to bypass the validation and launch attacks that expose weaknesses that would otherwise be prevented, such as injection.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Eliminating Metacharacters"", Page 435."
-CWE-184,EN-Incomplete Blacklist (Type: Base),"An application uses a ""blacklist"" of prohibited values, but the blacklist is incomplete.
-If an incomplete blacklist is used as a security mechanism, then the software may allow unintended values to pass into the application logic.",,unclassified,"Exploiting Software: How to Break Code
-Blacklist defenses as a breeding ground for vulnerability variants: http://seclists.org/fulldisclosure/2006/Feb/0040.html
-The Art of Software Security Assessment: Chapter 8, ""Eliminating Metacharacters"", Page 435."
-CWE-185,EN-Incorrect Regular Expression (Type: Class),"The software specifies a regular expression in a way that causes data to be improperly matched or compared.
-When the regular expression is used in protection mechanisms such as filtering or validation, this may allow an attacker to bypass the intended restrictions on the incoming data.",,unclassified,"Writing Secure Code: Chapter 10, ""Using Regular Expressions for Checking Input"" Page 350"
-CWE-186,EN-Overly Restrictive Regular Expression (Type: Base),"A regular expression is overly restrictive, which prevents dangerous values from being detected.
-When the regular expression is used in protection mechanisms such as filtering or validation, this may allow an attacker to bypass the intended restrictions on the incoming data.",,unclassified,
-CWE-187,EN-Partial Comparison (Type: Base),"The software performs a comparison that only examines a portion of a factor before determining whether there is a match, such as a substring, leading to resultant weaknesses.
-For example, an attacker might succeed in authentication by providing a small password that matches the associated portion of the larger, correct password.",,unclassified,
-CWE-191,EN-Integer Underflow (Wrap or Wraparound) (Type: Base),"The product subtracts one value from another, such that the result is less than the minimum allowable integer value, which produces a value that is not equal to the correct result.
-This can happen in signed and unsigned cases.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 7: Integer Overflows."" Page 119"
-CWE-193,EN-Off-by-one Error (Type: Base),"A product calculates or uses an incorrect maximum or minimum value that is 1 more, or 1 less, than the correct value.
-This can happen in signed and unsigned cases.",,unclassified,"Third Generation Exploits: http://www.blackhat.com/presentations/bh-europe-01/halvar-flake/bh-europe-01-halvarflake.ppt
-Off-by-one errors: a brief explanation: http://marc.theaimsgroup.com/?l=secprog&m=108379742110553&w=2
-The Frame Pointer Overwrite: http://kaizo.org/mirrors/phrack/phrack55/P55-08
-Exploiting Software: How to Break Code (The buffer overflow chapter)
-24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89
-The Art of Software Security Assessment: Chapter 5, ""Off-by-One Errors"", Page 180."
-CWE-195,EN-Signed to Unsigned Conversion Error (Type: Variant),"A signed-to-unsigned conversion error takes place when a signed primitive is used as an unsigned value, usually as a size variable.
-It is dangerous to rely on implicit casts between signed and unsigned numbers because the result can take on an unexpected value and violate assumptions made by the program.",,unclassified,"The Art of Software Security Assessment: Chapter 6, ""Type Conversions"", Page 223."
-CWE-198,EN-Use of Incorrect Byte Ordering (Type: Base),"The software receives input from an upstream component, but it does not account for byte ordering (e.g. big-endian and little-endian) when processing the input, causing an incorrect number or value to be used.
-When a primitive is cast to a smaller primitive, the high order bits of the large value are lost in the conversion, potentially resulting in an unexpected value that is not equal to the original value. This value may be required as an index into a buffer, a loop iterator, or simply necessary state data. In any case, the value cannot be trusted and the system will be in an undefined state. While this method may be employed viably to isolate the low bits of a value, this usage is rare, and truncation usually implies that an implementation error has occurred.",,unclassified,
-CWE-201,EN-Information Exposure Through Sent Data (Type: Variant),"The accidental exposure of sensitive information through sent data refers to the transmission of data which are either sensitive in and of itself or useful in the further exploitation of the system through standard data channels.
-The information either
-is regarded as sensitive within the product's own functionality, such as a private message; or
-provides information about the product or its environment that could be useful in an attack but is normally not available to the attacker, such as the installation path of a product that is remotely accessible.
-Many information exposures are resultant (e.g. PHP script error revealing the full path of the program), but they can also be primary (e.g. timing discrepancies in cryptography). There are many different types of problems that involve information exposures. Their severity can range widely depending on the type of information that is revealed.",,unclassified,
-CWE-203,EN-Information Exposure Through Discrepancy (Type: Class),"The product behaves differently or sends different responses in a way that exposes security-relevant information about the state of the product, such as whether a particular operation was successful or not.
-In situations where data should not be tied to individual users, but a large number of users should be able to make queries that ""scrub"" the identity of users, it may be possible to get information about a user -- e.g., by specifying search terms that are known to be unique to that user.",,unclassified,
-CWE-204,EN-Response Discrepancy Information Exposure (Type: Base),"The software provides different responses to incoming requests in a way that allows an actor to determine system state information that is outside of that actor's control sphere.
-This issue frequently occurs during authentication, where a difference in failed-login messages could allow an attacker to determine if the username is valid or not. These exposures can be inadvertent (bug) or intentional (design).",,unclassified,"24 Deadly Sins of Software Security: ""Sin 12: Information Leakage."" Page 191"
-CWE-205,EN-Information Exposure Through Behavioral Discrepancy (Type: Base),"The product's actions indicate important differences based on (1) the internal state of the product or (2) differences from other products in the same class.
-For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,unclassified,
-CWE-206,EN-Information Exposure of Internal State Through Behavioral Inconsistency (Type: Variant),"Two separate operations in a product cause the product to behave differently in a way that is observable to an attacker and reveals security-relevant information about the internal state of the product, such as whether a particular operation was successful or not.
-For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,unclassified,
-CWE-207,EN-Information Exposure Through an External Behavioral Inconsistency (Type: Variant),"The product behaves differently than other products like it, in a way that is observable to an attacker and exposes security-relevant information about which product is being used.
-For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,unclassified,
-CWE-208,EN-Information Exposure Through Timing Discrepancy (Type: Base),"Two separate operations in a product require different amounts of time to complete, in a way that is observable to an actor and reveals security-relevant information about the state of the product, such as whether a particular operation was successful or not.
-For example, attacks such as OS fingerprinting rely heavily on both behavioral and response discrepancies.",,unclassified,
-CWE-210,EN-Information Exposure Through Self-generated Error Message (Type: Base),"The software identifies an error condition and creates its own diagnostic or error messages that contain sensitive information.
-The sensitive information may be valuable information on its own (such as a password), or it may be useful for launching other, more deadly attacks. If an attack fails, an attacker may use error information provided by the server to launch another more focused attack. For example, an attempt to exploit a path traversal weakness (CWE-22) might yield the full pathname of the installed application. In turn, this could be used to select the proper number of "".."" sequences to navigate to the targeted file. An attack using SQL injection (CWE-89) might not initially succeed, but an error message could reveal the malformed query, which would expose query logic and possibly even passwords or other sensitive information used within the query.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 12: Information Leakage."" Page 191
-The Art of Software Security Assessment: Chapter 3, ""Overly Verbose Error Messages"", Page 75."
-CWE-211,EN-Information Exposure Through Externally-generated Error Message (Type: Base),"The software performs an operation that triggers an external diagnostic or error message that is not directly generated by the software, such as an error generated by the programming language interpreter that the software uses. The error can contain sensitive system information.
-The sensitive information may be valuable information on its own (such as a password), or it may be useful for launching other, more deadly attacks. If an attack fails, an attacker may use error information provided by the server to launch another more focused attack. For example, an attempt to exploit a path traversal weakness (CWE-22) might yield the full pathname of the installed application. In turn, this could be used to select the proper number of "".."" sequences to navigate to the targeted file. An attack using SQL injection (CWE-89) might not initially succeed, but an error message could reveal the malformed query, which would expose query logic and possibly even passwords or other sensitive information used within the query.",,unclassified,
-CWE-212,EN-Improper Cross-boundary Removal of Sensitive Data (Type: Base),"The software uses a resource that contains sensitive data, but it does not properly remove that data before it stores, transfers, or shares the resource with actors in another control sphere.
-Resources that may contain sensitive data include documents, packets, messages, databases, etc. While this data may be useful to an individual user or small set of users who share the resource, it may need to be removed before the resource can be shared outside of the trusted group. The process of removal is sometimes called cleansing or scrubbing.
-For example, software that is used for editing documents might not remove sensitive data such as reviewer comments or the local pathname where the document is stored. Or, a proxy might not remove an internal IP address from headers before making an outgoing request to an Internet site.",,unclassified,
-CWE-213,EN-Intentional Information Exposure (Type: Base),"A product's design or configuration explicitly requires the publication of information that could be regarded as sensitive by an administrator.
-Resources that may contain sensitive data include documents, packets, messages, databases, etc. While this data may be useful to an individual user or small set of users who share the resource, it may need to be removed before the resource can be shared outside of the trusted group. The process of removal is sometimes called cleansing or scrubbing.
-For example, software that is used for editing documents might not remove sensitive data such as reviewer comments or the local pathname where the document is stored. Or, a proxy might not remove an internal IP address from headers before making an outgoing request to an Internet site.",,unclassified,
-CWE-214,EN-Information Exposure Through Process Environment (Type: Variant),"A process is invoked with sensitive arguments, environment variables, or other elements that can be seen by other processes on the operating system.
-Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified,
-CWE-215,EN-Information Exposure Through Debug Information (Type: Variant),"The application contains debugging code that can expose sensitive information to untrusted parties.
-Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified,
-CWE-216,EN-Containment Errors (Container Errors) (Type: Class),"This tries to cover various problems in which improper data are included within a ""container.""
-Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified,
-CWE-217,EN-DEPRECATED: Failure to Protect Stored Data from Modification (Type: Base),"This weakness has been deprecated because it incorporated and confused multiple weaknesses. The issues formerly covered in this weakness can be found at CWE-766 and CWE-767.
-Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified,
-CWE-218,EN-DEPRECATED (Duplicate): Failure to provide confidentiality for stored data (Type: Base),"This weakness has been deprecated because it was a duplicate of CWE-493. All content has been transferred to CWE-493.
-Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified,
-CWE-219,EN-Sensitive Data Under Web Root (Type: Variant),"The application stores sensitive data under the web document root with insufficient access control, which might make it accessible to untrusted parties.
-Many operating systems allow a user to list information about processes that are owned by other users. This information could include command line arguments or environment variable settings. When this data contains sensitive information such as credentials, it might allow other users to launch an attack against the software or related resources.",,unclassified,
-CWE-220,EN-Sensitive Data Under FTP Root (Type: Variant),"The application stores sensitive data under the FTP document root with insufficient access control, which might make it accessible to untrusted parties.
-Many file operations are intended to take place within a restricted directory. By using special elements such as "".."" and ""/"" separators, attackers can escape outside of the restricted location to access files or directories that are elsewhere on the system. One of the most common special elements is the ""../"" sequence, which in most modern operating systems is interpreted as the parent directory of the current location. This is referred to as relative path traversal. Path traversal also covers the use of absolute pathnames such as ""/usr/local/bin"", which may also be useful in accessing unexpected files. This is referred to as absolute path traversal.
-In many programming languages, the injection of a null byte (the 0 or NUL) may allow an attacker to truncate a generated filename to widen the scope of attack. For example, the software may add "".txt"" to any pathname, thus limiting the attacker to text files, but a null injection may effectively remove this restriction.",,unclassified,
-CWE-221,EN-Information Loss or Omission (Type: Class),"The software does not record, or improperly records, security-relevant information that leads to an incorrect decision or hampers later analysis.
-This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified,
-CWE-222,EN-Truncation of Security-relevant Information (Type: Base),"The application truncates the display, recording, or processing of security-relevant information in a way that can obscure the source or nature of an attack.
-This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified,
-CWE-223,EN-Omission of Security-relevant Information (Type: Base),"The application does not record or display information that would be important for identifying the source or nature of an attack, or determining if an action is safe.
-This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified,"The Art of Software Security Assessment: Chapter 2, ""Accountability"", Page 40."
-CWE-224,EN-Obscured Security-relevant Information by Alternate Name (Type: Base),"The software records security-relevant information according to an alternate name of the affected entity, instead of the canonical name.
-This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified,Writing Secure Code
-CWE-225,EN-DEPRECATED (Duplicate): General Information Management Problems (Type: Base),"This weakness can be found at CWE-199.
-This can be resultant, e.g. a buffer overflow might trigger a crash before the product can log the event.",,unclassified,
-CWE-226,EN-Sensitive Information Uncleared Before Release (Type: Base),"The software does not fully clear previously used information in a data structure, file, or other resource, before making that resource available to a party in another control sphere.
-This typically results from new data that is not as long as the old data, which leaves portions of the old data still available. Equivalent errors can occur in other situations where the length of data is variable but the associated data structure is not. If memory is not cleared after use, it may allow unintended actors to read the data when the memory is reallocated.",,unclassified,
-CWE-227,EN-Improper Fulfillment of API Contract (API Abuse) (Type: Class),"The software uses an API in a manner contrary to its intended use.
-An API is a contract between a caller and a callee. The most common forms of API misuse occurs when the caller does not honor its end of this contract. For example, if a program does not call chdir() after calling chroot(), it violates the contract that specifies how to change the active root directory in a secure fashion. Another good example of library abuse is expecting the callee to return trustworthy DNS information to the caller. In this case, the caller misuses the callee API by making certain assumptions about its behavior (that the return value can be used for authentication purposes). One can also violate the caller-callee contract from the other side. For example, if a coder subclasses SecureRandom and returns a non-random value, the contract is violated.",,unclassified,
-CWE-228,EN-Improper Handling of Syntactically Invalid Structure (Type: Class),"The product does not handle or incorrectly handles input that is not syntactically well-formed with respect to the associated specification.
-An API is a contract between a caller and a callee. The most common forms of API misuse occurs when the caller does not honor its end of this contract. For example, if a program does not call chdir() after calling chroot(), it violates the contract that specifies how to change the active root directory in a secure fashion. Another good example of library abuse is expecting the callee to return trustworthy DNS information to the caller. In this case, the caller misuses the callee API by making certain assumptions about its behavior (that the return value can be used for authentication purposes). One can also violate the caller-callee contract from the other side. For example, if a coder subclasses SecureRandom and returns a non-random value, the contract is violated.",,unclassified,
-CWE-229,EN-Improper Handling of Values (Type: Base),"The software does not properly handle when the expected number of values for parameters, fields, or arguments is not provided in input, or if those values are undefined.
-An API is a contract between a caller and a callee. The most common forms of API misuse occurs when the caller does not honor its end of this contract. For example, if a program does not call chdir() after calling chroot(), it violates the contract that specifies how to change the active root directory in a secure fashion. Another good example of library abuse is expecting the callee to return trustworthy DNS information to the caller. In this case, the caller misuses the callee API by making certain assumptions about its behavior (that the return value can be used for authentication purposes). One can also violate the caller-callee contract from the other side. For example, if a coder subclasses SecureRandom and returns a non-random value, the contract is violated.",,unclassified,
-CWE-23,EN-Relative Path Traversal (Type: Base),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize sequences such as "".."" that can resolve to a location that is outside of that directory.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified,"OWASP Attack listing: http://www.owasp.org/index.php/Relative_Path_Traversal
-The Art of Software Security Assessment: Chapter 9, ""Filenames and Paths"", Page 503."
-CWE-230,EN-Improper Handling of Missing Values (Type: Variant),"The software does not handle or incorrectly handles when a parameter, field, or argument name is specified, but the associated value is missing, i.e. it is empty, blank, or null.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified,
-CWE-231,EN-Improper Handling of Extra Values (Type: Variant),"The software does not handle or incorrectly handles when more values are provided than expected.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified,
-CWE-232,EN-Improper Handling of Undefined Values (Type: Variant),"The software does not handle or incorrectly handles when a value is not defined or supported for the associated parameter, field, or argument name.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified,
-CWE-233,EN-Improper Handling of Parameters (Type: Base),"The software does not properly handle when the expected number of parameters, fields, or arguments is not provided in input, or if those parameters are undefined.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified,
-CWE-235,EN-Improper Handling of Extra Parameters (Type: Variant),"The software does not handle or incorrectly handles when the number of parameters, fields, or arguments with the same name exceeds the expected amount.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified,
-CWE-236,EN-Improper Handling of Undefined Parameters (Type: Variant),"The software does not handle or incorrectly handles when a particular parameter, field, or argument name is not defined or supported by the product.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified,
-CWE-237,EN-Improper Handling of Structural Elements (Type: Base),"The software does not handle or incorrectly handles inputs that are related to complex structures.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified,
-CWE-238,EN-Improper Handling of Incomplete Structural Elements (Type: Variant),"The software does not handle or incorrectly handles when a particular structural element is not completely specified.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified,
-CWE-239,EN-Failure to Handle Incomplete Element (Type: Variant),"The software does not properly handle when a particular element is not completely specified.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified,
-CWE-24,EN-Path Traversal: ../filedir (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize ""../"" sequences that can resolve to a location that is outside of that directory.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The ""../"" manipulation is the canonical manipulation for operating systems that use ""/"" as directory separators, such as UNIX- and Linux-based systems. In some cases, it is useful for bypassing protection schemes in environments for which ""/"" is supported but not the primary separator, such as Windows, which uses ""\"" but can also accept ""/"".",,unclassified,
-CWE-240,EN-Improper Handling of Inconsistent Structural Elements (Type: Variant),"The software does not handle or incorrectly handles when two or more structural elements should be consistent, but are not.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The ""../"" manipulation is the canonical manipulation for operating systems that use ""/"" as directory separators, such as UNIX- and Linux-based systems. In some cases, it is useful for bypassing protection schemes in environments for which ""/"" is supported but not the primary separator, such as Windows, which uses ""\"" but can also accept ""/"".",,unclassified,
-CWE-241,EN-Improper Handling of Unexpected Data Type (Type: Base),"The software does not handle or incorrectly handles when a particular element is not the expected type, e.g. it expects a digit (0-9) but is provided with a letter (A-Z).
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The ""../"" manipulation is the canonical manipulation for operating systems that use ""/"" as directory separators, such as UNIX- and Linux-based systems. In some cases, it is useful for bypassing protection schemes in environments for which ""/"" is supported but not the primary separator, such as Windows, which uses ""\"" but can also accept ""/"".",,unclassified,
-CWE-244,EN-Improper Clearing of Heap Memory Before Release (Heap Inspection) (Type: Variant),"Using realloc() to resize buffers that store sensitive information can leave the sensitive information exposed to attack, because it is not removed from memory.
-When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,unclassified,
-CWE-245,EN-J2EE Bad Practices: Direct Management of Connections (Type: Variant),"The J2EE application directly manages connections, instead of using the container's connection management facilities.
-When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,unclassified,
-CWE-246,EN-J2EE Bad Practices: Direct Use of Sockets (Type: Variant),"The J2EE application directly uses sockets instead of using framework method calls.
-When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,unclassified,
-CWE-247,EN-DEPRECATED (Duplicate): Reliance on DNS Lookups in a Security Decision (Type: Base),"This entry has been deprecated because it was a duplicate of CWE-350. All content has been transferred to CWE-350.
-When sensitive data such as a password or an encryption key is not removed from memory, it could be exposed to an attacker using a ""heap inspection"" attack that reads the sensitive data using memory dumps or other methods. The realloc() function is commonly used to increase the size of a block of allocated memory. This operation often requires copying the contents of the old memory block into a new and larger block. This operation leaves the contents of the original block intact but inaccessible to the program, preventing the program from being able to scrub sensitive data from memory. If an attacker can later examine the contents of a memory dump, the sensitive data could be exposed.",,unclassified,
-CWE-248,EN-Uncaught Exception (Type: Base),"An exception is thrown from a function, but it is not caught.
-When an exception is not caught, it may cause the program to crash or expose sensitive information.",,unclassified,
-CWE-249,EN-DEPRECATED: Often Misused: Path Manipulation (Type: Variant),"This entry has been deprecated because of name confusion and an accidental combination of multiple weaknesses. Most of its content has been transferred to CWE-785.
-When an exception is not caught, it may cause the program to crash or expose sensitive information.",,unclassified,
-CWE-25,EN-Path Traversal: /../filedir (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize ""/../"" sequences that can resolve to a location that is outside of that directory.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-Sometimes a program checks for ""../"" at the beginning of the input, so a ""/../"" can bypass that check.",,unclassified,
-CWE-26,EN-Path Traversal: /dir/../filename (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize ""/dir/../filename"" sequences that can resolve to a location that is outside of that directory.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The '/dir/../filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only checks for ""../"" at the beginning of the input, so a ""/../"" can bypass that check.",,unclassified,
-CWE-260,EN-Password in Configuration File (Type: Variant),"The software stores a password in a configuration file that might be accessible to actors who do not know the password.
-This can result in compromise of the system for which the password is used. An attacker could gain access to this file and learn the stored password or worse yet, change the password to one of their choosing.",,unclassified,Building Secure Software: How to Avoid Security Problems the Right Way
-CWE-261,EN-Weak Cryptography for Passwords (Type: Variant),"Obscuring a password with a trivial encoding does not protect the password.
-This can result in compromise of the system for which the password is used. An attacker could gain access to this file and learn the stored password or worse yet, change the password to one of their choosing.",,unclassified,"Building Secure Software: How to Avoid Security Problems the Right Way
-24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279"
-CWE-266,EN-Incorrect Privilege Assignment (Type: Base),"A product incorrectly assigns a privilege to a particular actor, creating an unintended sphere of control for that actor.
-Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,unclassified,Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html
-CWE-267,EN-Privilege Defined With Unsafe Actions (Type: Base),"A particular privilege, role, capability, or right can be used to perform unsafe actions that were not intended, even when it is assigned to the correct entity.
-Just as neglecting to include functionality for the management of password aging is dangerous, so is allowing password aging to continue unchecked. Passwords must be given a maximum life span, after which a user is required to update with a new and different password.",,unclassified,Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html
-CWE-27,EN-Path Traversal: dir/../../filename (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize multiple internal ""../"" sequences that can resolve to a location that is outside of that directory.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The 'directory/../../filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only removes one ""../"" sequence, so multiple ""../"" can bypass that check. Alternately, this manipulation could be used to bypass a check for ""../"" at the beginning of the pathname, moving up more than one directory level.",,unclassified,
-CWE-270,EN-Privilege Context Switching Error (Type: Base),"The software does not properly manage privileges while it is switching between different contexts that have different privileges or spheres of control.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The 'directory/../../filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only removes one ""../"" sequence, so multiple ""../"" can bypass that check. Alternately, this manipulation could be used to bypass a check for ""../"" at the beginning of the pathname, moving up more than one directory level.",,unclassified,"Writing Secure Code: Chapter 7, ""Running with Least Privilege"" Page 207
-Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html"
-CWE-272,EN-Least Privilege Violation (Type: Base),"The elevated privilege level required to perform operations such as chroot() should be dropped immediately after the operation is performed.
-In some contexts, a system executing with elevated permissions will hand off a process/file/etc. to another process or user. If the privileges of an entity are not reduced, then elevated privileges are spread throughout a system and possibly to an attacker.",,unclassified,
-CWE-274,EN-Improper Handling of Insufficient Privileges (Type: Base),"The software does not handle or incorrectly handles when it has insufficient privileges to perform an operation, leading to resultant weaknesses.
-If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,unclassified,
-CWE-277,EN-Insecure Inherited Permissions (Type: Variant),"A product defines a set of insecure permissions that are inherited by objects that are created by the program.
-If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,unclassified,
-CWE-278,EN-Insecure Preserved Inherited Permissions (Type: Variant),"A product inherits a set of insecure permissions for an object, e.g. when copying from an archive file, without user awareness or involvement.
-If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,unclassified,
-CWE-279,EN-Incorrect Execution-Assigned Permissions (Type: Variant),"While it is executing, the software sets the permissions of an object in a way that violates the intended permissions that have been specified by the user.
-If the drop fails, the software will continue to run with the raised privileges, which might provide additional access to unprivileged users.",,unclassified,
-CWE-28,EN-Path Traversal: ..\filedir (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize ""..\"" sequences that can resolve to a location that is outside of that directory.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified,
-CWE-280,EN-Improper Handling of Insufficient Permissions or Privileges (Type: Base),"The application does not handle or incorrectly handles when it has insufficient privileges to access resources or functionality as specified by their permissions. This may cause it to follow unexpected code paths that may leave the application in an invalid state.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified,
-CWE-281,EN-Improper Preservation of Permissions (Type: Base),"The software does not preserve permissions or incorrectly preserves permissions when copying, restoring, or sharing objects, which can cause them to have less restrictive permissions than intended.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified,
-CWE-282,EN-Improper Ownership Management (Type: Class),"The software assigns the wrong ownership, or does not properly verify the ownership, of an object or resource.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified,
-CWE-283,EN-Unverified Ownership (Type: Base),"The software does not properly verify that a critical resource is owned by the proper entity.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The '..\' manipulation is the canonical manipulation for operating systems that use ""\"" as directory separators, such as Windows. However, it is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified,
-CWE-284,EN-Improper Access Control (Type: Class),"The software does not restrict or incorrectly restricts access to a resource from an unauthorized actor.
-Access control involves the use of several protection mechanisms such as authentication (proving the identity of an actor) authorization (ensuring that a given actor can access a resource), and accountability (tracking of activities that were performed). When any mechanism is not applied or otherwise fails, attackers can compromise the security of the software by gaining privileges, reading sensitive information, executing commands, evading detection, etc.
-There are two distinct behaviors that can introduce access control weaknesses:
-Specification: incorrect privileges, permissions, ownership, etc. are explicitly specified for either the user or the resource (for example, setting a password file to be world-writable, or giving administrator capabilities to a guest user). This action could be performed by the program or the administrator.
-Enforcement: the mechanism contains errors that prevent it from properly enforcing the specified access control requirements (e.g., allowing the user to specify their own privileges, or allowing a syntactically-incorrect ACL to produce insecure settings). This problem occurs within the program itself, in that it does not actually enforce the intended security policy that the administrator specifies.",,unclassified,"Writing Secure Code: Chapter 6, ""Determining Appropriate Access Control"" Page 171
-24 Deadly Sins of Software Security: ""Sin 17: Failure to Protect Stored Data."" Page 253"
-CWE-286,EN-Incorrect User Management (Type: Class),"The software does not properly manage a user within its environment.
-Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,unclassified,
-CWE-288,EN-Authentication Bypass Using an Alternate Path or Channel (Type: Base),"A product requires authentication, but the product has an alternate path or channel that does not require authentication.
-Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,unclassified,
-CWE-289,EN-Authentication Bypass by Alternate Name (Type: Variant),"The software performs authentication based on the name of a resource being accessed, or the name of the actor performing the access, but it does not properly check all possible names for that resource or actor.
-Users can be assigned to the wrong group (class) of permissions resulting in unintended access rights to sensitive objects.",,unclassified,
-CWE-29,EN-Path Traversal: \..\filename (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '\..\filename' (leading backslash dot dot) sequences that can resolve to a location that is outside of that directory.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-This is similar to CWE-25, except using ""\"" instead of ""/"". Sometimes a program checks for ""..\"" at the beginning of the input, so a ""\..\"" can bypass that check. It is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified,
-CWE-290,EN-Authentication Bypass by Spoofing (Type: Base),"This attack-focused weakness is caused by improperly implemented authentication schemes that are subject to spoofing attacks.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-This is similar to CWE-25, except using ""\"" instead of ""/"". Sometimes a program checks for ""..\"" at the beginning of the input, so a ""\..\"" can bypass that check. It is also useful for bypassing path traversal protection schemes that only assume that the ""/"" separator is valid.",,unclassified,"The Art of Software Security Assessment: Chapter 3, ""Spoofing and Identification"", Page 72."
-CWE-295,EN-Improper Certificate Validation (Type: Base),"The software does not validate, or incorrectly validates, a certificate.
-When a certificate is invalid or malicious, it might allow an attacker to spoof a trusted entity by using a man-in-the-middle (MITM) attack. The software might connect to a malicious host while believing it is a trusted host, or the software might be deceived into accepting spoofed data that appears to originate from a trusted host.",,unclassified,"Why Eve and Mallory Love Android: An Analysis of Android SSL (In)Security: http://www2.dcsec.uni-hannover.de/files/android/p50-fahl.pdf
-Computer Security: Art and Science"
-CWE-30,EN-Path Traversal: \dir\..\filename (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '\dir\..\filename' (leading backslash dot dot) sequences that can resolve to a location that is outside of that directory.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-This is similar to CWE-26, except using ""\"" instead of ""/"". The '\dir\..\filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only checks for ""..\"" at the beginning of the input, so a ""\..\"" can bypass that check.",,unclassified,
-CWE-300,EN-Channel Accessible by Non-Endpoint (Man-in-the-Middle) (Type: Class),"The product does not adequately verify the identity of actors at both ends of a communication channel, or does not adequately ensure the integrity of the channel, in a way that allows the channel to be accessed or influenced by an actor that is not an endpoint.
-In order to establish secure communication between two parties, it is often important to adequately verify the identity of entities at each end of the communication channel. Inadequate or inconsistent verification may result in insufficient or incorrect identification of either communicating entity. This can have negative consequences such as misplaced trust in the entity at the other end of the channel. An attacker can leverage this by interposing between the communicating entities and masquerading as the original entity. In the absence of sufficient verification of identity, such an attacker can eavesdrop and potentially modify the communication between the original entities.",,unclassified,Computer Security: Art and Science
-CWE-302,EN-Authentication Bypass by Assumed-Immutable Data (Type: Variant),"The authentication scheme or implementation uses key data elements that are assumed to be immutable, but can be controlled or modified by the attacker.
-A mutual authentication protocol requires each party to respond to a random challenge by the other party by encrypting it with a pre-shared key. Often, however, such protocols employ the same pre-shared key for communication with a number of different entities. A malicious user or an attacker can easily compromise this protocol without possessing the correct key by employing a reflection attack on the protocol.",,unclassified,
-CWE-303,EN-Incorrect Implementation of Authentication Algorithm (Type: Base),"The requirements for the software dictate the use of an established authentication algorithm, but the implementation of the algorithm is incorrect.
-This incorrect implementation may allow authentication to be bypassed.",,unclassified,
-CWE-304,EN-Missing Critical Step in Authentication (Type: Base),"The software implements an authentication technique, but it skips a step that weakens the technique.
-Authentication techniques should follow the algorithms that define them exactly, otherwise authentication can be bypassed or more easily subjected to brute force attacks.",,unclassified,
-CWE-305,EN-Authentication Bypass by Primary Weakness (Type: Base),"The authentication algorithm is sound, but the implemented mechanism can be bypassed as the result of a separate weakness that is primary to the authentication error.
-Authentication techniques should follow the algorithms that define them exactly, otherwise authentication can be bypassed or more easily subjected to brute force attacks.",,unclassified,
-CWE-307,EN-Improper Restriction of Excessive Authentication Attempts (Type: Base),"The software does not implement sufficient measures to prevent multiple failed authentication attempts within in a short time frame, making it more susceptible to brute force attacks.
-Authentication techniques should follow the algorithms that define them exactly, otherwise authentication can be bypassed or more easily subjected to brute force attacks.",,unclassified,"Weak Password Brings 'Happiness' to Twitter Hacker: http://www.wired.com/threatlevel/2009/01/professed-twitt/
-OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI"
-CWE-31,EN-Path Traversal: dir\..\..\filename (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize 'dir\..\..\filename' (multiple internal backslash dot dot) sequences that can resolve to a location that is outside of that directory.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The 'dir\..\..\filename' manipulation is useful for bypassing some path traversal protection schemes. Sometimes a program only removes one ""..\"" sequence, so multiple ""..\"" can bypass that check. Alternately, this manipulation could be used to bypass a check for ""..\"" at the beginning of the pathname, moving up more than one directory level.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299"
-CWE-312,EN-Cleartext Storage of Sensitive Information (Type: Base),"The application stores sensitive information in cleartext within a resource that might be accessible to another control sphere.
-Because the information is stored in cleartext, attackers could potentially read it. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified,"Writing Secure Code: Chapter 9, ""Protecting Secret Data"" Page 299
-The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Encryption"", Page 43.
-Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/"
-CWE-313,EN-Cleartext Storage in a File or on Disk (Type: Variant),"The application stores sensitive information in cleartext in a file, or on disk.
-The sensitive information could be read by attackers with access to the file, or with physical or administrator access to the raw disk. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified,
-CWE-314,EN-Cleartext Storage in the Registry (Type: Variant),"The application stores sensitive information in cleartext in the registry.
-Attackers can read the information by accessing the registry key. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified,
-CWE-315,EN-Cleartext Storage of Sensitive Information in a Cookie (Type: Variant),"The application stores sensitive information in cleartext in a cookie.
-Attackers can use widely-available tools to view the cookie and read the sensitive information. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified,
-CWE-316,EN-Cleartext Storage of Sensitive Information in Memory (Type: Variant),"The application stores sensitive information in cleartext in memory.
-The sensitive memory might be saved to disk, stored in a core dump, or remain uncleared if the application crashes, or if the programmer does not properly clear the memory before freeing it.
-It could be argued that such problems are usually only exploitable by those with administrator privileges. However, swapping could cause the memory to be written to disk and leave it accessible to physical attack afterwards. Core dump files might have insecure permissions or be stored in archive files that are accessible to untrusted people. Or, uncleared sensitive memory might be inadvertently exposed to attackers due to another weakness.",,unclassified,
-CWE-317,EN-Cleartext Storage of Sensitive Information in GUI (Type: Variant),"The application stores sensitive information in cleartext within the GUI.
-An attacker can often obtain data from a GUI, even if hidden, by using an API to directly access GUI objects such as windows and menus. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified,
-CWE-318,EN-Cleartext Storage of Sensitive Information in Executable (Type: Variant),"The application stores sensitive information in cleartext in an executable.
-Attackers can reverse engineer binary code to obtain secret data. This is especially easy when the cleartext is plain ASCII. Even if the information is encoded in a way that is not human-readable, certain techniques could determine which encoding is being used, then decode the information.",,unclassified,
-CWE-32,EN-Path Traversal: ... (Triple Dot) (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '...' (triple dot) sequences that can resolve to a location that is outside of that directory.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The '...' manipulation is useful for bypassing some path traversal protection schemes. On some Windows systems, it is equivalent to ""..\.."" and might bypass checks that assume only two dots are valid. Incomplete filtering, such as removal of ""./"" sequences, can ultimately produce valid "".."" sequences due to a collapse into unsafe value (CWE-182).",,unclassified,
-CWE-325,EN-Missing Required Cryptographic Step (Type: Base),"The software does not implement a required step in a cryptographic algorithm, resulting in weaker encryption than advertised by that algorithm.
-Cryptographic implementations should follow the algorithms that define them exactly, otherwise encryption can be weaker than expected.",,unclassified,
-CWE-326,EN-Inadequate Encryption Strength (Type: Class),"The software stores or transmits sensitive data using an encryption scheme that is theoretically sound, but is not strong enough for the level of protection required.
-A weak encryption scheme can be subjected to brute force attacks that have a reasonable chance of succeeding using current attack methods and resources.",,unclassified,"Writing Secure Code: Chapter 8, ""Cryptographic Foibles"" Page 259
-24 Deadly Sins of Software Security: ""Sin 21: Using the Wrong Cryptography."" Page 315"
-CWE-328,EN-Reversible One-Way Hash (Type: Base),"The product uses a hashing algorithm that produces a hash value that can be used to determine the original input, or to find an input that can produce the same hash, more efficiently than brute force techniques.
-This weakness is especially dangerous when the hash is used in security algorithms that require the one-way property to hold. For example, if an authentication system takes an incoming password and generates a hash, then compares the hash to another hash that it has stored in its authentication database, then the ability to create a collision could allow an attacker to provide an alternate password that produces the same target hash, bypassing authentication.",,unclassified,"MD5 considered harmful today: http://www.phreedom.org/research/rogue-ca/
-The Art of Software Security Assessment: Chapter 2, ""Common Vulnerabilities of Integrity"", Page 47.
-RFC2898 - PKCS #5: Password-Based Cryptography Specification Version 2.0: http://tools.ietf.org/html/rfc2898
-How To Safely Store A Password: http://codahale.com/how-to-safely-store-a-password/
-Tarsnap - The scrypt key derivation function and encryption utility: http://www.tarsnap.com/scrypt.html
-How Companies Can Beef Up Password Security (interview with Thomas H. Ptacek): http://krebsonsecurity.com/2012/06/how-companies-can-beef-up-password-security/
-Password security: past, present, future: http://www.openwall.com/presentations/PHDays2012-Password-Security/
-Our password hashing has no clothes: http://www.troyhunt.com/2012/06/our-password-hashing-has-no-clothes.html
-Should we really use bcrypt/scrypt?: http://www.analyticalengine.net/2012/06/should-we-really-use-bcryptscrypt/"
-CWE-33,EN-Path Traversal: .... (Multiple Dot) (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '....' (multiple dot) sequences that can resolve to a location that is outside of that directory.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The '....' manipulation is useful for bypassing some path traversal protection schemes. On some Windows systems, it is equivalent to ""..\..\.."" and might bypass checks that assume only two dots are valid. Incomplete filtering, such as removal of ""./"" sequences, can ultimately produce valid "".."" sequences due to a collapse into unsafe value (CWE-182).",,unclassified,
-CWE-331,EN-Insufficient Entropy (Type: Base),"The software uses an algorithm or scheme that produces insufficient entropy, leaving patterns or clusters of values that are more likely to occur than others.
-When software generates predictable values in a context requiring unpredictability, it may be possible for an attacker to guess the next value that will be generated, and use this guess to impersonate another user or access sensitive information.",,unclassified,Building Secure Software: How to Avoid Security Problems the Right Way
-CWE-334,EN-Small Space of Random Values (Type: Base),"The number of possible random values is smaller than needed by the product, making it more susceptible to brute force attacks.
-The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,unclassified,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf
-24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299"
-CWE-335,EN-PRNG Seed Error (Type: Class),"A Pseudo-Random Number Generator (PRNG) uses seeds incorrectly.
-The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299"
-CWE-336,EN-Same Seed in PRNG (Type: Base),"A PRNG uses the same seed each time the product is initialized. If an attacker can guess (or knows) the seed, then he/she may be able to determine the ""random"" number produced from the PRNG.
-The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,unclassified,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf
-CWE-337,EN-Predictable Seed in PRNG (Type: Base),"A PRNG is initialized from a predictable seed, e.g. using process ID or system time.
-The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,unclassified,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf
-24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299"
-CWE-339,EN-Small Seed Space in PRNG (Type: Base),"A PRNG uses a relatively small space of seeds.
-The rate at which true random numbers can be generated is limited. It is important that one uses them only when they are needed for security.",,unclassified,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf
-CWE-34,EN-Path Traversal: ....// (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '....//' (doubled dot dot slash) sequences that can resolve to a location that is outside of that directory.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The '....//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then ""....//"" can collapse into the ""../"" unsafe value (CWE-182). It could also be useful when "".."" is removed, if the operating system treats ""//"" and ""/"" as equivalent.",,unclassified,
-CWE-340,EN-Predictability Problems (Type: Class),"Weaknesses in this category are related to schemes that generate numbers or identifiers that are more predictable than required by the application.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The '....//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then ""....//"" can collapse into the ""../"" unsafe value (CWE-182). It could also be useful when "".."" is removed, if the operating system treats ""//"" and ""/"" as equivalent.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299"
-CWE-341,EN-Predictable from Observable State (Type: Base),"A number or object is predictable based on observations that the attacker can make about the state of the system or network, such as time, process ID, etc.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The '....//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then ""....//"" can collapse into the ""../"" unsafe value (CWE-182). It could also be useful when "".."" is removed, if the operating system treats ""//"" and ""/"" as equivalent.",,unclassified,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf
-24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299"
-CWE-342,EN-Predictable Exact Value from Previous Values (Type: Base),"An exact value or random number can be precisely predicted by observing previous values.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The '....//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then ""....//"" can collapse into the ""../"" unsafe value (CWE-182). It could also be useful when "".."" is removed, if the operating system treats ""//"" and ""/"" as equivalent.",,unclassified,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf
-24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299"
-CWE-343,EN-Predictable Value Range from Previous Values (Type: Base),"The software's random number generator produces a series of values which, when observed, can be used to infer a relatively small range of possibilities for the next value that could be generated.
-The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified,"SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf
-Strange Attractors and TCP/IP Sequence Number Analysis: http://www.bindview.com/Services/Razor/Papers/2001/tcpseq.cfm
-24 Deadly Sins of Software Security: ""Sin 20: Weak Random Numbers."" Page 299"
-CWE-344,EN-Use of Invariant Value in Dynamically Changing Context (Type: Base),"The product uses a constant value, name, or reference, but this value can (or should) vary across different environments.
-The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified,SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf
-CWE-345,EN-Insufficient Verification of Data Authenticity (Type: Class),"The software does not sufficiently verify the origin or authenticity of data, in a way that causes it to accept invalid data.
-The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 15: Not Updating Easily."" Page 231"
-CWE-346,EN-Origin Validation Error (Type: Base),"The software does not properly verify that the source of data or communication is valid.
-The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified,
-CWE-347,EN-Improper Verification of Cryptographic Signature (Type: Base),"The software does not verify, or incorrectly verifies, the cryptographic signature for data.
-The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified,
-CWE-348,EN-Use of Less Trusted Source (Type: Base),"The software has two different sources of the same data or information, but it uses the source that has less support for verification, is less trusted, or is less resistant to attack.
-The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified,
-CWE-349,EN-Acceptance of Extraneous Untrusted Data With Trusted Data (Type: Base),"The software, when processing trusted data, accepts any untrusted data that is also included with the trusted data, treating the untrusted data as if it were trusted.
-The output of a random number generator should not be predictable based on observations of previous values. In some cases, an attacker cannot predict the exact value that will be produced next, but can narrow down the possibilities significantly. This reduces the amount of effort to perform a brute force attack. For example, suppose the product generates random numbers between 1 and 100, but it always produces a larger value until it reaches 100. If the generator produces an 80, then the attacker knows that the next value will be somewhere between 81 and 100. Instead of 100 possibilities, the attacker only needs to consider 20.",,unclassified,
-CWE-35,EN-Path Traversal: .../...// (Type: Variant),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '.../...//' (doubled triple dot slash) sequences that can resolve to a location that is outside of that directory.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.
-The '.../...//' manipulation is useful for bypassing some path traversal protection schemes. If ""../"" is filtered in a sequential fashion, as done by some regular expression engines, then "".../...//"" can collapse into the ""../"" unsafe value (CWE-182). Removing the first ""../"" yields ""....//""; the second removal yields ""../"". Depending on the algorithm, the software could be susceptible to CWE-34 but not CWE-35, or vice versa.",,unclassified,
-CWE-350,EN-Reliance on Reverse DNS Resolution for a Security-Critical Action (Type: Variant),"The software performs reverse DNS resolution on an IP address to obtain the hostname and make a security decision, but it does not properly ensure that the IP address is truly associated with the hostname.
-When the software performs a reverse DNS resolution for an IP address, if an attacker controls the server for that IP address, then the attacker can cause the server to return an arbitrary hostname. As a result, the attacker may be able to bypass authentication, cause the wrong hostname to be recorded in log files to hide activities, or perform other attacks.
-Attackers can spoof DNS names by either (1) compromising a DNS server and modifying its records (sometimes called DNS cache poisoning), or (2) having legitimate control over a DNS server associated with their IP address.
-Since DNS names can be easily spoofed or misreported, and it may be difficult for the software to detect if a trusted DNS server has not been compromised, they do not constitute a valid authentication mechanism.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 15: Not Updating Easily."" Page 231
-24 Deadly Sins of Software Security: ""Sin 24: Trusting Network Name Resolution."" Page 361
-The Art of Software Security Assessment: Chapter 16, ""DNS Spoofing"", Page 1002."
-CWE-351,EN-Insufficient Type Distinction (Type: Base),"The software does not properly distinguish between different types of elements in a way that leads to insecure behavior.
-When the software performs a reverse DNS resolution for an IP address, if an attacker controls the server for that IP address, then the attacker can cause the server to return an arbitrary hostname. As a result, the attacker may be able to bypass authentication, cause the wrong hostname to be recorded in log files to hide activities, or perform other attacks.
-Attackers can spoof DNS names by either (1) compromising a DNS server and modifying its records (sometimes called DNS cache poisoning), or (2) having legitimate control over a DNS server associated with their IP address.
-Since DNS names can be easily spoofed or misreported, and it may be difficult for the software to detect if a trusted DNS server has not been compromised, they do not constitute a valid authentication mechanism.",,unclassified,
-CWE-356,EN-Product UI does not Warn User of Unsafe Actions (Type: Base),"The software's user interface does not warn the user before undertaking an unsafe action on behalf of that user. This makes it easier for attackers to trick users into inflicting damage to their system.
-Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,unclassified,
-CWE-357,EN-Insufficient UI Warning of Dangerous Operations (Type: Base),"The user interface provides a warning to a user regarding dangerous or sensitive operations, but the warning is not noticeable enough to warrant attention.
-Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,unclassified,
-CWE-358,EN-Improperly Implemented Security Check for Standard (Type: Base),"The software does not implement or incorrectly implements one or more security-relevant checks as specified by the design of a standardized algorithm, protocol, or technique.
-Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,unclassified,
-CWE-359,EN-Privacy Violation (Type: Class),"Mishandling private information, such as customer passwords or social security numbers, can compromise user privacy and is often illegal.
-Software systems should warn users that a potentially dangerous action may occur if the user proceeds. For example, if the user downloads a file from an unknown source and attempts to execute the file on their machine, then the application's GUI can indicate that the file is unsafe.",,unclassified,"AOL man pleads guilty to selling 92m email addies: http://www.theregister.co.uk/2005/02/07/aol_email_theft/
-Safe Harbor Privacy Framework: http://www.export.gov/safeharbor/
-Financial Privacy: The Gramm-Leach Bliley Act (GLBA): http://www.ftc.gov/privacy/glbact/index.html
-Health Insurance Portability and Accountability Act (HIPAA): http://www.hhs.gov/ocr/hipaa/
-California SB-1386: http://info.sen.ca.gov/pub/01-02/bill/sen/sb_1351-1400/sb_1386_bill_20020926_chaptered.html
-SECURITY REQUIREMENTS FOR CRYPTOGRAPHIC MODULES: http://csrc.nist.gov/publications/fips/fips140-2/fips1402.pdf
-Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/"
-CWE-36,EN-Absolute Path Traversal (Type: Base),"The software uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize absolute path sequences such as ""/abs/path"" that can resolve to a location that is outside of that directory.
-This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory.",,unclassified,"The Art of Software Security Assessment: Chapter 9, ""Filenames and Paths"", Page 503."
-CWE-363,EN-Race Condition Enabling Link Following (Type: Base),"The software checks the status of a file or directory before accessing it, which produces a race condition in which the file can be replaced with a link before the access is performed, causing the software to access the wrong file.
-While developers might expect that there is a very narrow time window between the time of check and time of use, there is still a race condition. An attacker could cause the software to slow down (e.g. with memory consumption), causing the time window to become larger. Alternately, in some situations, the attacker could win the race by performing a large number of attacks.",,unclassified,"The Art of Software Security Assessment: Chapter 9, ""Race Conditions"", Page 526."
-CWE-368,EN-Context Switching Race Condition (Type: Base),"A product performs a series of non-atomic actions to switch between contexts that cross privilege or other security boundaries, but a race condition allows an attacker to modify or misrepresent the product's behavior during the switch.
-This is commonly seen in web browser vulnerabilities in which the attacker can perform certain actions while the browser is transitioning from a trusted to an untrusted domain, or vice versa, and the browser performs the actions on one domain using the trust level and resources of the other domain.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205"
-CWE-37,EN-Path Traversal: /absolute/pathname/here (Type: Variant),"A software system that accepts input in the form of a slash absolute path ('/absolute/pathname/here') without appropriate validation can allow an attacker to traverse the file system to unintended locations or access arbitrary files.
-This weakness typically occurs when an unexpected value is provided to the product, or if an error occurs that is not properly detected. It frequently occurs in calculations involving physical dimensions such as size, length, width, and height.",,unclassified,
-CWE-372,EN-Incomplete Internal State Distinction (Type: Base),"The software does not properly determine which state it is in, causing it to assume it is in state X when in fact it is in state Y, causing it to perform incorrect operations in a security-relevant manner.
-If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,unclassified,
-CWE-373,EN-DEPRECATED: State Synchronization Error (Type: Base),"This entry was deprecated because it overlapped the same concepts as race condition (CWE-362) and Improper Synchronization (CWE-662).
-If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,unclassified,
-CWE-377,EN-Insecure Temporary File (Type: Base),"Creating and using insecure temporary files can leave application and system data vulnerable to attack.
-If the revocation status of a certificate is not checked before each action that requires privileges, the system may be subject to a race condition. If a certificate is revoked after the initial check, all subsequent actions taken with the owner of the revoked certificate will lose all benefits guaranteed by the certificate. In fact, it is almost certain that the use of a revoked certificate indicates malicious activity.",,unclassified,"Writing Secure Code: Chapter 23, ""Creating Temporary Files Securely"" Page 682
-The Art of Software Security Assessment: Chapter 9, ""Temporary Files"", Page 538.
-The Art of Software Security Assessment: Chapter 11, ""File Squatting"", Page 662."
-CWE-38,EN-Path Traversal: \absolute\pathname\here (Type: Variant),"A software system that accepts input in the form of a backslash absolute path ('\absolute\pathname\here') without appropriate validation can allow an attacker to traverse the file system to unintended locations or access arbitrary files.
-On some operating systems, the fact that the temporary file exists may be apparent to any user with sufficient privileges to access that directory. Since the file is visible, the application that is using the temporary file could be known. If one has access to list the processes on the system, the attacker has gained information about what the user is doing at that time. By correlating this with the applications the user is running, an attacker could potentially discover what a user's actions are. From this, higher levels of security could be breached.",,unclassified,
-CWE-382,EN-J2EE Bad Practices: Use of System.exit() (Type: Variant),"A J2EE application uses System.exit(), which also shuts down its container.
-On some operating systems, the fact that the temporary file exists may be apparent to any user with sufficient privileges to access that directory. Since the file is visible, the application that is using the temporary file could be known. If one has access to list the processes on the system, the attacker has gained information about what the user is doing at that time. By correlating this with the applications the user is running, an attacker could potentially discover what a user's actions are. From this, higher levels of security could be breached.",,unclassified,
-CWE-383,EN-J2EE Bad Practices: Direct Use of Threads (Type: Variant),"Thread management in a Web application is forbidden in some circumstances and is always highly error prone.
-Thread management in a web application is forbidden by the J2EE standard in some circumstances and is always highly error prone. Managing threads is difficult and is likely to interfere in unpredictable ways with the behavior of the application container. Even without interfering with the container, thread management usually leads to bugs that are hard to detect and diagnose like deadlock, race conditions, and other synchronization errors.",,unclassified,
-CWE-386,EN-Symbolic Name not Mapping to Correct Object (Type: Base),"A constant symbolic reference to an object is used, even though the reference can resolve to a different object over time.
-In some instances, knowing when data is transmitted between parties can provide a malicious user with privileged information. Also, externally monitoring the timing of operations can potentially reveal sensitive data. For example, a cryptographic operation can expose its internal state if the time it takes to perform the operation varies, based on the state.
-Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,unclassified,
-CWE-39,EN-Path Traversal: C:dirname (Type: Variant),"An attacker can inject a drive letter or Windows volume letter ('C:dirname') into a software system to potentially redirect access to an unintended location or arbitrary file.
-In some instances, knowing when data is transmitted between parties can provide a malicious user with privileged information. Also, externally monitoring the timing of operations can potentially reveal sensitive data. For example, a cryptographic operation can expose its internal state if the time it takes to perform the operation varies, based on the state.
-Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,unclassified,
-CWE-392,EN-Missing Report of Error Condition (Type: Base),"The software encounters an error but does not provide a status code or return value to indicate that an error has occurred.
-In some instances, knowing when data is transmitted between parties can provide a malicious user with privileged information. Also, externally monitoring the timing of operations can potentially reveal sensitive data. For example, a cryptographic operation can expose its internal state if the time it takes to perform the operation varies, based on the state.
-Covert channels are frequently classified as either storage or timing channels. Some examples of covert timing channels are the system's paging rate, the time a certain transaction requires to execute, and the time it takes to gain access to a shared bus.",,unclassified,
-CWE-393,EN-Return of Wrong Status Code (Type: Base),"A function or operation returns an incorrect return value or status code that does not indicate an error, but causes the product to modify its behavior based on the incorrect result.
-This can lead to unpredictable behavior. If the function is used to make security-critical decisions or provide security-critical information, then the wrong status code can cause the software to assume that an action is safe, even when it is not.",,unclassified,
-CWE-394,EN-Unexpected Status Code or Return Value (Type: Base),"The software does not properly check when a function or operation returns a value that is legitimate for the function, but is not expected by the software.
-This can lead to unpredictable behavior. If the function is used to make security-critical decisions or provide security-critical information, then the wrong status code can cause the software to assume that an action is safe, even when it is not.",,unclassified,
-CWE-395,EN-Use of NullPointerException Catch to Detect NULL Pointer Dereference (Type: Base),"Catching NullPointerException should not be used as an alternative to programmatic checks to prevent dereferencing a null pointer.
-This can lead to unpredictable behavior. If the function is used to make security-critical decisions or provide security-critical information, then the wrong status code can cause the software to assume that an action is safe, even when it is not.",,unclassified,
-CWE-396,EN-Declaration of Catch for Generic Exception (Type: Base),"Catching overly broad exceptions promotes complex error handling code that is more likely to contain security vulnerabilities.
-Multiple catch blocks can get ugly and repetitive, but ""condensing"" catch blocks by catching a high-level class like Exception can obscure exceptions that deserve special treatment or that should not be caught at this point in the program. Catching an overly broad exception essentially defeats the purpose of Java's typed exceptions, and can become particularly dangerous if the program grows and begins to throw new types of exceptions. The new exception types will not receive any attention.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 9: Catching Exceptions."" Page 157"
-CWE-397,EN-Declaration of Throws for Generic Exception (Type: Base),"Throwing overly broad exceptions promotes complex error handling code that is more likely to contain security vulnerabilities.
-Declaring a method to throw Exception or Throwable makes it difficult for callers to perform proper error handling and error recovery. Java's exception mechanism, for example, is set up to make it easy for callers to anticipate what can go wrong and write code to handle each specific exceptional circumstance. Declaring that a method throws a generic form of exception defeats this system.",,unclassified,
-CWE-398,EN-Indicator of Poor Code Quality (Type: Class),"The code has features that do not directly introduce a weakness or vulnerability, but indicate that the product has not been carefully developed or maintained.
-Programs are more likely to be secure when good development practices are followed. If a program is complex, difficult to maintain, not portable, or shows evidence of neglect, then there is a higher likelihood that weaknesses are buried in the code.",,unclassified,
-CWE-40,EN-Path Traversal: \\UNC\share\name\ (Windows UNC Share) (Type: Variant),"An attacker can inject a Windows UNC share ('\\UNC\share\name') into a software system to potentially redirect access to an unintended location or arbitrary file.
-Programs are more likely to be secure when good development practices are followed. If a program is complex, difficult to maintain, not portable, or shows evidence of neglect, then there is a higher likelihood that weaknesses are buried in the code.",,unclassified,"The Art of Software Security Assessment: Chapter 11, ""Filelike Objects"", Page 664."
-CWE-402,EN-Transmission of Private Resources into a New Sphere (Resource Leak) (Type: Class),"The software makes resources available to untrusted parties when those resources are only intended to be accessed by the software.
-This is often triggered by improper handling of malformed data or unexpectedly interrupted sessions.",,unclassified,
-CWE-403,EN-Exposure of File Descriptor to Unintended Control Sphere (File Descriptor Leak) (Type: Base),"A process does not close sensitive file descriptors before invoking a child process, which allows the child to perform unauthorized I/O operations using those descriptors.
-When a new process is forked or executed, the child process inherits any open file descriptors. When the child process has fewer privileges than the parent process, this might introduce a vulnerability if the child process can access the file descriptor but does not have the privileges to access the associated file.",,unclassified,"File descriptors and setuid applications: https://blogs.oracle.com/paulr/entry/file_descriptors_and_setuid_applications
-Introduction to Secure Coding Guide: https://developer.apple.com/library/mac/#documentation/security/conceptual/SecureCodingGuide/Articles/AccessControl.html"
-CWE-405,EN-Asymmetric Resource Consumption (Amplification) (Type: Class),"Software that does not appropriately monitor or control resource consumption can lead to adverse system performance.
-This situation is amplified if the software allows malicious users or attackers to consume more resources than their access level permits. Exploiting such a weakness can lead to asymmetric resource consumption, aiding in amplification attacks against the system or the network.",,unclassified,
-CWE-406,EN-Insufficient Control of Network Message Volume (Network Amplification) (Type: Base),"The software does not sufficiently monitor or control transmitted network traffic volume, so that an actor can cause the software to transmit more traffic than should be allowed for that actor.
-In the absence of a policy to restrict asymmetric resource consumption, the application or system cannot distinguish between legitimate transmissions and traffic intended to serve as an amplifying attack on target systems. Systems can often be configured to restrict the amount of traffic sent out on behalf of a client, based on the client's origin or access level. This is usually defined in a resource allocation policy. In the absence of a mechanism to keep track of transmissions, the system or application can be easily abused to transmit asymmetrically greater traffic than the request or client should be permitted to.",,unclassified,
-CWE-408,EN-Incorrect Behavior Order: Early Amplification (Type: Base),"The software allows an entity to perform a legitimate but expensive operation before authentication or authorization has taken place.
-In the absence of a policy to restrict asymmetric resource consumption, the application or system cannot distinguish between legitimate transmissions and traffic intended to serve as an amplifying attack on target systems. Systems can often be configured to restrict the amount of traffic sent out on behalf of a client, based on the client's origin or access level. This is usually defined in a resource allocation policy. In the absence of a mechanism to keep track of transmissions, the system or application can be easily abused to transmit asymmetrically greater traffic than the request or client should be permitted to.",,unclassified,
-CWE-409,EN-Improper Handling of Highly Compressed Data (Data Amplification) (Type: Base),"The software does not handle or incorrectly handles a compressed input with a very high compression ratio that produces a large output.
-An example of data amplification is a ""decompression bomb,"" a small ZIP file that can produce a large amount of data when it is decompressed.",,unclassified,
-CWE-41,EN-Improper Resolution of Path Equivalence (Type: Base),"The system or application is vulnerable to file system contents disclosure through path equivalence. Path equivalence involves the use of special characters in file and directory names. The associated manipulations are intended to generate multiple names for the same object.
-Path equivalence is usually employed in order to circumvent access controls expressed using an incomplete set of file name or file path representations. This is different from path traversal, wherein the manipulations are performed to generate a name for a different object.",,unclassified,
-CWE-410,EN-Insufficient Resource Pool (Type: Base),"The software's resource pool is not large enough to handle peak demand, which allows an attacker to prevent others from accessing the resource by using a (relatively) large number of requests for resources.
-Frequently the consequence is a ""flood"" of connection or sessions.",,unclassified,"Writing Secure Code: Chapter 17, ""Protecting Against Denial of Service Attacks"" Page 517"
-CWE-412,EN-Unrestricted Externally Accessible Lock (Type: Base),"The software properly checks for the existence of a lock, but the lock can be externally controlled or influenced by an actor that is outside of the intended sphere of control.
-This prevents the software from acting on associated resources or performing other behaviors that are controlled by the presence of the lock. Relevant locks might include an exclusive lock or mutex, or modifying a shared resource that is treated as a lock. If the lock can be held for an indefinite period of time, then the denial of service could be permanent.",,unclassified,
-CWE-413,EN-Improper Resource Locking (Type: Base),"The software does not lock or does not correctly lock a resource when the software must have exclusive access to the resource.
-When a resource is not properly locked, an attacker could modify the resource while it is being operated on by the software. This might violate the software's assumption that the resource will not change, potentially leading to unexpected behaviors.",,unclassified,
-CWE-414,EN-Missing Lock Check (Type: Base),"A product does not check to see if a lock is present before performing sensitive operations on a resource.
-When a resource is not properly locked, an attacker could modify the resource while it is being operated on by the software. This might violate the software's assumption that the resource will not change, potentially leading to unexpected behaviors.",,unclassified,
-CWE-419,EN-Unprotected Primary Channel (Type: Base),"The software uses a primary channel for administration or restricted functionality, but it does not properly protect the channel.
-The use of previously-freed memory can have any number of adverse consequences, ranging from the corruption of valid data to the execution of arbitrary code, depending on the instantiation and timing of the flaw. The simplest way data corruption may occur involves the system's reuse of the freed memory. Use-after-free errors have two common and sometimes overlapping causes:
-Error conditions and other exceptional circumstances.
-Confusion over which part of the program is responsible for freeing the memory.
-In this scenario, the memory in question is allocated to another pointer validly at some point after it has been freed. The original pointer to the freed memory is used again and points to somewhere within the new allocation. As the data is changed, it corrupts the validly used memory; this induces undefined behavior in the process.
-If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,unclassified,
-CWE-42,EN-Path Equivalence: filename. (Trailing Dot) (Type: Variant),"A software system that accepts path input in the form of trailing dot ('filedir.') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files.
-The use of previously-freed memory can have any number of adverse consequences, ranging from the corruption of valid data to the execution of arbitrary code, depending on the instantiation and timing of the flaw. The simplest way data corruption may occur involves the system's reuse of the freed memory. Use-after-free errors have two common and sometimes overlapping causes:
-Error conditions and other exceptional circumstances.
-Confusion over which part of the program is responsible for freeing the memory.
-In this scenario, the memory in question is allocated to another pointer validly at some point after it has been freed. The original pointer to the freed memory is used again and points to somewhere within the new allocation. As the data is changed, it corrupts the validly used memory; this induces undefined behavior in the process.
-If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,unclassified,
-CWE-420,EN-Unprotected Alternate Channel (Type: Base),"The software protects a primary channel, but it does not use the same level of protection for an alternate channel.
-The use of previously-freed memory can have any number of adverse consequences, ranging from the corruption of valid data to the execution of arbitrary code, depending on the instantiation and timing of the flaw. The simplest way data corruption may occur involves the system's reuse of the freed memory. Use-after-free errors have two common and sometimes overlapping causes:
-Error conditions and other exceptional circumstances.
-Confusion over which part of the program is responsible for freeing the memory.
-In this scenario, the memory in question is allocated to another pointer validly at some point after it has been freed. The original pointer to the freed memory is used again and points to somewhere within the new allocation. As the data is changed, it corrupts the validly used memory; this induces undefined behavior in the process.
-If the newly allocated data chances to hold a class, in C++ for example, various function pointers may be scattered within the heap data. If one of these function pointers is overwritten with an address to valid shellcode, execution of arbitrary code can be achieved.",,unclassified,
-CWE-421,EN-Race Condition During Access to Alternate Channel (Type: Base),"The product opens an alternate channel to communicate with an authorized user, but the channel is accessible to other actors.
-This creates a race condition that allows an attacker to access the channel before the authorized user does.",,unclassified,"Discovering and Exploiting Named Pipe Security Flaws for Fun and Profit: http://www.blakewatts.com/namedpipepaper.html
-24 Deadly Sins of Software Security: ""Sin 13: Race Conditions."" Page 205"
-CWE-422,EN-Unprotected Windows Messaging Channel (Shatter) (Type: Variant),"The software does not properly verify the source of a message in the Windows Messaging System while running at elevated privileges, creating an alternate channel through which an attacker can directly send a message to the product.
-This creates a race condition that allows an attacker to access the channel before the authorized user does.",,unclassified,"Exploiting design flaws in the Win32 API for privilege escalation. Or... Shatter Attacks - How to break Windows: http://web.archive.org/web/20060115174629/http://security.tombom.co.uk/shatter.html
-The Art of Software Security Assessment: Chapter 2, ""Design Review."" Page 34.
-The Art of Software Security Assessment: Chapter 12, ""Shatter Attacks"", Page 694."
-CWE-423,EN-DEPRECATED (Duplicate): Proxied Trusted Channel (Type: Base),"This entry has been deprecated because it was a duplicate of CWE-441. All content has been transferred to CWE-441.
-This creates a race condition that allows an attacker to access the channel before the authorized user does.",,unclassified,
-CWE-424,EN-Improper Protection of Alternate Path (Type: Class),"The product does not sufficiently protect all possible paths that a user can take to access restricted functionality or resources.
-This creates a race condition that allows an attacker to access the channel before the authorized user does.",,unclassified,
-CWE-425,EN-Direct Request (Forced Browsing) (Type: Base),"The web application does not adequately enforce appropriate authorization on all restricted URLs, scripts, or files.
-Web applications susceptible to direct request attacks often make the false assumption that such resources can only be reached through a given navigation path and so only apply authorization at certain points in the path.",,unclassified,
-CWE-427,EN-Uncontrolled Search Path Element (Type: Base),"The product uses a fixed or controlled search path to find resources, but one or more locations in that path can be under the control of unintended actors.
-Although this weakness can occur with any type of resource, it is frequently introduced when a product uses a directory search path to find executables or code libraries, but the path contains a directory that can be modified by an attacker, such as ""/tmp"" or the current working directory.
-In Windows-based systems, when the LoadLibrary or LoadLibraryEx function is called with a DLL name that does not contain a fully qualified path, the function follows a search order that includes two path elements that might be uncontrolled:
-the directory from which the program has been loaded
-the current working directory.
-In some cases, the attack can be conducted remotely, such as when SMB or WebDAV network shares are used.
-In some Unix-based systems, a PATH might be created that contains an empty element, e.g. by splicing an empty variable into the PATH. This empty element can be interpreted as equivalent to the current working directory, which might be an untrusted search element.",,unclassified,"Double clicking on MS Office documents from Windows Explorer may execute arbitrary programs in some cases
-ACROS Security: Remote Binary Planting in Apple iTunes for Windows (ASPR #2010-08-18-1)
-Automatic Detection of Vulnerable Dynamic Component Loadings: http://www.cs.ucdavis.edu/research/tech-reports/2010/CSE-2010-2.pdf
-Dynamic-Link Library Search Order: http://msdn.microsoft.com/en-us/library/ms682586%28v=VS.85%29.aspx
-Dynamic-Link Library Security: http://msdn.microsoft.com/en-us/library/ff919712%28VS.85%29.aspx
-An update on the DLL-preloading remote attack vector: http://blogs.technet.com/b/srd/archive/2010/08/23/an-update-on-the-dll-preloading-remote-attack-vector.aspx
-Insecure Library Loading Could Allow Remote Code Execution: http://www.microsoft.com/technet/security/advisory/2269637.mspx
-Application DLL Load Hijacking: http://blog.rapid7.com/?p=5325
-DLL Hijacking: Facts and Fiction: http://threatpost.com/en_us/blogs/dll-hijacking-facts-and-fiction-082610"
-CWE-428,EN-Unquoted Search Path or Element (Type: Base),"The product uses a search path that contains an unquoted element, in which the element contains whitespace or other separators. This can cause the product to access resources in a parent path.
-If a malicious individual has access to the file system, it is possible to elevate privileges by inserting such a file as ""C:\Program.exe"" to be run by a privileged program making use of WinExec.",,unclassified,"The Art of Software Security Assessment: Chapter 11, ""Process Loading"", Page 654."
-CWE-43,EN-Path Equivalence: filename.... (Multiple Trailing Dot) (Type: Variant),"A software system that accepts path input in the form of multiple trailing dot ('filedir....') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files.
-If a malicious individual has access to the file system, it is possible to elevate privileges by inserting such a file as ""C:\Program.exe"" to be run by a privileged program making use of WinExec.",,unclassified,
-CWE-430,EN-Deployment of Wrong Handler (Type: Base),"The wrong ""handler"" is assigned to process an object.
-An example of deploying the wrong handler would be calling a servlet to reveal source code of a .JSP file, or automatically ""determining"" type of the object even if it is contradictory to an explicitly specified type.",,unclassified,"The Art of Software Security Assessment: Chapter 3, ""File Handlers"", Page 74."
-CWE-431,EN-Missing Handler (Type: Base),"A handler is not available or implemented.
-When an exception is thrown and not caught, the process has given up an opportunity to decide if a given failure or event is worth a change in execution.",,unclassified,"The Art of Software Security Assessment: Chapter 3, ""File Handlers"", Page 74."
-CWE-432,EN-Dangerous Signal Handler not Disabled During Sensitive Operations (Type: Base),"The application uses a signal handler that shares state with other signal handlers, but it does not properly mask or prevent those signal handlers from being invoked while the original signal handler is still running.
-During the execution of a signal handler, it can be interrupted by another handler when a different signal is sent. If the two handlers share state - such as global variables - then an attacker can corrupt the state by sending another signal before the first handler has completed execution.",,unclassified,
-CWE-433,EN-Unparsed Raw Web Content Delivery (Type: Variant),"The software stores raw content or supporting code under the web document root with an extension that is not specifically handled by the server.
-If code is stored in a file with an extension such as "".inc"" or "".pl"", and the web server does not have a handler for that extension, then the server will likely send the contents of the file directly to the requester without the pre-processing that was expected. When that file contains sensitive information such as database credentials, this may allow the attacker to compromise the application or associated components.",,unclassified,"The Art of Software Security Assessment: Chapter 3, ""File Handlers"", Page 74."
-CWE-435,EN-Interaction Error (Type: Class),"An interaction error occurs when two entities work correctly when running independently, but they interact in unexpected ways when they are run together.
-This could apply to products, systems, components, etc.",,unclassified,
-CWE-436,EN-Interpretation Conflict (Type: Base),"Product A handles inputs or steps differently than Product B, which causes A to perform incorrect actions based on its perception of B's state.
-This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified,"On Interpretation Conflict Vulnerabilities
-Insertion, Evasion, and Denial of Service: Eluding Network Intrusion Detection: http://www.insecure.org/stf/secnet_ids/secnet_ids.pdf
-0x00 vs ASP file upload scripts: http://www.security-assessment.com/Whitepapers/0x00_vs_ASP_File_Uploads.pdf
-Poison NULL byte
-Re: Corsaire Security Advisory - Multiple vendor MIME RFC2047 encoding: http://marc.theaimsgroup.com/?l=bugtraq&m=109525864717484&w=2"
-CWE-437,EN-Incomplete Model of Endpoint Features (Type: Base),"A product acts as an intermediary or monitor between two or more endpoints, but it does not have a complete model of an endpoint's features, behaviors, or state, potentially causing the product to perform incorrect actions based on this incomplete model.
-This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified,
-CWE-439,EN-Behavioral Change in New Version or Environment (Type: Base),"A's behavior or functionality changes with a new version of A, or a new environment, which is not known (or manageable) by B.
-This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified,
-CWE-44,EN-Path Equivalence: file.name (Internal Dot) (Type: Variant),"A software system that accepts path input in the form of internal dot ('file.ordir') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files.
-This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified,
-CWE-440,EN-Expected Behavior Violation (Type: Base),"A feature, API, or function being used by a product behaves differently than the product expects.
-This is generally found in proxies, firewalls, anti-virus software, and other intermediary devices that allow, deny, or modify traffic based on how the client or server is expected to behave.",,unclassified,
-CWE-441,EN-Unintended Proxy or Intermediary (Confused Deputy) (Type: Class),"The software receives a request, message, or directive from an upstream component, but the software does not sufficiently preserve the original source of the request before forwarding the request to an external actor that is outside of the software's control sphere. This causes the software to appear to be the source of the request, leading it to act as a proxy or other intermediary between the upstream component and the external actor.
-If an attacker cannot directly contact a target, but the software has access to the target, then the attacker can send a request to the software and have it be forwarded from the target. The request would appear to be coming from the software's system, not the attacker's system. As a result, the attacker can bypass access controls (such as firewalls) or hide the source of malicious requests, since the requests would not be coming directly from the attacker.
-Since proxy functionality and message-forwarding often serve a legitimate purpose, this issue only becomes a vulnerability when:
-The software runs with different privileges or on a different system, or otherwise has different levels of access than the upstream component;
-The attacker is prevented from making the request directly to the target; and
-The attacker can create a request that the proxy does not explicitly intend to be forwarded on the behalf of the requester. Such a request might point to an unexpected hostname, port number, or service. Or, the request might be sent to an allowed service, but the request could contain disallowed directives, commands, or resources.",,unclassified,The Confused Deputy (or why capabilities might have been invented): http://www.cap-lore.com/CapTheory/ConfusedDeputy.html
-CWE-443,EN-DEPRECATED (Duplicate): HTTP response splitting (Type: Base),"This weakness can be found at CWE-113.
-If an attacker cannot directly contact a target, but the software has access to the target, then the attacker can send a request to the software and have it be forwarded from the target. The request would appear to be coming from the software's system, not the attacker's system. As a result, the attacker can bypass access controls (such as firewalls) or hide the source of malicious requests, since the requests would not be coming directly from the attacker.
-Since proxy functionality and message-forwarding often serve a legitimate purpose, this issue only becomes a vulnerability when:
-The software runs with different privileges or on a different system, or otherwise has different levels of access than the upstream component;
-The attacker is prevented from making the request directly to the target; and
-The attacker can create a request that the proxy does not explicitly intend to be forwarded on the behalf of the requester. Such a request might point to an unexpected hostname, port number, or service. Or, the request might be sent to an allowed service, but the request could contain disallowed directives, commands, or resources.",,unclassified,
-CWE-444,EN-Inconsistent Interpretation of HTTP Requests (HTTP Request Smuggling) (Type: Base),"When malformed or abnormal HTTP requests are interpreted by one or more entities in the data flow between the user and the web server, such as a proxy or firewall, they can be interpreted inconsistently, allowing the attacker to ""smuggle"" a request to one device without the other device being aware of it.
-If an attacker cannot directly contact a target, but the software has access to the target, then the attacker can send a request to the software and have it be forwarded from the target. The request would appear to be coming from the software's system, not the attacker's system. As a result, the attacker can bypass access controls (such as firewalls) or hide the source of malicious requests, since the requests would not be coming directly from the attacker.
-Since proxy functionality and message-forwarding often serve a legitimate purpose, this issue only becomes a vulnerability when:
-The software runs with different privileges or on a different system, or otherwise has different levels of access than the upstream component;
-The attacker is prevented from making the request directly to the target; and
-The attacker can create a request that the proxy does not explicitly intend to be forwarded on the behalf of the requester. Such a request might point to an unexpected hostname, port number, or service. Or, the request might be sent to an allowed service, but the request could contain disallowed directives, commands, or resources.",,unclassified,HTTP Request Smuggling: http://www.cgisecurity.com/lib/HTTP-Request-Smuggling.pdf
-CWE-446,EN-UI Discrepancy for Security Feature (Type: Base),"The user interface does not correctly enable or configure a security feature, but the interface provides feedback that causes the user to believe that the feature is in a secure state.
-When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified,
-CWE-447,EN-Unimplemented or Unsupported Feature in UI (Type: Base),"A UI function for a security feature appears to be supported and gives feedback to the user that suggests that it is supported, but the underlying functionality is not implemented.
-When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified,
-CWE-448,EN-Obsolete Feature in UI (Type: Base),"A UI function is obsolete and the product does not warn the user.
-When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified,
-CWE-449,EN-The UI Performs the Wrong Action (Type: Base),"The UI performs the wrong action with respect to the user's request.
-When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified,
-CWE-45,EN-Path Equivalence: file...name (Multiple Internal Dot) (Type: Variant),"A software system that accepts path input in the form of multiple internal dot ('file...dir') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files.
-When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified,
-CWE-450,EN-Multiple Interpretations of UI Input (Type: Base),"The UI has multiple interpretations of user input but does not prompt the user when it selects the less secure interpretation.
-When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified,
-CWE-451,EN-UI Misrepresentation of Critical Information (Type: Base),"The UI does not properly represent critical information to the user, allowing the information - or its source - to be obscured or spoofed. This is often a component in phishing attacks.
-When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified,
-CWE-453,EN-Insecure Default Variable Initialization (Type: Base),"The software, by default, initializes an internal variable with an insecure or less secure value than is possible.
-When the user interface does not properly reflect what the user asks of it, then it can lead the user into a false sense of security. For example, the user might check a box to enable a security option to enable encrypted communications, but the software does not actually enable the encryption. Alternately, the user might provide a ""restrict ALL'"" access control rule, but the software only implements ""restrict SOME"".",,unclassified,
-CWE-454,EN-External Initialization of Trusted Variables or Data Stores (Type: Base),"The software initializes critical internal variables or data stores using inputs that can be modified by untrusted actors.
-A software system should be reluctant to trust variables that have been initialized outside of its trust boundary, especially if they are initialized by users. They may have been initialized incorrectly. If an attacker can initialize the variable, then he/she can influence what the vulnerable system will do.",,unclassified,
-CWE-455,EN-Non-exit on Failed Initialization (Type: Base),"The software does not exit or otherwise modify its operation when security-relevant errors occur during initialization, such as when a configuration file has a format error, which can cause the software to execute in a less secure fashion than intended by the administrator.
-A software system should be reluctant to trust variables that have been initialized outside of its trust boundary, especially if they are initialized by users. They may have been initialized incorrectly. If an attacker can initialize the variable, then he/she can influence what the vulnerable system will do.",,unclassified,
-CWE-456,EN-Missing Initialization of a Variable (Type: Base),"The software does not initialize critical variables, which causes the execution environment to use unexpected values.
-A software system should be reluctant to trust variables that have been initialized outside of its trust boundary, especially if they are initialized by users. They may have been initialized incorrectly. If an attacker can initialize the variable, then he/she can influence what the vulnerable system will do.",,unclassified,"The Art of Software Security Assessment: Chapter 7, ""Variable Initialization"", Page 312."
-CWE-458,EN-DEPRECATED: Incorrect Initialization (Type: Base),"This weakness has been deprecated because its name and description did not match. The description duplicated CWE-454, while the name suggested a more abstract initialization problem. Please refer to CWE-665 for the more abstract problem.
-In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,unclassified,
-CWE-459,EN-Incomplete Cleanup (Type: Base),"The software does not properly ""clean up"" and remove temporary or supporting resources after they have been used.
-In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,unclassified,
-CWE-46,EN-Path Equivalence: filename (Trailing Space) (Type: Variant),"A software system that accepts path input in the form of trailing space ('filedir ') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files.
-In some languages such as C and C++, stack variables are not initialized by default. They generally contain junk data with the contents of stack memory before the function was invoked. An attacker can sometimes control or read these contents. In other languages or conditions, a variable that is not explicitly initialized can be given a default value that has security implications, depending on the logic of the program. The presence of an uninitialized variable can sometimes indicate a typographic error in the code.",,unclassified,
-CWE-463,EN-Deletion of Data Structure Sentinel (Type: Base),"The accidental deletion of a data-structure sentinel can cause serious programming logic problems.
-Often times data-structure sentinels are used to mark structure of the data structure. A common example of this is the null character at the end of strings. Another common example is linked lists which may contain a sentinel to mark the end of the list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the deletion or modification outside of some wrapper interface which provides safety.",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""NUL-Termination Problems"", Page 452."
-CWE-466,EN-Return of Pointer Value Outside of Expected Range (Type: Base),"A function can return a pointer to memory that is outside of the buffer that the pointer is expected to reference.
-Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 5: Buffer Overruns."" Page 89"
-CWE-47,EN-Path Equivalence: filename (Leading Space) (Type: Variant),"A software system that accepts path input in the form of leading space (' filedir') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files.
-Data-structure sentinels are often used to mark the structure of data. A common example of this is the null character at the end of strings or a special sentinel to mark the end of a linked list. It is dangerous to allow this type of control data to be easily accessible. Therefore, it is important to protect from the addition or modification of sentinels.",,unclassified,
-CWE-470,EN-Use of Externally-Controlled Input to Select Classes or Code (Unsafe Reflection) (Type: Base),"The application uses external input with reflection to select which classes or code to use, but it does not sufficiently prevent the input from selecting improper classes or code.
-If the application uses external inputs to determine which class to instantiate or which method to invoke, then an attacker could supply values to select unexpected classes or methods. If this occurs, then the attacker could create control flow paths that were not intended by the developer. These paths could bypass authentication or access control checks, or otherwise cause the application to behave in an unexpected manner. This situation becomes a doomsday scenario if the attacker can upload files into a location that appears on the application's classpath (CWE-427) or add new entries to the application's classpath (CWE-426). Under either of these conditions, the attacker can use reflection to introduce new, malicious behavior into the application.",,unclassified,
-CWE-471,EN-Modification of Assumed-Immutable Data (MAID) (Type: Base),"The software does not properly protect an assumed-immutable element from being modified by an attacker.
-If the application uses external inputs to determine which class to instantiate or which method to invoke, then an attacker could supply values to select unexpected classes or methods. If this occurs, then the attacker could create control flow paths that were not intended by the developer. These paths could bypass authentication or access control checks, or otherwise cause the application to behave in an unexpected manner. This situation becomes a doomsday scenario if the attacker can upload files into a location that appears on the application's classpath (CWE-427) or add new entries to the application's classpath (CWE-426). Under either of these conditions, the attacker can use reflection to introduce new, malicious behavior into the application.",,unclassified,
-CWE-472,EN-External Control of Assumed-Immutable Web Parameter (Type: Base),"The web application does not sufficiently verify inputs that are assumed to be immutable but are actually externally controllable, such as hidden form fields.
-If a web product does not properly protect assumed-immutable values from modification in hidden form fields, parameters, cookies, or URLs, this can lead to modification of critical data. Web applications often mistakenly make the assumption that data passed to the client in hidden fields or cookies is not susceptible to tampering. Improper validation of data that are user-controllable can lead to the application processing incorrect, and often malicious, input.
-For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 4: Use of Magic URLs, Predictable Cookies, and Hidden Form Fields."" Page 75
-The Art of Software Security Assessment: Chapter 17, ""Embedding State in HTML and URLs"", Page 1032."
-CWE-473,EN-PHP External Variable Modification (Type: Variant),"A PHP application does not properly protect against the modification of variables from external sources, such as query parameters or cookies. This can expose the application to numerous weaknesses that would not exist otherwise.
-If a web product does not properly protect assumed-immutable values from modification in hidden form fields, parameters, cookies, or URLs, this can lead to modification of critical data. Web applications often mistakenly make the assumption that data passed to the client in hidden fields or cookies is not susceptible to tampering. Improper validation of data that are user-controllable can lead to the application processing incorrect, and often malicious, input.
-For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,unclassified,
-CWE-474,EN-Use of Function with Inconsistent Implementations (Type: Base),"The code uses a function that has inconsistent implementations across operating systems and versions, which might cause security-relevant portability problems.
-If a web product does not properly protect assumed-immutable values from modification in hidden form fields, parameters, cookies, or URLs, this can lead to modification of critical data. Web applications often mistakenly make the assumption that data passed to the client in hidden fields or cookies is not susceptible to tampering. Improper validation of data that are user-controllable can lead to the application processing incorrect, and often malicious, input.
-For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,unclassified,
-CWE-475,EN-Undefined Behavior for Input to API (Type: Base),"The behavior of this function is undefined unless its control parameter is set to a specific value.
-If a web product does not properly protect assumed-immutable values from modification in hidden form fields, parameters, cookies, or URLs, this can lead to modification of critical data. Web applications often mistakenly make the assumption that data passed to the client in hidden fields or cookies is not susceptible to tampering. Improper validation of data that are user-controllable can lead to the application processing incorrect, and often malicious, input.
-For example, custom cookies commonly store session data or persistent data across sessions. This kind of session data is normally involved in security related decisions on the server side, such as user authentication and access control. Thus, the cookies might contain sensitive data such as user credentials and privileges. This is a dangerous practice, as it can often lead to improper reliance on the value of the client-provided cookie by the server side application.",,unclassified,
-CWE-477,EN-Use of Obsolete Functions (Type: Base),"The code uses deprecated or obsolete functions, which suggests that the code has not been actively reviewed or maintained.
-NULL pointer dereference issues can occur through a number of flaws, including race conditions, and simple programming omissions.",,unclassified,
-CWE-478,EN-Missing Default Case in Switch Statement (Type: Variant),"The code does not have a default case in a switch statement, which might lead to complex logical errors and resultant weaknesses.
-NULL pointer dereference issues can occur through a number of flaws, including race conditions, and simple programming omissions.",,unclassified,"The Art of Software Security Assessment: Chapter 7, ""Switch Statements"", Page 337."
-CWE-48,EN-Path Equivalence: file name (Internal Whitespace) (Type: Variant),"A software system that accepts path input in the form of internal space ('file(SPACE)name') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files.
-Non-reentrant functions are functions that cannot safely be called, interrupted, and then recalled before the first call has finished without resulting in memory corruption. This can lead to an unexpected system state an unpredictable results with a variety of potential consequences depending on context, including denial of service and code execution.
-Many functions are not reentrant, but some of them can result in the corruption of memory if they are used in a signal handler. The function call syslog() is an example of this. In order to perform its functionality, it allocates a small amount of memory as ""scratch space."" If syslog() is suspended by a signal call and the signal handler calls syslog(), the memory used by both of these functions enters an undefined, and possibly, exploitable state. Implementations of malloc() and free() manage metadata in global structures in order to track which memory is allocated versus which memory is available, but they are non-reentrant. Simultaneous calls to these functions can cause corruption of the metadata.",,unclassified,
-CWE-485,EN-Insufficient Encapsulation (Type: Class),"The product does not sufficiently encapsulate critical data or functionality.
-Encapsulation is about drawing strong boundaries. In a web browser that might mean ensuring that your mobile code cannot be abused by other mobile code. On the server it might mean differentiation between validated data and unvalidated data, between one user's data and another's, or between data users are allowed to see and data that they are not.",,unclassified,
-CWE-488,EN-Exposure of Data Element to Wrong Session (Type: Variant),"The product does not sufficiently enforce boundaries between the states of different sessions, causing data to be provided to, or used by, the wrong session.
-Data can ""bleed"" from one session to another through member variables of singleton objects, such as Servlets, and objects from a shared pool.
-In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,unclassified,
-CWE-489,EN-Leftover Debug Code (Type: Base),"The application can be deployed with active debugging code that can create unintended entry points.
-Data can ""bleed"" from one session to another through member variables of singleton objects, such as Servlets, and objects from a shared pool.
-In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,unclassified,
-CWE-49,EN-Path Equivalence: filename/ (Trailing Slash) (Type: Variant),"A software system that accepts path input in the form of trailing slash ('filedir/') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files.
-Data can ""bleed"" from one session to another through member variables of singleton objects, such as Servlets, and objects from a shared pool.
-In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,unclassified,
-CWE-491,EN-Public cloneable() Method Without Final (Object Hijack) (Type: Variant),"A class has a cloneable() method that is not declared final, which allows an object to be created without calling the constructor. This can cause the object to be in an unexpected state.
-Data can ""bleed"" from one session to another through member variables of singleton objects, such as Servlets, and objects from a shared pool.
-In the case of Servlets, developers sometimes do not understand that, unless a Servlet implements the SingleThreadModel interface, the Servlet is a singleton; there is only one instance of the Servlet, and that single instance is used and re-used to handle multiple requests that are processed simultaneously by different threads. A common result is that developers use Servlet member fields in such a way that one user may inadvertently see another user's data. In other words, storing user data in Servlet member fields introduces a data access race condition.",,unclassified,"OWASP , Attack Category : Mobile code: object hijack: http://www.owasp.org/index.php/Mobile_code:_object_hijack"
-CWE-495,EN-Private Array-Typed Field Returned From A Public Method (Type: Variant),"The product has a method that is declared public, but returns a reference to a private array, which could then be modified in unexpected ways.
-An attacker can execute malicious code by compromising the host server, performing DNS spoofing, or modifying the code in transit.",,unclassified,
-CWE-496,EN-Public Data Assigned to Private Array-Typed Field (Type: Variant),"Assigning public data to a private array is equivalent to giving public access to the array.
-An attacker can execute malicious code by compromising the host server, performing DNS spoofing, or modifying the code in transit.",,unclassified,
-CWE-497,EN-Exposure of System Data to an Unauthorized Control Sphere (Type: Variant),"Exposing system data or debugging information helps an adversary learn about the system and form an attack plan.
-An information exposure occurs when system data or debugging information leaves the program through an output stream or logging function that makes it accessible to unauthorized parties. An attacker can also cause errors to occur by submitting unusual requests to the web application. The response to these errors can reveal detailed system information, deny service, cause security mechanisms to fail, and crash the server. An attacker can use error messages that reveal technologies, operating systems, and product versions to tune the attack against known vulnerabilities in these technologies. An application may use diagnostic methods that provide significant implementation details such as stack traces as part of its error handling mechanism.",,unclassified,
-CWE-5,EN-J2EE Misconfiguration: Data Transmission Without Encryption (Type: Variant),"Information sent over a network can be compromised while in transit. An attacker may be able to read/modify the contents if the data are sent in plaintext or are weakly encrypted.
-Serializable classes are effectively open classes since data cannot be hidden in them. Classes that do not explicitly deny serialization can be serialized by any other class, which can then in turn use the data stored inside it.",,unclassified,
-CWE-50,EN-Path Equivalence: //multiple/leading/slash (Type: Variant),"A software system that accepts path input in the form of multiple leading slash ('//multiple/leading/slash') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files.
-Serializable classes are effectively open classes since data cannot be hidden in them. Classes that do not explicitly deny serialization can be serialized by any other class, which can then in turn use the data stored inside it.",,unclassified,
-CWE-501,EN-Trust Boundary Violation (Type: Base),"The product mixes trusted and untrusted data in the same data structure or structured message.
-By combining trusted and untrusted data in the same data structure, it becomes easier for programmers to mistakenly trust unvalidated data.",,unclassified,
-CWE-506,EN-Embedded Malicious Code (Type: Class),"The application contains code that appears to be malicious in nature.
-Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified,
-CWE-507,EN-Trojan Horse (Type: Base),"The software appears to contain benign or useful functionality, but it also contains code that is hidden from normal operation that violates the intended security policy of the user or the system administrator.
-Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified,"Writing Secure Code: Chapter 7, ""Viruses, Trojans, and Worms In a Nutshell"" Page 208"
-CWE-508,EN-Non-Replicating Malicious Code (Type: Base),"Non-replicating malicious code only resides on the target system or software that is attacked; it does not attempt to spread to other systems.
-Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified,
-CWE-509,EN-Replicating Malicious Code (Virus or Worm) (Type: Base),"Replicating malicious code, including viruses and worms, will attempt to attack other systems once it has successfully compromised the target system or software.
-Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified,
-CWE-51,EN-Path Equivalence: /multiple//internal/slash (Type: Variant),"A software system that accepts path input in the form of multiple internal slash ('/multiple//internal/slash/') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files.
-Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified,
-CWE-510,EN-Trapdoor (Type: Base),"A trapdoor is a hidden piece of code that responds to a special input, allowing its user access to resources without passing through the normal security enforcement mechanism.
-Malicious flaws have acquired colorful names, including Trojan horse, trapdoor, timebomb, and logic-bomb. A developer might insert malicious code with the intent to subvert the security of an application or its host system at some time in the future. It generally refers to a program that performs a useful service but exploits rights of the program's user in a way the user does not intend.",,unclassified,
-CWE-511,EN-Logic/Time Bomb (Type: Base),"The software contains code that is designed to disrupt the legitimate operation of the software (or its environment) when a certain time passes, or when a certain logical condition is met.
-When the time bomb or logic bomb is detonated, it may perform a denial of service such as crashing the system, deleting critical data, or degrading system response time. This bomb might be placed within either a replicating or non-replicating Trojan horse.",,unclassified,Mobile App Top 10 List: http://www.veracode.com/blog/2010/12/mobile-app-top-10-list/
-CWE-512,EN-Spyware (Type: Base),"The software collects personally identifiable information about a human user or the user's activities, but the software accesses this information using other resources besides itself, and it does not require that user's explicit approval or direct input into the software.
-""Spyware"" is a commonly used term with many definitions and interpretations. In general, it is meant to software that collects information or installs functionality that human users might not allow if they were fully aware of the actions being taken by the software. For example, a user might expect that tax software would collect a social security number and include it when filing a tax return, but that same user would not expect gaming software to obtain the social security number from that tax software's data.",,unclassified,
-CWE-514,EN-Covert Channel (Type: Class),"A covert channel is a path that can be used to transfer information in a way not intended by the system's designers.
-Typically the system has not given authorization for the transmission and has no knowledge of its occurrence.",,unclassified,
-CWE-516,EN-DEPRECATED (Duplicate): Covert Timing Channel (Type: Base),"This weakness can be found at CWE-385.
-Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,unclassified,
-CWE-52,EN-Path Equivalence: /multiple/trailing/slash// (Type: Variant),"A software system that accepts path input in the form of multiple trailing slash ('/multiple/trailing/slash//') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files.
-Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,unclassified,
-CWE-520,EN-.NET Misconfiguration: Use of Impersonation (Type: Variant),"Allowing a .NET application to run at potentially escalated levels of access to the underlying operating and file systems can be dangerous and result in various forms of attacks.
-Covert storage channels occur when out-of-band data is stored in messages for the purpose of memory reuse. Covert channels are frequently classified as either storage or timing channels. Examples would include using a file intended to hold only audit information to convey user passwords--using the name of a file or perhaps status bits associated with it that can be read by all users to signal the contents of the file. Steganography, concealing information in such a manner that no one but the intended recipient knows of the existence of the message, is a good example of a covert storage channel.",,unclassified,
-CWE-521,EN-Weak Password Requirements (Type: Base),"The product does not require that users should have strong passwords, which makes it easier for attackers to compromise user accounts.
-An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279"
-CWE-522,EN-Insufficiently Protected Credentials (Type: Base),"This weakness occurs when the application transmits or stores authentication credentials and uses an insecure method that is susceptible to unauthorized interception and/or retrieval.
-An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279"
-CWE-523,EN-Unprotected Transport of Credentials (Type: Variant),"Login pages not using adequate measures to protect the user name and password while they are in transit from the client to the server.
-An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified,
-CWE-524,EN-Information Exposure Through Caching (Type: Variant),"The application uses a cache to maintain a pool of objects, threads, connections, pages, or passwords to minimize the time it takes to access them or the resources to which they connect. If implemented improperly, these caches can allow access to unauthorized information or cause a denial of service vulnerability.
-An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified,
-CWE-525,EN-Information Exposure Through Browser Caching (Type: Variant),"For each web page, the application should have an appropriate caching policy specifying the extent to which the page and its form fields should be cached.
-An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified,
-CWE-526,EN-Information Exposure Through Environmental Variables (Type: Variant),"Environmental variables may contain sensitive information about a remote server.
-An authentication mechanism is only as strong as its credentials. For this reason, it is important to require users to have strong passwords. Lack of password complexity significantly reduces the search space when trying to guess user's passwords, making brute-force attacks easier.",,unclassified,
-CWE-527,EN-Exposure of CVS Repository to an Unauthorized Control Sphere (Type: Variant),"The product stores a CVS repository in a directory or other container that is accessible to actors outside of the intended control sphere.
-Information contained within a CVS subdirectory on a web server or other server could be recovered by an attacker and used for malicious purposes. This information may include usernames, filenames, path root, and IP addresses.",,unclassified,
-CWE-528,EN-Exposure of Core Dump File to an Unauthorized Control Sphere (Type: Variant),"The product generates a core dump file in a directory that is accessible to actors outside of the intended control sphere.
-Information contained within a CVS subdirectory on a web server or other server could be recovered by an attacker and used for malicious purposes. This information may include usernames, filenames, path root, and IP addresses.",,unclassified,
-CWE-529,EN-Exposure of Access Control List Files to an Unauthorized Control Sphere (Type: Variant),"The product stores access control list files in a directory or other container that is accessible to actors outside of the intended control sphere.
-Exposure of these access control list files may give the attacker information about the configuration of the site or system. This information may then be used to bypass the intended security policy or identify trusted systems from which an attack can be launched.",,unclassified,
-CWE-53,EN-Path Equivalence: \multiple\\internal\backslash (Type: Variant),"A software system that accepts path input in the form of multiple internal backslash ('\multiple\trailing\\slash') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files.
-Exposure of these access control list files may give the attacker information about the configuration of the site or system. This information may then be used to bypass the intended security policy or identify trusted systems from which an attack can be launched.",,unclassified,
-CWE-530,EN-Exposure of Backup File to an Unauthorized Control Sphere (Type: Variant),"A backup file is stored in a directory that is accessible to actors outside of the intended control sphere.
-Often, old files are renamed with an extension such as .~bk to distinguish them from production files. The source code for old files that have been renamed in this manner and left in the webroot can often be retrieved. This renaming may have been performed automatically by the web server, or manually by the administrator.",,unclassified,
-CWE-531,EN-Information Exposure Through Test Code (Type: Variant),"Accessible test applications can pose a variety of security risks. Since developers or administrators rarely consider that someone besides themselves would even know about the existence of these applications, it is common for them to contain sensitive information or functions.
-Often, old files are renamed with an extension such as .~bk to distinguish them from production files. The source code for old files that have been renamed in this manner and left in the webroot can often be retrieved. This renaming may have been performed automatically by the web server, or manually by the administrator.",,unclassified,
-CWE-533,EN-Information Exposure Through Server Log Files (Type: Variant),"A server.log file was found. This can give information on whatever application left the file. Usually this can give full path names and system information, and sometimes usernames and passwords.
-While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified,
-CWE-534,EN-Information Exposure Through Debug Log Files (Type: Variant),"The application does not sufficiently restrict access to a log file that is used for debugging.
-While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified,
-CWE-535,EN-Information Exposure Through Shell Error Message (Type: Variant),"A command shell error message indicates that there exists an unhandled exception in the web application code. In many cases, an attacker can leverage the conditions that cause these errors in order to gain unauthorized access to the system.
-While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified,
-CWE-536,EN-Information Exposure Through Servlet Runtime Error Message (Type: Variant),"A servlet error message indicates that there exists an unhandled exception in your web application code and may provide useful information to an attacker.
-While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified,
-CWE-537,EN-Information Exposure Through Java Runtime Error Message (Type: Variant),"In many cases, an attacker can leverage the conditions that cause unhandled exception errors in order to gain unauthorized access to the system.
-While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified,
-CWE-538,EN-File and Directory Information Exposure (Type: Base),"The product stores sensitive information in files or directories that are accessible to actors outside of the intended control sphere.
-While logging all information may be helpful during development stages, it is important that logging levels be set appropriately before a product ships so that sensitive user data and system information are not accidentally exposed to potential attackers.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 12: Information Leakage."" Page 191"
-CWE-539,EN-Information Exposure Through Persistent Cookies (Type: Variant),"Persistent cookies are cookies that are stored on the browser's hard drive. This can cause security and privacy issues depending on the information stored in the cookie and how it is accessed.
-Cookies are small bits of data that are sent by the web application but stored locally in the browser. This lets the application use the cookie to pass information between pages and store variable information. The web application controls what information is stored in a cookie and how it is used. Typical types of information stored in cookies are session Identifiers, personalization and customization information, and in rare cases even usernames to enable automated logins. There are two different types of cookies: session cookies and persistent cookies. Session cookies just live in the browser's memory, and are not stored anywhere, but persistent cookies are stored on the browser's hard drive.",,unclassified,
-CWE-54,EN-Path Equivalence: filedir\ (Trailing Backslash) (Type: Variant),"A software system that accepts path input in the form of trailing backslash ('filedir\') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files.
-Cookies are small bits of data that are sent by the web application but stored locally in the browser. This lets the application use the cookie to pass information between pages and store variable information. The web application controls what information is stored in a cookie and how it is used. Typical types of information stored in cookies are session Identifiers, personalization and customization information, and in rare cases even usernames to enable automated logins. There are two different types of cookies: session cookies and persistent cookies. Session cookies just live in the browser's memory, and are not stored anywhere, but persistent cookies are stored on the browser's hard drive.",,unclassified,
-CWE-540,EN-Information Exposure Through Source Code (Type: Variant),"Source code on a web server often contains sensitive information and should generally not be accessible to users.
-There are situations where it is critical to remove source code from an area or server. For example, obtaining Perl source code on a system allows an attacker to understand the logic of the script and extract extremely useful information such as code bugs or logins and passwords.",,unclassified,
-CWE-541,EN-Information Exposure Through Include Source Code (Type: Variant),"If an include file source is accessible, the file can contain usernames and passwords, as well as sensitive information pertaining to the application and system.
-There are situations where it is critical to remove source code from an area or server. For example, obtaining Perl source code on a system allows an attacker to understand the logic of the script and extract extremely useful information such as code bugs or logins and passwords.",,unclassified,
-CWE-542,EN-Information Exposure Through Cleanup Log Files (Type: Variant),"The application does not properly protect or delete a log file related to cleanup.
-There are situations where it is critical to remove source code from an area or server. For example, obtaining Perl source code on a system allows an attacker to understand the logic of the script and extract extremely useful information such as code bugs or logins and passwords.",,unclassified,
-CWE-543,EN-Use of Singleton Pattern Without Synchronization in a Multithreaded Context (Type: Variant),"The software uses the singleton pattern when creating a resource within a multithreaded environment.
-The use of a singleton pattern may not be thread-safe.",,unclassified,Thread-Specifc Storage for C/C++: http://www.cs.wustl.edu/~schmidt/PDF/TSS-pattern.pdf
-CWE-544,EN-Missing Standardized Error Handling Mechanism (Type: Base),"The software does not use a standardized method for handling errors throughout the code, which might introduce inconsistent error handling and resultant weaknesses.
-If the application handles error messages individually, on a one-by-one basis, this is likely to result in inconsistent error handling. The causes of errors may be lost. Also, detailed information about the causes of an error may be unintentionally returned to the user.",,unclassified,
-CWE-545,EN-Use of Dynamic Class Loading (Type: Variant),"Dynamically loaded code has the potential to be malicious.
-If the application handles error messages individually, on a one-by-one basis, this is likely to result in inconsistent error handling. The causes of errors may be lost. Also, detailed information about the causes of an error may be unintentionally returned to the user.",,unclassified,
-CWE-546,EN-Suspicious Comment (Type: Variant),"The code contains comments that suggest the presence of bugs, incomplete functionality, or weaknesses.
-Many suspicious comments, such as BUG, HACK, FIXME, LATER, LATER2, TODO, in the code indicate missing security functionality and checking. Others indicate code problems that programmers should fix, such as hard-coded variables, error handling, not using stored procedures, and performance issues.",,unclassified,
-CWE-547,"EN-Use of Hard-coded, Security-relevant Constants (Type: Variant)","The program uses hard-coded constants instead of symbolic names for security-critical values, which increases the likelihood of mistakes during code maintenance or security policy change.
-If the developer does not find all occurrences of the hard-coded constants, an incorrect policy decision may be made if one of the constants is not changed. Making changes to these values will require code changes that may be difficult or impossible once the system is released to the field. In addition, these hard-coded values may become available to attackers if the code is ever disclosed.",,unclassified,
-CWE-548,EN-Information Exposure Through Directory Listing (Type: Variant),"A directory listing is inappropriately exposed, yielding potentially sensitive information to attackers.
-A directory listing provides an attacker with the complete index of all the resources located inside of the directory. The specific risks and consequences vary depending on which files are listed and accessible.",,unclassified,
-CWE-549,EN-Missing Password Field Masking (Type: Variant),"The software does not mask passwords during entry, increasing the potential for attackers to observe and capture passwords.
-A directory listing provides an attacker with the complete index of all the resources located inside of the directory. The specific risks and consequences vary depending on which files are listed and accessible.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279"
-CWE-55,EN-Path Equivalence: /./ (Single Dot Directory) (Type: Variant),"A software system that accepts path input in the form of single dot directory exploit ('/./') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files.
-A directory listing provides an attacker with the complete index of all the resources located inside of the directory. The specific risks and consequences vary depending on which files are listed and accessible.",,unclassified,
-CWE-550,EN-Information Exposure Through Server Error Message (Type: Variant),"Certain conditions, such as network failure, will cause a server error message to be displayed.
-While error messages in and of themselves are not dangerous, per se, it is what an attacker can glean from them that might cause eventual problems.",,unclassified,
-CWE-551,EN-Incorrect Behavior Order: Authorization Before Parsing and Canonicalization (Type: Base),"If a web server does not fully parse requested URLs before it examines them for authorization, it may be possible for an attacker to bypass authorization protection.
-For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,unclassified,
-CWE-552,EN-Files or Directories Accessible to External Parties (Type: Base),"Files or directories are accessible in the environment that should not be.
-For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,unclassified,
-CWE-553,EN-Command Shell in Externally Accessible Directory (Type: Variant),"A possible shell file exists in /cgi-bin/ or other accessible directories. This is extremely dangerous and can be used by an attacker to execute commands on the web server.
-For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,unclassified,
-CWE-554,EN-ASP.NET Misconfiguration: Not Using Input Validation Framework (Type: Variant),"The ASP.NET application does not use an input validation framework.
-For instance, the character strings /./ and / both mean current directory. If /SomeDirectory is a protected directory and an attacker requests /./SomeDirectory, the attacker may be able to gain access to the resource if /./ is not converted to / before the authorization check is performed.",,unclassified,
-CWE-555,EN-J2EE Misconfiguration: Plaintext Password in Configuration File (Type: Variant),"The J2EE application stores a plaintext password in a configuration file.
-Storing a plaintext password in a configuration file allows anyone who can read the file to access the password-protected resource, making it an easy target for attackers.",,unclassified,
-CWE-556,EN-ASP.NET Misconfiguration: Use of Identity Impersonation (Type: Variant),"Configuring an ASP.NET application to run with impersonated credentials may give the application unnecessary privileges.
-The use of impersonated credentials allows an ASP.NET application to run with either the privileges of the client on whose behalf it is executing or with arbitrary privileges granted in its configuration.",,unclassified,
-CWE-558,EN-Use of getlogin() in Multithreaded Application (Type: Variant),"The application uses the getlogin() function in a multithreaded context, potentially causing it to return incorrect values.
-The getlogin() function returns a pointer to a string that contains the name of the user associated with the calling process. The function is not reentrant, meaning that if it is called from another process, the contents are not locked out and the value of the string can be changed by another process. This makes it very risky to use because the username can be changed by other processes, so the results of the function cannot be trusted.",,unclassified,
-CWE-56,EN-Path Equivalence: filedir* (Wildcard) (Type: Variant),"A software system that accepts path input in the form of asterisk wildcard ('filedir*') without appropriate validation can lead to ambiguous path resolution and allow an attacker to traverse the file system to unintended locations or access arbitrary files.
-The getlogin() function returns a pointer to a string that contains the name of the user associated with the calling process. The function is not reentrant, meaning that if it is called from another process, the contents are not locked out and the value of the string can be changed by another process. This makes it very risky to use because the username can be changed by other processes, so the results of the function cannot be trusted.",,unclassified,
-CWE-560,EN-Use of umask() with chmod-style Argument (Type: Variant),"The product calls umask() with an incorrect argument that is specified as if it is an argument to chmod().
-The getlogin() function returns a pointer to a string that contains the name of the user associated with the calling process. The function is not reentrant, meaning that if it is called from another process, the contents are not locked out and the value of the string can be changed by another process. This makes it very risky to use because the username can be changed by other processes, so the results of the function cannot be trusted.",,unclassified,
-CWE-561,EN-Dead Code (Type: Variant),"The software contains dead code, which can never be executed.
-Dead code is source code that can never be executed in a running program. The surrounding code makes it impossible for a section of code to ever be executed.",,unclassified,
-CWE-562,EN-Return of Stack Variable Address (Type: Base),"A function returns the address of a stack variable, which will cause unintended program behavior, typically in the form of a crash.
-Dead code is source code that can never be executed in a running program. The surrounding code makes it impossible for a section of code to ever be executed.",,unclassified,
-CWE-563,EN-Unused Variable (Type: Variant),"The variable's value is assigned but never used, making it a dead store.
-It is likely that the variable is simply vestigial, but it is also possible that the unused variable points out a bug.",,unclassified,
-CWE-564,EN-SQL Injection: Hibernate (Type: Variant),"Using Hibernate to execute a dynamic SQL statement built with user-controlled input can allow an attacker to modify the statement's meaning or to execute arbitrary SQL commands.
-It is likely that the variable is simply vestigial, but it is also possible that the unused variable points out a bug.",,unclassified,
-CWE-565,EN-Reliance on Cookies without Validation and Integrity Checking (Type: Base),"The application relies on the existence or values of cookies when performing security-critical operations, but it does not properly ensure that the setting is valid for the associated user.
-Attackers can easily modify cookies, within the browser or by implementing the client-side code outside of the browser. Reliance on cookies without detailed validation and integrity checking can allow attackers to bypass authentication, conduct injection attacks such as SQL injection and cross-site scripting, or otherwise modify inputs in unexpected ways.",,unclassified,
-CWE-566,EN-Authorization Bypass Through User-Controlled SQL Primary Key (Type: Variant),"The software uses a database table that includes records that should not be accessible to an actor, but it executes a SQL statement with a primary key that can be controlled by that actor.
-When a user can set a primary key to any value, then the user can modify the key to point to unauthorized records.
-Database access control errors occur when:
-Data enters a program from an untrusted source.
-The data is used to specify the value of a primary key in a SQL query.
-The untrusted source does not have the permissions to be able to access all rows in the associated table.",,unclassified,
-CWE-567,EN-Unsynchronized Access to Shared Data in a Multithreaded Context (Type: Base),"The product does not properly synchronize shared data, such as static variables across threads, which can lead to undefined behavior and unpredictable data changes.
-Within servlets, shared static variables are not protected from concurrent access, but servlets are multithreaded. This is a typical programming mistake in J2EE applications, since the multithreading is handled by the framework. When a shared variable can be influenced by an attacker, one thread could wind up modifying the variable to contain data that is not valid for a different thread that is also using the data within the variable.
-Note that this weakness is not unique to servlets.",,unclassified,
-CWE-568,EN-finalize() Method Without super.finalize() (Type: Variant),"The software contains a finalize() method that does not call super.finalize().
-The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,unclassified,
-CWE-57,EN-Path Equivalence: fakedir/../realdir/filename (Type: Variant),"The software contains protection mechanisms to restrict access to 'realdir/filename', but it constructs pathnames using external input in the form of 'fakedir/../realdir/filename' that are not handled by those mechanisms. This allows attackers to perform unauthorized actions against the targeted file.
-The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,unclassified,
-CWE-570,EN-Expression is Always False (Type: Variant),"The software contains an expression that will always evaluate to false.
-The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,unclassified,
-CWE-571,EN-Expression is Always True (Type: Variant),"The software contains an expression that will always evaluate to true.
-The Java Language Specification states that it is a good practice for a finalize() method to call super.finalize().",,unclassified,
-CWE-572,EN-Call to Thread run() instead of start() (Type: Variant),"The program calls a thread's run() method instead of calling start(), which causes the code to run in the thread of the caller instead of the callee.
-In most cases a direct call to a Thread object's run() method is a bug. The programmer intended to begin a new thread of control, but accidentally called run() instead of start(), so the run() method will execute in the caller's thread of control.",,unclassified,
-CWE-573,EN-Improper Following of Specification by Caller (Type: Class),"The software does not follow or incorrectly follows the specifications as required by the implementation language, environment, framework, protocol, or platform.
-When leveraging external functionality, such as an API, it is important that the caller does so in accordance with the requirements of the external functionality or else unintended behaviors may result, possibly leaving the system vulnerable to any number of exploits.",,unclassified,
-CWE-574,EN-EJB Bad Practices: Use of Synchronization Primitives (Type: Variant),"The program violates the Enterprise JavaBeans (EJB) specification by using thread synchronization primitives.
-The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not use thread synchronization primitives to synchronize execution of multiple instances."" The specification justifies this requirement in the following way: ""This rule is required to ensure consistent runtime semantics because while some EJB containers may use a single JVM to execute all enterprise bean's instances, others may distribute the instances across multiple JVMs.""",,unclassified,
-CWE-575,EN-EJB Bad Practices: Use of AWT Swing (Type: Variant),"The program violates the Enterprise JavaBeans (EJB) specification by using AWT/Swing.
-The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not use the AWT functionality to attempt to output information to a display, or to input information from a keyboard."" The specification justifies this requirement in the following way: ""Most servers do not allow direct interaction between an application program and a keyboard/display attached to the server system.""",,unclassified,
-CWE-576,EN-EJB Bad Practices: Use of Java I/O (Type: Variant),"The program violates the Enterprise JavaBeans (EJB) specification by using the java.io package.
-The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not use the java.io package to attempt to access files and directories in the file system."" The specification justifies this requirement in the following way: ""The file system APIs are not well-suited for business components to access data. Business components should use a resource manager API, such as JDBC, to store data.""",,unclassified,
-CWE-577,EN-EJB Bad Practices: Use of Sockets (Type: Variant),"The program violates the Enterprise JavaBeans (EJB) specification by using sockets.
-The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""An enterprise bean must not attempt to listen on a socket, accept connections on a socket, or use a socket for multicast."" The specification justifies this requirement in the following way: ""The EJB architecture allows an enterprise bean instance to be a network socket client, but it does not allow it to be a network server. Allowing the instance to become a network server would conflict with the basic function of the enterprise bean-- to serve the EJB clients.""",,unclassified,
-CWE-578,EN-EJB Bad Practices: Use of Class Loader (Type: Variant),"The program violates the Enterprise JavaBeans (EJB) specification by using the class loader.
-The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""The enterprise bean must not attempt to create a class loader; obtain the current class loader; set the context class loader; set security manager; create a new security manager; stop the JVM; or change the input, output, and error streams."" The specification justifies this requirement in the following way: ""These functions are reserved for the EJB container. Allowing the enterprise bean to use these functions could compromise security and decrease the container's ability to properly manage the runtime environment.""",,unclassified,
-CWE-579,EN-J2EE Bad Practices: Non-serializable Object Stored in Session (Type: Variant),"The application stores a non-serializable object as an HttpSession attribute, which can hurt reliability.
-The Enterprise JavaBeans specification requires that every bean provider follow a set of programming guidelines designed to ensure that the bean will be portable and behave consistently in any EJB container. In this case, the program violates the following EJB guideline: ""The enterprise bean must not attempt to create a class loader; obtain the current class loader; set the context class loader; set security manager; create a new security manager; stop the JVM; or change the input, output, and error streams."" The specification justifies this requirement in the following way: ""These functions are reserved for the EJB container. Allowing the enterprise bean to use these functions could compromise security and decrease the container's ability to properly manage the runtime environment.""",,unclassified,
-CWE-58,EN-Path Equivalence: Windows 8.3 Filename (Type: Variant),"The software contains a protection mechanism that restricts access to a long filename on a Windows operating system, but the software does not properly restrict access to the equivalent short ""8.3"" filename.
-On later Windows operating systems, a file can have a ""long name"" and a short name that is compatible with older Windows file systems, with up to 8 characters in the filename and 3 characters for the extension. These ""8.3"" filenames, therefore, act as an alternate name for files with long names, so they are useful pathname equivalence manipulations.",,unclassified,"Writing Secure Code
-The Art of Software Security Assessment: Chapter 11, ""DOS 8.3 Filenames"", Page 673."
-CWE-580,EN-clone() Method Without super.clone() (Type: Variant),"The software contains a clone() method that does not call super.clone() to obtain the new object.
-All implementations of clone() should obtain the new object by calling super.clone(). If a class does not follow this convention, a subclass's clone() method will return an object of the wrong type.",,unclassified,
-CWE-581,EN-Object Model Violation: Just One of Equals and Hashcode Defined (Type: Base),"The software does not maintain equal hashcodes for equal objects.
-Java objects are expected to obey a number of invariants related to equality. One of these invariants is that equal objects must have equal hashcodes. In other words, if a.equals(b) == true then a.hashCode() == b.hashCode().",,unclassified,
-CWE-582,"EN-Array Declared Public, Final, and Static (Type: Variant)","The program declares an array public, final, and static, which is not sufficient to prevent the array's contents from being modified.
-Because arrays are mutable objects, the final constraint requires that the array object itself be assigned only once, but makes no guarantees about the values of the array elements. Since the array is public, a malicious program can change the values stored in the array. As such, in most cases an array declared public, final and static is a bug.",,unclassified,
-CWE-583,EN-finalize() Method Declared Public (Type: Variant),"The program violates secure coding principles for mobile code by declaring a finalize() method public.
-A program should never call finalize explicitly, except to call super.finalize() inside an implementation of finalize(). In mobile code situations, the otherwise error prone practice of manual garbage collection can become a security threat if an attacker can maliciously invoke one of your finalize() methods because it is declared with public access.",,unclassified,
-CWE-584,EN-Return Inside Finally Block (Type: Base),"The code has a return statement inside a finally block, which will cause any thrown exception in the try block to be discarded.
-A program should never call finalize explicitly, except to call super.finalize() inside an implementation of finalize(). In mobile code situations, the otherwise error prone practice of manual garbage collection can become a security threat if an attacker can maliciously invoke one of your finalize() methods because it is declared with public access.",,unclassified,
-CWE-585,EN-Empty Synchronized Block (Type: Variant),"The software contains an empty synchronized block.
-An empty synchronized block does not actually accomplish any synchronization and may indicate a troubled section of code. An empty synchronized block can occur because code no longer needed within the synchronized block is commented out without removing the synchronized block.",,unclassified,Intrinsic Locks and Synchronization (in Java): http://java.sun.com/docs/books/tutorial/essential/concurrency/locksync.html
-CWE-586,EN-Explicit Call to Finalize() (Type: Variant),"The software makes an explicit call to the finalize() method from outside the finalizer.
-While the Java Language Specification allows an object's finalize() method to be called from outside the finalizer, doing so is usually a bad idea. For example, calling finalize() explicitly means that finalize() will be called more than once: the first time will be the explicit call and the last time will be the call that is made after the object is garbage collected.",,unclassified,
-CWE-587,EN-Assignment of a Fixed Address to a Pointer (Type: Base),"The software sets a pointer to a specific address other than NULL or 0.
-Using a fixed address is not portable because that address will probably not be valid in all environments or platforms.",,unclassified,
-CWE-588,EN-Attempt to Access Child of a Non-structure Pointer (Type: Variant),"Casting a non-structure type to a structure type and accessing a field can lead to memory access errors or data corruption.
-Using a fixed address is not portable because that address will probably not be valid in all environments or platforms.",,unclassified,
-CWE-589,EN-Call to Non-ubiquitous API (Type: Variant),"The software uses an API function that does not exist on all versions of the target platform. This could cause portability problems or inconsistencies that allow denial of service or other consequences.
-Some functions that offer security features supported by the OS are not available on all versions of the OS in common use. Likewise, functions are often deprecated or made obsolete for security reasons and should not be used.",,unclassified,
-CWE-590,EN-Free of Memory not on the Heap (Type: Variant),"The application calls free() on a pointer to memory that was not allocated using associated heap allocation functions such as malloc(), calloc(), or realloc().
-When free() is called on an invalid pointer, the program's memory management data structures may become corrupted. This corruption can cause the program to crash or, in some circumstances, an attacker may be able to cause free() to operate on controllable memory locations to modify critical program variables or execute code.",,unclassified,Valgrind: http://valgrind.org/
-CWE-591,EN-Sensitive Data Storage in Improperly Locked Memory (Type: Variant),"The application stores sensitive data in memory that is not locked, or that has been incorrectly locked, which might cause the memory to be written to swap files on disk by the virtual memory manager. This can make the data more accessible to external actors.
-On Windows systems the VirtualLock function can lock a page of memory to ensure that it will remain present in memory and not be swapped to disk. However, on older versions of Windows, such as 95, 98, or Me, the VirtualLock() function is only a stub and provides no protection. On POSIX systems the mlock() call ensures that a page will stay resident in memory but does not guarantee that the page will not appear in the swap. Therefore, it is unsuitable for use as a protection mechanism for sensitive data. Some platforms, in particular Linux, do make the guarantee that the page will not be swapped, but this is non-standard and is not portable. Calls to mlock() also require supervisor privilege. Return values for both of these calls must be checked to ensure that the lock operation was actually successful.",,unclassified,
-CWE-592,EN-Authentication Bypass Issues (Type: Class),"The software does not properly perform authentication, allowing it to be bypassed through various methods.
-On Windows systems the VirtualLock function can lock a page of memory to ensure that it will remain present in memory and not be swapped to disk. However, on older versions of Windows, such as 95, 98, or Me, the VirtualLock() function is only a stub and provides no protection. On POSIX systems the mlock() call ensures that a page will stay resident in memory but does not guarantee that the page will not appear in the swap. Therefore, it is unsuitable for use as a protection mechanism for sensitive data. Some platforms, in particular Linux, do make the guarantee that the page will not be swapped, but this is non-standard and is not portable. Calls to mlock() also require supervisor privilege. Return values for both of these calls must be checked to ensure that the lock operation was actually successful.",,unclassified,"The Art of Software Security Assessment: Chapter 2, ""Untrustworthy Credentials"", Page 37."
-CWE-593,EN-Authentication Bypass: OpenSSL CTX Object Modified after SSL Objects are Created (Type: Variant),"The software modifies the SSL context after connection creation has begun.
-If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,unclassified,
-CWE-594,EN-J2EE Framework: Saving Unserializable Objects to Disk (Type: Variant),"When the J2EE container attempts to write unserializable objects to disk there is no guarantee that the process will complete successfully.
-If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,unclassified,
-CWE-595,EN-Comparison of Object References Instead of Object Contents (Type: Base),"The program compares object references instead of the contents of the objects themselves, preventing it from detecting equivalent objects.
-If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,unclassified,
-CWE-596,EN-Incorrect Semantic Object Comparison (Type: Base),"The software does not correctly compare two objects based on their conceptual content.
-If the program modifies the SSL_CTX object after creating SSL objects from it, there is the possibility that older SSL objects created from the original context could all be affected by that change.",,unclassified,
-CWE-597,EN-Use of Wrong Operator in String Comparison (Type: Variant),"The product uses the wrong operator when comparing a string, such as using ""=="" when the equals() method should be used instead.
-In Java, using == or != to compare two strings for equality actually compares two objects for equality, not their values. Chances are good that the two references will never be equal. While this weakness often only affects program correctness, if the equality is used for a security decision, it could be leveraged to affect program security.",,unclassified,"The Art of Software Security Assessment: Chapter 6, ""Typos"", Page 289."
-CWE-598,EN-Information Exposure Through Query Strings in GET Request (Type: Variant),"The web application uses the GET method to process requests that contain sensitive information, which can expose that information through the browser's history, Referers, web logs, and other sources.
-In Java, using == or != to compare two strings for equality actually compares two objects for equality, not their values. Chances are good that the two references will never be equal. While this weakness often only affects program correctness, if the equality is used for a security decision, it could be leveraged to affect program security.",,unclassified,
-CWE-599,EN-Missing Validation of OpenSSL Certificate (Type: Variant),"The software uses OpenSSL and trusts or uses a certificate without using the SSL_get_verify_result() function to ensure that the certificate satisfies all necessary security requirements.
-This could allow an attacker to use an invalid certificate to claim to be a trusted host, use expired certificates, or conduct other attacks that could be detected if the certificate is properly validated.",,unclassified,
-CWE-6,EN-J2EE Misconfiguration: Insufficient Session-ID Length (Type: Variant),"The J2EE application is configured to use an insufficient session ID length.
-If an attacker can guess or steal a session ID, then he/she may be able to take over the user's session (called session hijacking). The number of possible session IDs increases with increased session ID length, making it more difficult to guess or steal a session ID.",,unclassified,No description: http://www.securiteam.com/securityreviews/5TP0F0UEVQ.html
-CWE-600,EN-Uncaught Exception in Servlet (Type: Base),"The Servlet does not catch all exceptions, which may reveal sensitive debugging information.
-When a Servlet throws an exception, the default error response the Servlet container sends back to the user typically includes debugging information. This information is of great value to an attacker. For example, a stack trace might show the attacker a malformed SQL query string, the type of database being used, and the version of the application container. This information enables the attacker to target known vulnerabilities in these components.",,unclassified,
-CWE-603,EN-Use of Client-Side Authentication (Type: Base),"A client/server product performs authentication within client code but not in server code, allowing server-side authentication to be bypassed via a modified client that omits the authentication check.
-Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified,"The Art of Software Security Assessment: Chapter 2, ""Untrustworthy Credentials"", Page 37."
-CWE-605,EN-Multiple Binds to the Same Port (Type: Base),"When multiple sockets are allowed to bind to the same port, other services on that port may be stolen or spoofed.
-Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified,
-CWE-606,EN-Unchecked Input for Loop Condition (Type: Base),"The product does not properly check inputs that are used for loop conditions, potentially leading to a denial of service because of excessive looping.
-Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified,"The Art of Software Security Assessment: Chapter 7, ""Looping Constructs"", Page 327."
-CWE-607,EN-Public Static Final Field References Mutable Object (Type: Variant),"A public or protected static final field references a mutable object, which allows the object to be changed by malicious code, or accidentally from another package.
-Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified,
-CWE-608,EN-Struts: Non-private Field in ActionForm Class (Type: Variant),"An ActionForm class contains a field that has not been declared private, which can be accessed without using a setter or getter.
-Client-side authentication is extremely weak and may be breached easily. Any attacker may read the source code and reverse-engineer the authentication mechanism to access parts of the application which would otherwise be protected.",,unclassified,
-CWE-609,EN-Double-Checked Locking (Type: Base),"The program uses double-checked locking to access a resource without the overhead of explicit synchronization, but the locking is insufficient.
-Double-checked locking refers to the situation where a programmer checks to see if a resource has been initialized, grabs a lock, checks again to see if the resource has been initialized, and then performs the initialization if it has not occurred yet. This should not be done, as is not guaranteed to work in all languages and on all architectures. In summary, other threads may not be operating inside the synchronous block and are not guaranteed to see the operations execute in the same order as they would appear inside the synchronous block.",,unclassified,"The ""Double-Checked Locking is Broken"" Declaration: http://www.cs.umd.edu/~pugh/java/memoryModel/DoubleCheckedLocking.html
-JSR 133 (Java Memory Model) FAQ: http://www.cs.umd.edu/~pugh/java/memoryModel/jsr-133-faq.html#dcl
-The Art of Software Security Assessment: Chapter 13, ""Threading Vulnerabilities"", Page 815."
-CWE-610,EN-Externally Controlled Reference to a Resource in Another Sphere (Type: Class),"The product uses an externally controlled name or reference that resolves to a resource that is outside of the intended control sphere.
-",,unclassified,
-CWE-611,EN-Improper Restriction of XML External Entity Reference (XXE) (Type: Variant),"The software processes an XML document that can contain XML entities with URIs that resolve to documents outside of the intended sphere of control, causing the product to embed incorrect documents into its output.
-XML documents optionally contain a Document Type Definition (DTD), which, among other features, enables the definition of XML entities. It is possible to define an entity by providing a substitution string in the form of a URI. The XML parser can access the contents of this URI and embed these contents back into the XML document for further processing.
-By submitting an XML file that defines an external entity with a file:// URI, an attacker can cause the processing application to read the contents of a local file. For example, a URI such as ""file:///c:/winnt/win.ini"" designates (in Windows) the file C:\Winnt\win.ini, or file:///etc/passwd designates the password file in Unix-based systems. Using URIs with other schemes such as http://, the attacker can force the application to make outgoing requests to servers that the attacker cannot reach directly, which can be used to bypass firewall restrictions or hide the source of attacks such as port scanning.
-Once the content of the URI is read, it is fed back into the application that is processing the XML. This application may echo back the data (e.g. in an error message), thereby exposing the file contents.",,unclassified,"XML External Entity (XXE) Processing: https://www.owasp.org/index.php/XML_External_Entity_(XXE)_Processing
-XML External Entity Attacks (XXE): https://www.owasp.org/images/5/5d/XML_Exteral_Entity_Attack.pdf
-XXE (Xml eXternal Entity) Attack: http://www.securiteam.com/securitynews/6D0100A5PU.html
-XML External Entities (XXE) Attack: http://projects.webappsec.org/w/page/13247003/XML%20External%20Entities
-XML Denial of Service Attacks and Defenses: http://msdn.microsoft.com/en-us/magazine/ee335713.aspx
-Preventing XXE in PHP: http://websec.io/2012/08/27/Preventing-XEE-in-PHP.html"
-CWE-612,EN-Information Exposure Through Indexing of Private Data (Type: Variant),"The product performs an indexing routine against private documents, but does not sufficiently verify that the actors who can access the index also have the privileges to access the private documents.
-When an indexing routine is applied against a group of private documents, and that index's results are available to outsiders who do not have access to those documents, then outsiders might be able to obtain sensitive information by conducting targeted searches. The risk is especially dangerous if search results include surrounding text that was not part of the search query. This issue can appear in search engines that are not configured (or implemented) to ignore critical files that should remain hidden; even without permissions to download these files directly, the remote user could read them.",,unclassified,
-CWE-613,EN-Insufficient Session Expiration (Type: Base),"According to WASC, ""Insufficient Session Expiration is when a web site permits an attacker to reuse old session credentials or session IDs for authorization.""
-When an indexing routine is applied against a group of private documents, and that index's results are available to outsiders who do not have access to those documents, then outsiders might be able to obtain sensitive information by conducting targeted searches. The risk is especially dangerous if search results include surrounding text that was not part of the search query. This issue can appear in search engines that are not configured (or implemented) to ignore critical files that should remain hidden; even without permissions to download these files directly, the remote user could read them.",,unclassified,
-CWE-614,EN-Sensitive Cookie in HTTPS Session Without Secure Attribute (Type: Variant),"The Secure attribute for sensitive cookies in HTTPS sessions is not set, which could cause the user agent to send those cookies in plaintext over an HTTP session.
-When an indexing routine is applied against a group of private documents, and that index's results are available to outsiders who do not have access to those documents, then outsiders might be able to obtain sensitive information by conducting targeted searches. The risk is especially dangerous if search results include surrounding text that was not part of the search query. This issue can appear in search engines that are not configured (or implemented) to ignore critical files that should remain hidden; even without permissions to download these files directly, the remote user could read them.",,unclassified,
-CWE-615,EN-Information Exposure Through Comments (Type: Variant),"While adding general comments is very useful, some programmers tend to leave important data, such as: filenames related to the web application, old links or links which were not meant to be browsed by users, old code fragments, etc.
-An attacker who finds these comments can map the application's structure and files, expose hidden parts of the site, and study the fragments of code to reverse engineer the application, which may help develop further attacks against the site.",,unclassified,
-CWE-616,EN-Incomplete Identification of Uploaded File Variables (PHP) (Type: Variant),"The PHP application uses an old method for processing uploaded files by referencing the four global variables that are set for each file (e.g. $varname, $varname_size, $varname_name, $varname_type). These variables could be overwritten by attackers, causing the application to process unauthorized files.
-These global variables could be overwritten by POST requests, cookies, or other methods of populating or overwriting these variables. This could be used to read or process arbitrary files by providing values such as ""/etc/passwd"".",,unclassified,"A Study in Scarlet - section 5, ""File Upload"""
-CWE-617,EN-Reachable Assertion (Type: Variant),"The product contains an assert() or similar statement that can be triggered by an attacker, which leads to an application exit or other behavior that is more severe than necessary.
-For example, if a server handles multiple simultaneous connections, and an assert() occurs in one single connection that causes all other connections to be dropped, this is a reachable assertion that leads to a denial of service.",,unclassified,
-CWE-618,EN-Exposed Unsafe ActiveX Method (Type: Base),"An ActiveX control is intended for use in a web browser, but it exposes dangerous methods that perform actions that are outside of the browser's security model (e.g. the zone or domain).
-ActiveX controls can exercise far greater control over the operating system than typical Java or javascript. Exposed methods can be subject to various vulnerabilities, depending on the implemented behaviors of those methods, and whether input validation is performed on the provided arguments. If there is no integrity checking or origin validation, this method could be invoked by attackers.",,unclassified,"No description: http://msdn.microsoft.com/workshop/components/activex/safety.asp
-No description: http://msdn.microsoft.com/workshop/components/activex/security.asp
-The Art of Software Security Assessment: Chapter 12, ""ActiveX Security"", Page 749."
-CWE-619,EN-Dangling Database Cursor (Cursor Injection) (Type: Base),"If a database cursor is not closed properly, then it could become accessible to other users while retaining the same privileges that were originally assigned, leaving the cursor ""dangling.""
-For example, an improper dangling cursor could arise from unhandled exceptions. The impact of the issue depends on the cursor's role, but SQL injection attacks are commonly possible.",,unclassified,"The Oracle Hacker's Handbook
-Cursor Injection: http://www.databasesecurity.com/dbsec/cursor-injection.pdf"
-CWE-62,EN-UNIX Hard Link (Type: Variant),"The software, when opening a file or directory, does not sufficiently account for when the name is associated with a hard link to a target that is outside of the intended control sphere. This could allow an attacker to cause the software to operate on unauthorized files.
-Failure for a system to check for hard links can result in vulnerability to different types of attacks. For example, an attacker can escalate their privileges if a file used by a privileged program is replaced with a hard link to a sensitive file (e.g. /etc/passwd). When the process opens the file, the attacker can assume the privileges of that process.",,unclassified,"The Art of Software Security Assessment: Chapter 9, ""Hard Links"", Page 518."
-CWE-620,EN-Unverified Password Change (Type: Variant),"When setting a new password for a user, the product does not require knowledge of the original password, or using another form of authentication.
-This could be used by an attacker to change passwords for another user, thus gaining the privileges associated with that user.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 19: Use of Weak Password-Based Systems."" Page 279"
-CWE-621,EN-Variable Extraction Error (Type: Base),"The product uses external input to determine the names of variables into which information is extracted, without verifying that the names of the specified variables are valid. This could cause the program to overwrite unintended variables.
-For example, in PHP, calling extract() or import_request_variables() without the proper arguments could allow arbitrary global variables to be overwritten, including superglobals. Similar functionality might be possible in other interpreted languages, including custom languages.",,unclassified,
-CWE-622,EN-Improper Validation of Function Hook Arguments (Type: Variant),"A product adds hooks to user-accessible API functions, but does not properly validate the arguments. This could lead to resultant vulnerabilities.
-Such hooks can be used in defensive software that runs with privileges, such as anti-virus or firewall, which hooks kernel calls. When the arguments are not validated, they could be used to bypass the protection scheme or attack the product itself.",,unclassified,
-CWE-623,EN-Unsafe ActiveX Control Marked Safe For Scripting (Type: Variant),"An ActiveX control is intended for restricted use, but it has been marked as safe-for-scripting.
-This might allow attackers to use dangerous functionality via a web page that accesses the control, which can lead to different resultant vulnerabilities, depending on the control's behavior.",,unclassified,"No description: http://msdn.microsoft.com/workshop/components/activex/safety.asp
-No description: http://msdn.microsoft.com/workshop/components/activex/security.asp
-No description: http://support.microsoft.com/kb/240797
-Writing Secure Code: Chapter 16, ""What ActiveX Components Are Safe for Initialization and Safe for Scripting?"" Page 510
-The Art of Software Security Assessment: Chapter 12, ""ActiveX Security"", Page 749."
-CWE-624,EN-Executable Regular Expression Error (Type: Base),"The product uses a regular expression that either (1) contains an executable component with user-controlled inputs, or (2) allows a user to enable execution by inserting pattern modifiers.
-Case (2) is possible in the PHP preg_replace() function, and possibly in other languages when a user-controlled input is inserted into a string that is later parsed as a regular expression.",,unclassified,
-CWE-625,EN-Permissive Regular Expression (Type: Base),"The product uses a regular expression that does not sufficiently restrict the set of allowed values.
-This effectively causes the regexp to accept substrings that match the pattern, which produces a partial comparison to the target. In some cases, this can lead to other weaknesses. Common errors include:
-not identifying the beginning and end of the target string
-using wildcards instead of acceptable character ranges
-others",,unclassified,"The Art of Software Security Assessment: Chapter 8, ""Character Stripping Vulnerabilities"", Page 437."
-CWE-626,EN-Null Byte Interaction Error (Poison Null Byte) (Type: Variant),"The product does not properly handle null bytes or NUL characters when passing data between different representations or components.
-A null byte (NUL character) can have different meanings across representations or languages. For example, it is a string terminator in standard C libraries, but Perl and PHP strings do not treat it as a terminator. When two representations are crossed - such as when Perl or PHP invokes underlying C functionality - this can produce an interaction error with unexpected results. Similar issues have been reported for ASP. Other interpreters written in C might also be affected.",,unclassified,"Poison NULL byte: http://insecure.org/news/P55-07.txt
-0x00 vs ASP file upload scripts: http://www.security-assessment.com/Whitepapers/0x00_vs_ASP_File_Uploads.pdf
-ShAnKaR: multiple PHP application poison NULL byte vulnerability: http://seclists.org/fulldisclosure/2006/Sep/0185.html"
-CWE-627,EN-Dynamic Variable Evaluation (Type: Base),"In a language where the user can influence the name of a variable at runtime, if the variable names are not controlled, an attacker can read or write to arbitrary variables, or access arbitrary functions.
-The resultant vulnerabilities depend on the behavior of the application, both at the crossover point and in any control/data flow that is reachable by the related variables or functions.",,unclassified,"Dynamic Evaluation Vulnerabilities in PHP applications: http://seclists.org/fulldisclosure/2006/May/0035.html
-A Study In Scarlet: Exploiting Common Vulnerabilities in PHP Applications: http://www.securereality.com.au/studyinscarlet.txt"
-CWE-628,EN-Function Call with Incorrectly Specified Arguments (Type: Base),"The product calls a function, procedure, or routine with arguments that are not correctly specified, leading to always-incorrect behavior and resultant weaknesses.
-There are multiple ways in which this weakness can be introduced, including:
-the wrong variable or reference;
-an incorrect number of arguments;
-incorrect order of arguments;
-wrong type of arguments; or
-wrong value.",,unclassified,
-CWE-636,EN-Not Failing Securely (Failing Open) (Type: Class),"When the product encounters an error condition or failure, its design requires it to fall back to a state that is less secure than other options that are available, such as selecting the weakest encryption algorithm or using the most permissive access control restrictions.
-By entering a less secure state, the product inherits the weaknesses associated with that state, making it easier to compromise. At the least, it causes administrators to have a false sense of security. This weakness typically occurs as a result of wanting to ""fail functional"" to minimize administration and support costs, instead of ""failing safe.""",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/
-Failing Securely: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/349.html"
-CWE-637,EN-Unnecessary Complexity in Protection Mechanism (Not Using Economy of Mechanism) (Type: Class),"The software uses a more complex mechanism than necessary, which could lead to resultant weaknesses when the mechanism is not correctly understood, modeled, configured, implemented, or used.
-Security mechanisms should be as simple as possible. Complex security mechanisms may engender partial implementations and compatibility problems, with resulting mismatches in assumptions and implemented security. A corollary of this principle is that data specifications should be as simple as possible, because complex data specifications result in complex validation code. Complex tasks and systems may also need to be guarded by complex security checks, so simple systems should be preferred.",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/
-Economy of Mechanism: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/348.html"
-CWE-638,EN-Not Using Complete Mediation (Type: Class),"The software does not perform access checks on a resource every time the resource is accessed by an entity, which can create resultant weaknesses if that entity's rights or privileges change over time.
-",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/
-Complete Mediation: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/346.html"
-CWE-65,EN-Windows Hard Link (Type: Variant),"The software, when opening a file or directory, does not sufficiently handle when the name is associated with a hard link to a target that is outside of the intended control sphere. This could allow an attacker to cause the software to operate on unauthorized files.
-Failure for a system to check for hard links can result in vulnerability to different types of attacks. For example, an attacker can escalate their privileges if a file used by a privileged program is replaced with a hard link to a sensitive file (e.g. AUTOEXEC.BAT). When the process opens the file, the attacker can assume the privileges of that process, or prevent the program from accurately processing data.",,unclassified,"The Art of Software Security Assessment: Chapter 11, ""Links"", Page 676."
-CWE-651,EN-Information Exposure Through WSDL File (Type: Variant),"The Web services architecture may require exposing a WSDL file that contains information on the publicly accessible services and how callers of these services should interact with them (e.g. what parameters they expect and what types they return).
-An information exposure may occur if any of the following apply:
-The WSDL file is accessible to a wider audience than intended.
-The WSDL file contains information on the methods/services that should not be publicly accessible or information about deprecated methods. This problem is made more likely due to the WSDL often being automatically generated from the code.
-Information in the WSDL file helps guess names/locations of methods/resources that should not be publicly accessible.",,unclassified,
-CWE-653,EN-Insufficient Compartmentalization (Type: Base),"The product does not sufficiently compartmentalize functionality or processes that require different privilege levels, rights, or permissions.
-When a weakness occurs in functionality that is accessible by lower-privileged users, then without strong boundaries, an attack might extend the scope of the damage to higher-privileged users.",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/
-Separation of Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/357.html"
-CWE-654,EN-Reliance on a Single Factor in a Security Decision (Type: Base),"A protection mechanism relies exclusively, or to a large extent, on the evaluation of a single condition or the integrity of a single object or entity in order to make a decision about granting access to restricted resources or functionality.
-When a weakness occurs in functionality that is accessible by lower-privileged users, then without strong boundaries, an attack might extend the scope of the damage to higher-privileged users.",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/
-Separation of Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/357.html"
-CWE-655,EN-Insufficient Psychological Acceptability (Type: Base),"The software has a protection mechanism that is too difficult or inconvenient to use, encouraging non-malicious users to disable or bypass the mechanism, whether by accident or on purpose.
-When a weakness occurs in functionality that is accessible by lower-privileged users, then without strong boundaries, an attack might extend the scope of the damage to higher-privileged users.",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/
-Psychological Acceptability: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/354.html
-Usability of Security: A Case Study: http://reports-archive.adm.cs.cmu.edu/anon/1998/CMU-CS-98-155.pdf
-24 Deadly Sins of Software Security: ""Sin 14: Poor Usability."" Page 217"
-CWE-656,EN-Reliance on Security Through Obscurity (Type: Base),"The software uses a protection mechanism whose strength depends heavily on its obscurity, such that knowledge of its algorithms or key data is sufficient to defeat the mechanism.
-This reliance on ""security through obscurity"" can produce resultant weaknesses if an attacker is able to reverse engineer the inner workings of the mechanism. Note that obscurity can be one small part of defense in depth, since it can create more work for an attacker; however, it is a significant risk if used as the primary means of protection.",,unclassified,"RFC: 793, TRANSMISSION CONTROL PROTOCOL: http://www.ietf.org/rfc/rfc0793.txt
-The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/
-Never Assuming that Your Secrets Are Safe: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/352.html"
-CWE-657,EN-Violation of Secure Design Principles (Type: Class),"The product violates well-established principles for secure design.
-This can introduce resultant weaknesses or make it easier for developers to introduce related weaknesses during implementation. Because code is centered around design, it can be resource-intensive to fix design problems.",,unclassified,"The Protection of Information in Computer Systems: http://web.mit.edu/Saltzer/www/publications/protection/
-Design Principles: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/358.html"
-CWE-66,EN-Improper Handling of File Names that Identify Virtual Resources (Type: Base),"The product does not handle or incorrectly handles a file name that identifies a ""virtual"" resource that is not directly specified within the directory that is associated with the file name, causing the product to perform file-based operations on a resource that is not a file.
-Virtual file names are represented like normal file names, but they are effectively aliases for other resources that do not behave like normal files. Depending on their functionality, they could be alternate entities. They are not necessarily listed in directories.",,unclassified,
-CWE-662,EN-Improper Synchronization (Type: Base),"The software attempts to use a shared resource in an exclusive manner, but does not prevent or incorrectly prevents use of the resource by another thread or process.
-Virtual file names are represented like normal file names, but they are effectively aliases for other resources that do not behave like normal files. Depending on their functionality, they could be alternate entities. They are not necessarily listed in directories.",,unclassified,
-CWE-663,EN-Use of a Non-reentrant Function in a Concurrent Context (Type: Base),"The software calls a non-reentrant function in a concurrent context in which a competing code sequence (e.g. thread or signal handler) may have an opportunity to call the same function or otherwise influence its state.
-Virtual file names are represented like normal file names, but they are effectively aliases for other resources that do not behave like normal files. Depending on their functionality, they could be alternate entities. They are not necessarily listed in directories.",,unclassified,"Java Concurrency API: http://java.sun.com/j2se/1.5.0/docs/api/java/util/concurrent/locks/ReentrantLock.html
-Use reentrant functions for safer signal handling: http://www.ibm.com/developerworks/linux/library/l-reent.html"
-CWE-664,EN-Improper Control of a Resource Through its Lifetime (Type: Class),"The software does not maintain or incorrectly maintains control over a resource throughout its lifetime of creation, use, and release.
-Resources often have explicit instructions on how to be created, used and destroyed. When software does not follow these instructions, it can lead to unexpected behaviors and potentially exploitable states.
-Even without explicit instructions, various principles are expected to be adhered to, such as ""Do not use an object until after its creation is complete,"" or ""do not use an object after it has been slated for destruction.""",,unclassified,
-CWE-666,EN-Operation on Resource in Wrong Phase of Lifetime (Type: Base),"The software performs an operation on a resource at the wrong phase of the resource's lifecycle, which can lead to unexpected behaviors.
-When a developer wants to initialize, use or release a resource, it is important to follow the specifications outlined for how to operate on that resource and to ensure that the resource is in the expected state. In this case, the software wants to perform a normally valid operation, initialization, use or release, on a resource when it is in the incorrect phase of its lifetime.",,unclassified,
-CWE-667,EN-Improper Locking (Type: Base),"The software does not properly acquire a lock on a resource, or it does not properly release a lock on a resource, leading to unexpected resource state changes and behaviors.
-When a developer wants to initialize, use or release a resource, it is important to follow the specifications outlined for how to operate on that resource and to ensure that the resource is in the expected state. In this case, the software wants to perform a normally valid operation, initialization, use or release, on a resource when it is in the incorrect phase of its lifetime.",,unclassified,
-CWE-668,EN-Exposure of Resource to Wrong Sphere (Type: Class),"The product exposes a resource to the wrong control sphere, providing unintended actors with inappropriate access to the resource.
-Resources such as files and directories may be inadvertently exposed through mechanisms such as insecure permissions, or when a program accidentally operates on the wrong object. For example, a program may intend that private files can only be provided to a specific user. This effectively defines a control sphere that is intended to prevent attackers from accessing these private files. If the file permissions are insecure, then parties other than the user will be able to access those files.
-A separate control sphere might effectively require that the user can only access the private files, but not any other files on the system. If the program does not ensure that the user is only requesting private files, then the user might be able to access other files on the system.
-In either case, the end result is that a resource has been exposed to the wrong party.",,unclassified,
-CWE-669,EN-Incorrect Resource Transfer Between Spheres (Type: Class),"The product does not properly transfer a resource/behavior to another sphere, or improperly imports a resource/behavior from another sphere, in a manner that provides unintended control over that resource.
-Resources such as files and directories may be inadvertently exposed through mechanisms such as insecure permissions, or when a program accidentally operates on the wrong object. For example, a program may intend that private files can only be provided to a specific user. This effectively defines a control sphere that is intended to prevent attackers from accessing these private files. If the file permissions are insecure, then parties other than the user will be able to access those files.
-A separate control sphere might effectively require that the user can only access the private files, but not any other files on the system. If the program does not ensure that the user is only requesting private files, then the user might be able to access other files on the system.
-In either case, the end result is that a resource has been exposed to the wrong party.",,unclassified,
-CWE-670,EN-Always-Incorrect Control Flow Implementation (Type: Class),"The code contains a control flow path that does not reflect the algorithm that the path is intended to implement, leading to incorrect behavior any time this path is navigated.
-This weakness captures cases in which a particular code segment is always incorrect with respect to the algorithm that it is implementing. For example, if a C programmer intends to include multiple statements in a single block but does not include the enclosing braces (CWE-483), then the logic is always incorrect. This issue is in contrast to most weaknesses in which the code usually behaves correctly, except when it is externally manipulated in malicious ways.",,unclassified,
-CWE-671,EN-Lack of Administrator Control over Security (Type: Class),"The product uses security features in a way that prevents the product's administrator from tailoring security settings to reflect the environment in which the product is being used. This introduces resultant weaknesses or prevents it from operating at a level of security that is desired by the administrator.
-If the product's administrator does not have the ability to manage security-related decisions at all times, then protecting the product from outside threats - including the product's developer - can become impossible. For example, a hard-coded account name and password cannot be changed by the administrator, thus exposing that product to attacks that the administrator can not prevent.",,unclassified,
-CWE-672,EN-Operation on a Resource after Expiration or Release (Type: Base),"The software uses, accesses, or otherwise operates on a resource after that resource has been expired, released, or revoked.
-If the product's administrator does not have the ability to manage security-related decisions at all times, then protecting the product from outside threats - including the product's developer - can become impossible. For example, a hard-coded account name and password cannot be changed by the administrator, thus exposing that product to attacks that the administrator can not prevent.",,unclassified,
-CWE-673,EN-External Influence of Sphere Definition (Type: Class),"The product does not prevent the definition of control spheres from external actors.
-Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,unclassified,
-CWE-674,EN-Uncontrolled Recursion (Type: Base),"The product does not properly control the amount of recursion that takes place, which consumes excessive resources, such as allocated memory or the program stack.
-Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,unclassified,
-CWE-675,EN-Duplicate Operations on Resource (Type: Class),"The product performs the same operation on a resource two or more times, when the operation should only be applied once.
-Typically, a product defines its control sphere within the code itself, or through configuration by the product's administrator. In some cases, an external party can change the definition of the control sphere. This is typically a resultant weakness.",,unclassified,
-CWE-683,EN-Function Call With Incorrect Order of Arguments (Type: Variant),"The software calls a function, procedure, or routine, but the caller specifies the arguments in an incorrect order, leading to resultant weaknesses.
-While this weakness might be caught by the compiler in some languages, it can occur more frequently in cases in which the called function accepts variable numbers or types of arguments, such as format strings in C. It also can occur in languages or environments that do not enforce strong typing.",,unclassified,
-CWE-684,EN-Incorrect Provision of Specified Functionality (Type: Base),"The code does not function according to its published specifications, potentially leading to incorrect usage.
-When providing functionality to an external party, it is important that the software behaves in accordance with the details specified. When requirements of nuances are not documented, the functionality may produce unintended behaviors for the caller, possibly leading to an exploitable state.",,unclassified,
-CWE-685,EN-Function Call With Incorrect Number of Arguments (Type: Variant),"The software calls a function, procedure, or routine, but the caller specifies too many arguments, or too few arguments, which may lead to undefined behavior and resultant weaknesses.
-When providing functionality to an external party, it is important that the software behaves in accordance with the details specified. When requirements of nuances are not documented, the functionality may produce unintended behaviors for the caller, possibly leading to an exploitable state.",,unclassified,
-CWE-686,EN-Function Call With Incorrect Argument Type (Type: Variant),"The software calls a function, procedure, or routine, but the caller specifies an argument that is the wrong data type, which may lead to resultant weaknesses.
-This weakness is most likely to occur in loosely typed languages, or in strongly typed languages in which the types of variable arguments cannot be enforced at compilation time, or where there is implicit casting.",,unclassified,
-CWE-687,EN-Function Call With Incorrectly Specified Argument Value (Type: Variant),"The software calls a function, procedure, or routine, but the caller specifies an argument that contains the wrong value, which may lead to resultant weaknesses.
-This weakness is most likely to occur in loosely typed languages, or in strongly typed languages in which the types of variable arguments cannot be enforced at compilation time, or where there is implicit casting.",,unclassified,
-CWE-688,EN-Function Call With Incorrect Variable or Reference as Argument (Type: Variant),"The software calls a function, procedure, or routine, but the caller specifies the wrong variable or reference as one of the arguments, which may lead to undefined behavior and resultant weaknesses.
-This weakness is most likely to occur in loosely typed languages, or in strongly typed languages in which the types of variable arguments cannot be enforced at compilation time, or where there is implicit casting.",,unclassified,
-CWE-69,EN-Improper Handling of Windows ::DATA Alternate Data Stream (Type: Variant),"The software does not properly prevent access to, or detect usage of, alternate data streams (ADS).
-An attacker can use an ADS to hide information about a file (e.g. size, the name of the process) from a system or file browser tools such as Windows Explorer and 'dir' at the command line utility. Alternately, the attacker might be able to bypass intended access restrictions for the associated data fork.",,unclassified,"Windows NTFS Alternate Data Streams: http://www.securityfocus.com/infocus/1822
-Writing Secure Code"
-CWE-691,EN-Insufficient Control Flow Management (Type: Class),"The code does not sufficiently manage its control flow during execution, creating conditions in which the control flow can be modified in unexpected ways.
-An attacker can use an ADS to hide information about a file (e.g. size, the name of the process) from a system or file browser tools such as Windows Explorer and 'dir' at the command line utility. Alternately, the attacker might be able to bypass intended access restrictions for the associated data fork.",,unclassified,
-CWE-693,EN-Protection Mechanism Failure (Type: Class),"The product does not use or incorrectly uses a protection mechanism that provides sufficient defense against directed attacks against the product.
-This weakness covers three distinct situations. A ""missing"" protection mechanism occurs when the application does not define any mechanism against a certain class of attack. An ""insufficient"" protection mechanism might provide some defenses - for example, against the most common attacks - but it does not protect against everything that is intended. Finally, an ""ignored"" mechanism occurs when a mechanism is available and in active use within the product, but the developer has not applied it in some code path.",,unclassified,
-CWE-694,EN-Use of Multiple Resources with Duplicate Identifier (Type: Base),"The software uses multiple resources that can have the same identifier, in a context in which unique identifiers are required.
-If the software assumes that each resource has a unique identifier, the software could operate on the wrong resource if attackers can cause multiple resources to be associated with the same identifier.",,unclassified,
-CWE-695,EN-Use of Low-Level Functionality (Type: Base),"The software uses low-level functionality that is explicitly prohibited by the framework or specification under which the software is supposed to operate.
-The use of low-level functionality can violate the specification in unexpected ways that effectively disable built-in protection mechanisms, introduce exploitable inconsistencies, or otherwise expose the functionality to attack.",,unclassified,
-CWE-696,EN-Incorrect Behavior Order (Type: Class),"The software performs multiple related behaviors, but the behaviors are performed in the wrong order in ways which may produce resultant weaknesses.
-The use of low-level functionality can violate the specification in unexpected ways that effectively disable built-in protection mechanisms, introduce exploitable inconsistencies, or otherwise expose the functionality to attack.",,unclassified,
-CWE-697,EN-Insufficient Comparison (Type: Class),"The software compares two entities in a security-relevant context, but the comparison is insufficient, which may lead to resultant weaknesses.
-This weakness class covers several possibilities:
-the comparison checks one factor incorrectly;
-the comparison should consider multiple factors, but it does not check some of those factors at all.",,unclassified,
-CWE-698,EN-Execution After Redirect (EAR) (Type: Base),"The web application sends a redirect to another location, but instead of exiting, it executes additional code.
-This weakness class covers several possibilities:
-the comparison checks one factor incorrectly;
-the comparison should consider multiple factors, but it does not check some of those factors at all.",,unclassified,Fear the EAR: Discovering and Mitigating Execution After Redirect Vulnerabilities: http://cs.ucsb.edu/~bboe/public/pubs/fear-the-ear-ccs2011.pdf
-CWE-7,EN-J2EE Misconfiguration: Missing Custom Error Page (Type: Variant),"The default error page of a web application should not display sensitive information about the software system.
-A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified,19 Deadly Sins of Software Security
-CWE-703,EN-Improper Check or Handling of Exceptional Conditions (Type: Class),"The software does not properly anticipate or handle exceptional conditions that rarely occur during normal operation of the software.
-A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified,"A Taxonomy of Security Faults in the UNIX Operating System: http://ftp.cerias.purdue.edu/pub/papers/taimur-aslam/aslam-taxonomy-msthesis.pdf
-Use of A Taxonomy of Security Faults: http://csrc.nist.gov/nissc/1996/papers/NISSC96/paper057/PAPER.PDF
-24 Deadly Sins of Software Security: ""Sin 8: C++ Catastrophes."" Page 143"
-CWE-704,EN-Incorrect Type Conversion or Cast (Type: Class),"The software does not correctly convert an object, resource or structure from one type to a different type.
-A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified,
-CWE-705,EN-Incorrect Control Flow Scoping (Type: Class),"The software does not properly return control flow to the proper location after it has completed a task or detected an unusual condition.
-A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified,
-CWE-706,EN-Use of Incorrectly-Resolved Name or Reference (Type: Class),"The software uses a name or reference to access a resource, but the name/reference resolves to a resource that is outside of the intended control sphere.
-A Web application must define a default error page for 4xx errors (e.g. 404), 5xx (e.g. 500) errors and catch java.lang.Throwable exceptions to prevent attackers from mining information from the application container's built-in error response.",,unclassified,
-CWE-707,EN-Improper Enforcement of Message or Data Structure (Type: Class),"The software does not enforce or incorrectly enforces that structured messages or data are well-formed before being read from an upstream component or sent to a downstream component.
-If a message is malformed it may cause the message to be incorrectly interpreted.
-This weakness typically applies in cases where the product prepares a control message that another process must act on, such as a command or query, and malicious input that was intended as data, can enter the control plane instead. However, this weakness also applies to more general cases where there are not always control implications.",,unclassified,
-CWE-708,EN-Incorrect Ownership Assignment (Type: Base),"The software assigns an owner to a resource, but the owner is outside of the intended control sphere.
-This may allow the resource to be manipulated by actors outside of the intended control sphere.",,unclassified,
-CWE-71,EN-Apple .DS_Store (Type: Variant),"Software operating in a MAC OS environment, where .DS_Store is in effect, must carefully manage hard links, otherwise an attacker may be able to leverage a hard link from .DS_Store to overwrite arbitrary files and gain privileges.
-This may allow the resource to be manipulated by actors outside of the intended control sphere.",,unclassified,
-CWE-710,EN-Coding Standards Violation (Type: Class),"The software does not follow certain coding rules for development, which can lead to resultant weaknesses or increase the severity of the associated vulnerabilities.
-This may allow the resource to be manipulated by actors outside of the intended control sphere.",,unclassified,
-CWE-72,EN-Improper Handling of Apple HFS+ Alternate Data Stream Path (Type: Variant),"The software does not properly handle special paths that may identify the data or resource fork of a file on the HFS+ file system.
-If the software chooses actions to take based on the file name, then if an attacker provides the data or resource fork, the software may take unexpected actions. Further, if the software intends to restrict access to a file, then an attacker might still be able to bypass intended access restrictions by requesting the data or resource fork for that file.",,unclassified,No description: http://docs.info.apple.com/article.html?artnum=300422
-CWE-733,EN-Compiler Optimization Removal or Modification of Security-critical Code (Type: Base),"The developer builds a security-critical protection mechanism into the software but the compiler optimizes the program such that the mechanism is removed or modified.
-When a resource is given a permissions setting that provides access to a wider range of actors than required, it could lead to the exposure of sensitive information, or the modification of that resource by unintended parties. This is especially dangerous when the resource is related to program configuration, execution or sensitive user data.",,unclassified,"Writing Secure Code: Chapter 9, ""A Compiler Optimization Caveat"" Page 322"
-CWE-75,EN-Failure to Sanitize Special Elements into a Different Plane (Special Element Injection) (Type: Class),"The software does not adequately filter user-controlled input for special elements with control implications.
-This weakness can lead to a wide variety of resultant weaknesses, depending on the behavior of the exposed method. It can apply to any number of technologies and approaches, such as ActiveX controls, Java functions, IOCTLs, and so on.
-The exposure can occur in a few different ways:
-1) The function/method was never intended to be exposed to outside actors.
-2) The function/method was only intended to be accessible to a limited set of actors, such as Internet-based access from a single web site.",,unclassified,
-CWE-756,EN-Missing Custom Error Page (Type: Class),"The software does not return custom error pages to the user, possibly exposing sensitive information.
-The programmer may assume that certain events or conditions will never occur or do not need to be worried about, such as low memory conditions, lack of access to resources due to restrictive permissions, or misbehaving clients or components. However, attackers may intentionally trigger these unusual conditions, thus violating the programmer's assumptions, possibly introducing instability, incorrect behavior, or a vulnerability.
-Note that this entry is not exclusively about the use of exceptions and exception handling, which are mechanisms for both checking and handling unusual or unexpected conditions.",,unclassified,
-CWE-757,EN-Selection of Less-Secure Algorithm During Negotiation (Algorithm Downgrade) (Type: Class),"A protocol or its implementation supports interaction between multiple actors and allows those actors to negotiate which algorithm should be used as a protection mechanism such as encryption or authentication, but it does not select the strongest algorithm that is available to both parties.
-When a security mechanism can be forced to downgrade to use a less secure algorithm, this can make it easier for attackers to compromise the software by exploiting weaker algorithm. The victim might not be aware that the less secure algorithm is being used. For example, if an attacker can force a communications channel to use cleartext instead of strongly-encrypted data, then the attacker could read the channel by sniffing, instead of going through extra effort of trying to decrypt the data using brute force techniques.",,unclassified,
-CWE-758,"EN-Reliance on Undefined, Unspecified, or Implementation-Defined Behavior (Type: Class)","The software uses an API function, data structure, or other entity in a way that relies on properties that are not always guaranteed to hold for that entity.
-This can lead to resultant weaknesses when the required properties change, such as when the software is ported to a different platform or if an interaction error (CWE-435) occurs.",,unclassified,
-CWE-759,EN-Use of a One-Way Hash without a Salt (Type: Base),"The software uses a one-way cryptographic hash against an input that should not be reversible, such as a password, but the software does not also use a salt as part of the input.
-This makes it easier for attackers to pre-compute the hash value using dictionary attack techniques such as rainbow tables.
-It should be noted that, despite common perceptions, the use of a good salt with a hash does not sufficiently increase the effort for an attacker who is targeting an individual password, or who has a large amount of computing resources available, such as with cloud-based services or specialized, inexpensive hardware. Offline password cracking can still be effective if the hash function is not expensive to compute; many cryptographic functions are designed to be efficient and can be vulnerable to attacks using massive computing resources, even if the hash is cryptographically strong. The use of a salt only slightly increases the computing requirements for an attacker compared to other strategies such as adaptive hash functions. See CWE-916 for more details.",,unclassified,"bcrypt: http://bcrypt.sourceforge.net/
-Tarsnap - The scrypt key derivation function and encryption utility: http://www.tarsnap.com/scrypt.html
-RFC2898 - PKCS #5: Password-Based Cryptography Specification Version 2.0: http://tools.ietf.org/html/rfc2898
-How Companies Can Beef Up Password Security (interview with Thomas H. Ptacek): http://krebsonsecurity.com/2012/06/how-companies-can-beef-up-password-security/
-Speed Hashing: http://www.codinghorror.com/blog/2012/04/speed-hashing.html
-Password security: past, present, future: http://www.openwall.com/presentations/PHDays2012-Password-Security/
-Password Storage Cheat Sheet: https://www.owasp.org/index.php/Password_Storage_Cheat_Sheet
-Enough With The Rainbow Tables: What You Need To Know About Secure Password Schemes: http://www.securityfocus.com/blogs/262
-The Importance of Being Canonical: http://erratasec.blogspot.com/2009/02/importance-of-being-canonical.html
-Password Hashing: http://phpsec.org/articles/2005/password-hashing.html
-Rainbow Hash Cracking: http://www.codinghorror.com/blog/archives/000949.html
-Rainbow table: http://en.wikipedia.org/wiki/Rainbow_table
-Writing Secure Code: Chapter 9, ""Creating a Salted Hash"" Page 302
-The Art of Software Security Assessment: Chapter 2, ""Salt Values"", Page 46.
-How To Safely Store A Password: http://codahale.com/how-to-safely-store-a-password/
-Our password hashing has no clothes: http://www.troyhunt.com/2012/06/our-password-hashing-has-no-clothes.html
-Should we really use bcrypt/scrypt?: http://www.analyticalengine.net/2012/06/should-we-really-use-bcryptscrypt/"
-CWE-760,EN-Use of a One-Way Hash with a Predictable Salt (Type: Base),"The software uses a one-way cryptographic hash against an input that should not be reversible, such as a password, but the software uses a predictable salt as part of the input.
-This makes it easier for attackers to pre-compute the hash value using dictionary attack techniques such as rainbow tables, effectively disabling the protection that an unpredictable salt would provide.
-It should be noted that, despite common perceptions, the use of a good salt with a hash does not sufficiently increase the effort for an attacker who is targeting an individual password, or who has a large amount of computing resources available, such as with cloud-based services or specialized, inexpensive hardware. Offline password cracking can still be effective if the hash function is not expensive to compute; many cryptographic functions are designed to be efficient and can be vulnerable to attacks using massive computing resources, even if the hash is cryptographically strong. The use of a salt only slightly increases the computing requirements for an attacker compared to other strategies such as adaptive hash functions. See CWE-916 for more details.",,unclassified,"bcrypt: http://bcrypt.sourceforge.net/
-Tarsnap - The scrypt key derivation function and encryption utility: http://www.tarsnap.com/scrypt.html
-RFC2898 - PKCS #5: Password-Based Cryptography Specification Version 2.0: http://tools.ietf.org/html/rfc2898
-How Companies Can Beef Up Password Security (interview with Thomas H. Ptacek): http://krebsonsecurity.com/2012/06/how-companies-can-beef-up-password-security/
-Speed Hashing: http://www.codinghorror.com/blog/2012/04/speed-hashing.html
-Password security: past, present, future: http://www.openwall.com/presentations/PHDays2012-Password-Security/
-Password Storage Cheat Sheet: https://www.owasp.org/index.php/Password_Storage_Cheat_Sheet
-Enough With The Rainbow Tables: What You Need To Know About Secure Password Schemes: http://www.securityfocus.com/blogs/262
-The Importance of Being Canonical: http://erratasec.blogspot.com/2009/02/importance-of-being-canonical.html
-Password Hashing: http://phpsec.org/articles/2005/password-hashing.html
-Rainbow Hash Cracking: http://www.codinghorror.com/blog/archives/000949.html
-Rainbow table: http://en.wikipedia.org/wiki/Rainbow_table
-Writing Secure Code: Chapter 9, ""Creating a Salted Hash"" Page 302
-The Art of Software Security Assessment: Chapter 2, ""Salt Values"", Page 46.
-How To Safely Store A Password: http://codahale.com/how-to-safely-store-a-password/
-Our password hashing has no clothes: http://www.troyhunt.com/2012/06/our-password-hashing-has-no-clothes.html
-Should we really use bcrypt/scrypt?: http://www.analyticalengine.net/2012/06/should-we-really-use-bcryptscrypt/"
-CWE-761,EN-Free of Pointer not at Start of Buffer (Type: Variant),"The application calls free() on a pointer to a memory resource that was allocated on the heap, but the pointer is not at the start of the buffer.
-This can cause the application to crash, or in some cases, modify critical program variables or execute code.
-This weakness often occurs when the memory is allocated explicitly on the heap with one of the malloc() family functions and free() is called, but pointer arithmetic has caused the pointer to be in the interior or end of the buffer.",,unclassified,"boost C++ Library Smart Pointers: http://www.boost.org/doc/libs/1_38_0/libs/smart_ptr/smart_ptr.htm
-Valgrind: http://valgrind.org/"
-CWE-763,EN-Release of Invalid Pointer or Reference (Type: Base),"The application attempts to return a memory resource to the system, but calls the wrong release function or calls the appropriate release function incorrectly.
-This weakness can take several forms, such as:
-The memory was allocated, explicitly or implicitly, via one memory management method and deallocated using a different, non-compatible function (CWE-762).
-The function calls or memory management routines chosen are appropriate, however they are used incorrectly, such as in CWE-761.",,unclassified,"boost C++ Library Smart Pointers: http://www.boost.org/doc/libs/1_38_0/libs/smart_ptr/smart_ptr.htm
-Valgrind: http://valgrind.org/"
-CWE-764,EN-Multiple Locks of a Critical Resource (Type: Variant),"The software locks a critical resource more times than intended, leading to an unexpected state in the system.
-When software is operating in a concurrent environment and repeatedly locks a critical resource, the consequences will vary based on the type of lock, the lock's implementation, and the resource being protected. In some situations such as with semaphores, the resources are pooled and extra locking calls will reduce the size of the total available pool, possibly leading to degraded performance or a denial of service. If this can be triggered by an attacker, it will be similar to an unrestricted lock (CWE-412). In the context of a binary lock, it is likely that any duplicate locking attempts will never succeed since the lock is already held and progress may not be possible.",,unclassified,
-CWE-765,EN-Multiple Unlocks of a Critical Resource (Type: Variant),"The software unlocks a critical resource more times than intended, leading to an unexpected state in the system.
-When software is operating in a concurrent environment and repeatedly unlocks a critical resource, the consequences will vary based on the type of lock, the lock's implementation, and the resource being protected. In some situations such as with semaphores, the resources are pooled and extra calls to unlock will increase the count for the number of available resources, likely resulting in a crash or unpredictable behavior when the system nears capacity.",,unclassified,
-CWE-785,EN-Use of Path Manipulation Function without Maximum-sized Buffer (Type: Variant),"The software invokes a function for normalizing paths or file names, but it provides an output buffer that is smaller than the maximum possible size, such as PATH_MAX.
-Passing an inadequately-sized output buffer to a path manipulation function can result in a buffer overflow. Such functions include realpath(), readlink(), PathAppend(), and others.",,unclassified,
-CWE-786,EN-Access of Memory Location Before Start of Buffer (Type: Base),"The software reads or writes to a buffer using an index or pointer that references a memory location prior to the beginning of the buffer.
-This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used.",,unclassified,
-CWE-787,EN-Out-of-bounds Write (Type: Base),"The software writes data past the end, or before the beginning, of the intended buffer.
-This typically occurs when the pointer or its index is incremented or decremented to a position beyond the bounds of the buffer or when pointer arithmetic results in a position outside of the valid memory location to name a few. This may result in corruption of sensitive information, a crash, or code execution among other things.",,unclassified,
-CWE-788,EN-Access of Memory Location After End of Buffer (Type: Base),"The software reads or writes to a buffer using an index or pointer that references a memory location after the end of the buffer.
-This typically occurs when a pointer or its index is decremented to a position before the buffer, when pointer arithmetic results in a position before the beginning of the valid memory location, or when a negative index is used. These problems may be resultant from missing sentinel values (CWE-463) or trusting a user-influenced input length variable.",,unclassified,
-CWE-790,EN-Improper Filtering of Special Elements (Type: Class),"The software receives data from an upstream component, but does not filter or incorrectly filters special elements before sending it to a downstream component.
-Cross-site scripting (XSS) vulnerabilities occur when:
-1. Untrusted data enters a web application, typically from a web request.
-2. The web application dynamically generates a web page that contains this untrusted data.
-3. During page generation, the application does not prevent the data from containing content that is executable by a web browser, such as JavaScript, HTML tags, HTML attributes, mouse events, Flash, ActiveX, etc.
-4. A victim visits the generated web page through a web browser, which contains malicious script that was injected using the untrusted data.
-5. Since the script comes from a web page that was sent by the web server, the victim's web browser executes the malicious script in the context of the web server's domain.
-6. This effectively violates the intention of the web browser's same-origin policy, which states that scripts in one domain should not be able to access resources or run code in a different domain.
-There are three main kinds of XSS:
-The server reads data directly from the HTTP request and reflects it back in the HTTP response. Reflected XSS exploits occur when an attacker causes a victim to supply dangerous content to a vulnerable web application, which is then reflected back to the victim and executed by the web browser. The most common mechanism for delivering malicious content is to include it as a parameter in a URL that is posted publicly or e-mailed directly to the victim. URLs constructed in this manner constitute the core of many phishing schemes, whereby an attacker convinces a victim to visit a URL that refers to a vulnerable site. After the site reflects the attacker's content back to the victim, the content is executed by the victim's browser.
-The application stores dangerous data in a database, message forum, visitor log, or other trusted data store. At a later time, the dangerous data is subsequently read back into the application and included in dynamic content. From an attacker's perspective, the optimal place to inject malicious content is in an area that is displayed to either many users or particularly interesting users. Interesting users typically have elevated privileges in the application or interact with sensitive data that is valuable to the attacker. If one of these users executes malicious content, the attacker may be able to perform privileged operations on behalf of the user or gain access to sensitive data belonging to the user. For example, the attacker might inject XSS into a log message, which might not be handled properly when an administrator views the logs.
-In DOM-based XSS, the client performs the injection of XSS into the page; in the other types, the server performs the injection. DOM-based XSS generally involves server-controlled, trusted script that is sent to the client, such as Javascript that performs sanity checks on a form before the user submits it. If the server-supplied script processes user-supplied data and then injects it back into the web page (such as with dynamic HTML), then DOM-based XSS is possible.
-Once the malicious script is injected, the attacker can perform a variety of malicious activities. The attacker could transfer private information, such as cookies that may include session information, from the victim's machine to the attacker. The attacker could send malicious requests to a web site on behalf of the victim, which could be especially dangerous to the site if the victim has administrator privileges to manage that site. Phishing attacks could be used to emulate trusted web sites and trick the victim into entering a password, allowing the attacker to compromise the victim's account on that web site. Finally, the script could exploit a vulnerability in the web browser itself possibly taking over the victim's machine, sometimes referred to as ""drive-by hacking.""
-In many cases, the attack can be launched without the victim even being aware of it. Even with careful users, attackers frequently use a variety of methods to encode the malicious portion of the attack, such as URL encoding or Unicode, so the request looks less suspicious.",,unclassified,
-CWE-791,EN-Incomplete Filtering of Special Elements (Type: Base),"The software receives data from an upstream component, but does not completely filter special elements before sending it to a downstream component.
-Cross-site scripting (XSS) vulnerabilities occur when:
-1. Untrusted data enters a web application, typically from a web request.
-2. The web application dynamically generates a web page that contains this untrusted data.
-3. During page generation, the application does not prevent the data from containing content that is executable by a web browser, such as JavaScript, HTML tags, HTML attributes, mouse events, Flash, ActiveX, etc.
-4. A victim visits the generated web page through a web browser, which contains malicious script that was injected using the untrusted data.
-5. Since the script comes from a web page that was sent by the web server, the victim's web browser executes the malicious script in the context of the web server's domain.
-6. This effectively violates the intention of the web browser's same-origin policy, which states that scripts in one domain should not be able to access resources or run code in a different domain.
-There are three main kinds of XSS:
-The server reads data directly from the HTTP request and reflects it back in the HTTP response. Reflected XSS exploits occur when an attacker causes a victim to supply dangerous content to a vulnerable web application, which is then reflected back to the victim and executed by the web browser. The most common mechanism for delivering malicious content is to include it as a parameter in a URL that is posted publicly or e-mailed directly to the victim. URLs constructed in this manner constitute the core of many phishing schemes, whereby an attacker convinces a victim to visit a URL that refers to a vulnerable site. After the site reflects the attacker's content back to the victim, the content is executed by the victim's browser.
-The application stores dangerous data in a database, message forum, visitor log, or other trusted data store. At a later time, the dangerous data is subsequently read back into the application and included in dynamic content. From an attacker's perspective, the optimal place to inject malicious content is in an area that is displayed to either many users or particularly interesting users. Interesting users typically have elevated privileges in the application or interact with sensitive data that is valuable to the attacker. If one of these users executes malicious content, the attacker may be able to perform privileged operations on behalf of the user or gain access to sensitive data belonging to the user. For example, the attacker might inject XSS into a log message, which might not be handled properly when an administrator views the logs.
-In DOM-based XSS, the client performs the injection of XSS into the page; in the other types, the server performs the injection. DOM-based XSS generally involves server-controlled, trusted script that is sent to the client, such as Javascript that performs sanity checks on a form before the user submits it. If the server-supplied script processes user-supplied data and then injects it back into the web page (such as with dynamic HTML), then DOM-based XSS is possible.
-Once the malicious script is injected, the attacker can perform a variety of malicious activities. The attacker could transfer private information, such as cookies that may include session information, from the victim's machine to the attacker. The attacker could send malicious requests to a web site on behalf of the victim, which could be especially dangerous to the site if the victim has administrator privileges to manage that site. Phishing attacks could be used to emulate trusted web sites and trick the victim into entering a password, allowing the attacker to compromise the victim's account on that web site. Finally, the script could exploit a vulnerability in the web browser itself possibly taking over the victim's machine, sometimes referred to as ""drive-by hacking.""
-In many cases, the attack can be launched without the victim even being aware of it. Even with careful users, attackers frequently use a variety of methods to encode the malicious portion of the attack, such as URL encoding or Unicode, so the request looks less suspicious.",,unclassified,
-CWE-792,EN-Incomplete Filtering of One or More Instances of Special Elements (Type: Variant),"The software receives data from an upstream component, but does not completely filter one or more instances of special elements before sending it to a downstream component.
-Incomplete filtering of this nature involves either
-only filtering a single instance of a special element when more exist, or
-not filtering all instances or all elements where multiple special elements exist.",,unclassified,
-CWE-793,EN-Only Filtering One Instance of a Special Element (Type: Variant),"The software receives data from an upstream component, but only filters a single instance of a special element before sending it to a downstream component.
-Incomplete filtering of this nature may be location-dependent, as in only the first or last element is filtered.",,unclassified,
-CWE-794,EN-Incomplete Filtering of Multiple Instances of Special Elements (Type: Variant),"The software receives data from an upstream component, but does not filter all instances of a special element before sending it to a downstream component.
-Incomplete filtering of this nature may be applied to
-sequential elements (special elements that appear next to each other) or
-non-sequential elements (special elements that appear multiple times in different locations).",,unclassified,
-CWE-795,EN-Only Filtering Special Elements at a Specified Location (Type: Base),"The software receives data from an upstream component, but only accounts for special elements at a specified location, thereby missing remaining special elements that may exist before sending it to a downstream component.
-A filter might only account for instances of special elements when they occur:
-relative to a marker (e.g. ""at the beginning/end of string; the second argument""), or
-at an absolute position (e.g. ""byte number 10"").
-This may leave special elements in the data that did not match the filter position, but still may be dangerous.",,unclassified,
-CWE-796,EN-Only Filtering Special Elements Relative to a Marker (Type: Variant),"The software receives data from an upstream component, but only accounts for special elements positioned relative to a marker (e.g. ""at the beginning/end of a string; the second argument""), thereby missing remaining special elements that may exist before sending it to a downstream component.
-A filter might only account for instances of special elements when they occur:
-relative to a marker (e.g. ""at the beginning/end of string; the second argument""), or
-at an absolute position (e.g. ""byte number 10"").
-This may leave special elements in the data that did not match the filter position, but still may be dangerous.",,unclassified,
-CWE-797,EN-Only Filtering Special Elements at an Absolute Position (Type: Variant),"The software receives data from an upstream component, but only accounts for special elements at an absolute position (e.g. ""byte number 10""), thereby missing remaining special elements that may exist before sending it to a downstream component.
-A filter might only account for instances of special elements when they occur:
-relative to a marker (e.g. ""at the beginning/end of string; the second argument""), or
-at an absolute position (e.g. ""byte number 10"").
-This may leave special elements in the data that did not match the filter position, but still may be dangerous.",,unclassified,
-CWE-799,EN-Improper Control of Interaction Frequency (Type: Class),"The software does not properly limit the number or frequency of interactions that it has with an actor, such as the number of incoming requests.
-This can allow the actor to perform actions more frequently than expected. The actor could be a human or an automated process such as a virus or bot. This could be used to cause a denial of service, compromise program logic (such as limiting humans to a single vote), or other consequences. For example, an authentication routine might not limit the number of times an attacker can guess a password. Or, a web site might conduct a poll but only expect humans to vote a maximum of once a day.",,unclassified,Insufficient Anti-automation: http://projects.webappsec.org/Insufficient+Anti-automation
-CWE-8,EN-J2EE Misconfiguration: Entity Bean Declared Remote (Type: Variant),"When an application exposes a remote interface for an entity bean, it might also expose methods that get or set the bean's data. These methods could be leveraged to read sensitive information, or to change data in ways that violate the application's expectations, potentially leading to other vulnerabilities.
-This can allow the actor to perform actions more frequently than expected. The actor could be a human or an automated process such as a virus or bot. This could be used to cause a denial of service, compromise program logic (such as limiting humans to a single vote), or other consequences. For example, an authentication routine might not limit the number of times an attacker can guess a password. Or, a web site might conduct a poll but only expect humans to vote a maximum of once a day.",,unclassified,
-CWE-81,EN-Improper Neutralization of Script in an Error Message Web Page (Type: Variant),"The software receives input from an upstream component, but it does not neutralize or incorrectly neutralizes special characters that could be interpreted as web-scripting elements when they are sent to an error page.
-Error pages may include customized 403 Forbidden or 404 Not Found pages.
-When an attacker can trigger an error that contains unneutralized input, then cross-site scripting attacks may be possible.",,unclassified,"24 Deadly Sins of Software Security: ""Sin 11: Failure to Handle Errors Correctly."" Page 183"
-CWE-82,EN-Improper Neutralization of Script in Attributes of IMG Tags in a Web Page (Type: Variant),"The web application does not neutralize or incorrectly neutralizes scripting elements within attributes of HTML IMG tags, such as the src attribute.
-Attackers can embed XSS exploits into the values for IMG attributes (e.g. SRC) that is streamed and then executed in a victim's browser. Note that when the page is loaded into a user's browsers, the exploit will automatically execute.",,unclassified,
-CWE-820,EN-Missing Synchronization (Type: Base),"The software utilizes a shared resource in a concurrent manner but does not attempt to synchronize access to the resource.
-If access to a shared resource is not synchronized, then the resource may not be in a state that is expected by the software. This might lead to unexpected or insecure behaviors, especially if an attacker can influence the shared resource.",,unclassified,
-CWE-821,EN-Incorrect Synchronization (Type: Base),"The software utilizes a shared resource in a concurrent manner but it does not correctly synchronize access to the resource.
-If access to a shared resource is not correctly synchronized, then the resource may not be in a state that is expected by the software. This might lead to unexpected or insecure behaviors, especially if an attacker can influence the shared resource.",,unclassified,
-CWE-822,EN-Untrusted Pointer Dereference (Type: Base),"The program obtains a value from an untrusted source, converts this value to a pointer, and dereferences the resulting pointer.
-An attacker can supply a pointer for memory locations that the program is not expecting. If the pointer is dereferenced for a write operation, the attack might allow modification of critical program state variables, cause a crash, or execute code. If the dereferencing operation is for a read, then the attack might allow reading of sensitive data, cause a crash, or set a program variable to an unexpected value (since the value will be read from an unexpected memory location).
-There are several variants of this weakness, including but not necessarily limited to:
-The untrusted value is directly invoked as a function call.
-In OS kernels or drivers where there is a boundary between ""userland"" and privileged memory spaces, an untrusted pointer might enter through an API or system call (see CWE-781 for one such example).
-Inadvertently accepting the value from an untrusted control sphere when it did not have to be accepted as input at all. This might occur when the code was originally developed to be run by a single user in a non-networked environment, and the code is then ported to or otherwise exposed to a networked environment.",,unclassified,
-CWE-823,EN-Use of Out-of-range Pointer Offset (Type: Base),"The program performs pointer arithmetic on a valid pointer, but it uses an offset that can point outside of the intended range of valid memory locations for the resulting pointer.
-While a pointer can contain a reference to any arbitrary memory location, a program typically only intends to use the pointer to access limited portions of memory, such as contiguous memory used to access an individual array.
-Programs may use offsets in order to access fields or sub-elements stored within structured data. The offset might be out-of-range if it comes from an untrusted source, is the result of an incorrect calculation, or occurs because of another error.
-If an attacker can control or influence the offset so that it points outside of the intended boundaries of the structure, then the attacker may be able to read or write to memory locations that are used elsewhere in the program. As a result, the attack might change the state of the software as accessed through program variables, cause a crash or instable behavior, and possibly lead to code execution.",,unclassified,"The Art of Software Security Assessment: Chapter 6, ""Pointer Arithmetic"", Page 277."
-CWE-824,EN-Access of Uninitialized Pointer (Type: Base),"The program accesses or uses a pointer that has not been initialized.
-If the pointer contains an uninitialized value, then the value might not point to a valid memory location. This could cause the program to read from or write to unexpected memory locations, leading to a denial of service. If the uninitialized pointer is used as a function call, then arbitrary functions could be invoked. If an attacker can influence the portion of uninitialized memory that is contained in the pointer, this weakness could be leveraged to execute code or perform other attacks.
-Depending on memory layout, associated memory management behaviors, and program operation, the attacker might be able to influence the contents of the uninitialized pointer, thus gaining more fine-grained control of the memory location to be accessed.",,unclassified,"The Art of Software Security Assessment: Chapter 7, ""Variable Initialization"", Page 312."
-CWE-825,EN-Expired Pointer Dereference (Type: Base),"The program dereferences a pointer that contains a location for memory that was previously valid, but is no longer valid.
-When a program releases memory, but it maintains a pointer to that memory, then the memory might be re-allocated at a later time. If the original pointer is accessed to read or write data, then this could cause the program to read or modify data that is in use by a different function or process. Depending on how the newly-allocated memory is used, this could lead to a denial of service, information exposure, or code execution.",,unclassified,
-CWE-826,EN-Premature Release of Resource During Expected Lifetime (Type: Base),"The program releases a resource that is still intended to be used by the program itself or another actor.
-This weakness focuses on errors in which the program should not release a resource, but performs the release anyway. This is different than a weakness in which the program releases a resource at the appropriate time, but it maintains a reference to the resource, which it later accesses. For this weaknesses, the resource should still be valid upon the subsequent access.
-When a program releases a resource that is still being used, it is possible that operations will still be taken on this resource, which may have been repurposed in the meantime, leading to issues similar to CWE-825. Consequences may include denial of service, information exposure, or code execution.",,unclassified,
-CWE-827,EN-Improper Control of Document Type Definition (Type: Base),"The software does not restrict a reference to a Document Type Definition (DTD) to the intended control sphere. This might allow attackers to reference arbitrary DTDs, possibly causing the software to expose files, consume excessive system resources, or execute arbitrary http requests on behalf of the attacker.
-As DTDs are processed, they might try to read or include files on the machine performing the parsing. If an attacker is able to control the DTD, then the attacker might be able to specify sensitive resources or requests or provide malicious content.
-For example, the SOAP specification prohibits SOAP messages from containing DTDs.",,unclassified,Apache CXF Security Advisory (CVE-2010-2076): http://svn.apache.org/repos/asf/cxf/trunk/security/CVE-2010-2076.pdf
-CWE-828,EN-Signal Handler with Functionality that is not Asynchronous-Safe (Type: Base),"The software defines a signal handler that contains code sequences that are not asynchronous-safe, i.e., the functionality is not reentrant, or it can be interrupted.
-This can lead to an unexpected system state with a variety of potential consequences depending on context, including denial of service and code execution.
-Signal handlers are typically intended to interrupt normal functionality of a program, or even other signals, in order to notify the process of an event. When a signal handler uses global or static variables, or invokes functions that ultimately depend on such state or its associated metadata, then it could corrupt system state that is being used by normal functionality. This could subject the program to race conditions or other weaknesses that allow an attacker to cause the program state to be corrupted. While denial of service is frequently the consequence, in some cases this weakness could be leveraged for code execution.
-There are several different scenarios that introduce this issue:
-Invocation of non-reentrant functions from within the handler. One example is malloc(), which modifies internal global variables as it manages memory. Very few functions are actually reentrant.
-Code sequences (not necessarily function calls) contain non-atomic use of global variables, or associated metadata or structures, that can be accessed by other functionality of the program, including other signal handlers. Frequently, the same function is registered to handle multiple signals.
-The signal handler function is intended to run at most one time, but instead it can be invoked multiple times. This could happen by repeated delivery of the same signal, or by delivery of different signals that have the same handler function (CWE-831).
-Note that in some environments or contexts, it might be possible for the signal handler to be interrupted itself.
-If both a signal handler and the normal behavior of the software have to operate on the same set of state variables, and a signal is received in the middle of the normal execution's modifications of those variables, the variables may be in an incorrect or corrupt state during signal handler execution, and possibly still incorrect or corrupt upon return.",,unclassified,"Delivering Signals for Fun and Profit: http://lcamtuf.coredump.cx/signals.txt
-Race Condition: Signal Handling: http://www.fortify.com/vulncat/en/vulncat/cpp/race_condition_signal_handling.html"
-CWE-829,EN-Inclusion of Functionality from Untrusted Control Sphere (Type: Class),"The software imports, requires, or includes executable functionality (such as a library) from a source that is outside of the intended control sphere.
-When including third-party functionality, such as a web widget, library, or other source of functionality, the software must effectively trust that functionality. Without sufficient protection mechanisms, the functionality could be malicious in nature (either by coming from an untrusted source, being spoofed, or being modified in transit from a trusted source). The functionality might also contain its own weaknesses, or grant access to additional functionality and state information that should be kept private to the base system, such as system state information, sensitive application data, or the DOM of a web application.
-This might lead to many different consequences depending on the included functionality, but some examples include injection of malware, information exposure by granting excessive privileges or permissions to the untrusted functionality, DOM-based XSS vulnerabilities, stealing user's cookies, or open redirect to malware (CWE-601).",,unclassified,"OWASP Enterprise Security API (ESAPI) Project: http://www.owasp.org/index.php/ESAPI
-Least Privilege: https://buildsecurityin.us-cert.gov/daisy/bsi/articles/knowledge/principles/351.html"
-CWE-83,EN-Improper Neutralization of Script in Attributes in a Web Page (Type: Variant),"The software does not neutralize or incorrectly neutralizes ""javascript:"" or other URIs from dangerous attributes within tags, such as onmouseover, onload, onerror, or style.
-When including third-party functionality, such as a web widget, library, or other source of functionality, the software must effectively trust that functionality. Without sufficient protection mechanisms, the functionality could be malicious in nature (either by coming from an untrusted source, being spoofed, or being modified in transit from a trusted source). The functionality might also contain its own weaknesses, or grant access to additional functionality and state information that should be kept private to the base system, such as system state information, sensitive application data, or the DOM of a web application.
-This might lead to many different consequences depending on the included functionality, but some examples include injection of malware, information exposure by granting excessive privileges or permissions to the untrusted functionality, DOM-based XSS vulnerabilities, stealing user's cookies, or open redirect to malware (CWE-601).",,unclassified,
-CWE-830,EN-Inclusion of Web Functionality from an Untrusted Source (Type: Base),"The software includes web functionality (such as a web widget) from another domain, which causes it to operate within the domain of the software, potentially granting total access and control of the software to the untrusted source.
-Including third party functionality in a web-based environment is risky, especially if the source of the functionality is untrusted.
-Even if the third party is a trusted source, the software may still be exposed to attacks and malicious behavior if that trusted source is compromised, or if the code is modified in transmission from the third party to the software.
-This weakness is common in ""mashup"" development on the web, which may include source functionality from other domains. For example, Javascript-based web widgets may be inserted by using '