Merge branch 'white/staging' into white/master

.pylintrc

@@ -282,7 +282,8 @@ disable=blacklisted-name,
         comparison-with-callable,
         unused-variable,
         method-cache-max-size-none,
-        consider-using-with
+        consider-using-with,
+        global-statement
 
 
 

AUTHORS

@@ -2,6 +2,7 @@ The PRIMARY AUTHORS are:
 
     * Ciro Goyeneche
     * Daniel Foguelman
+    * David Kraus
     * Diego Nadares
     * Eric Horvat
     * Esteban Guillardoy
@@ -11,44 +12,43 @@ The PRIMARY AUTHORS are:
     * Francisco Amato
     * Franco Linares
     * German Riera
+    * Gonzalo Martínez
+    * Ignacio Feijoo
     * Javier Montilva
     * Joaquín López Pereyra
     * Jorge Luis González Iznaga
     * Leonardo Lazzaro
     * Marcelo Pedraza
-    * Manuel Jose Sotomayor Torrealba
+    * Manuel José Sotomayor Torrealba
     * Mariano Marchetta
     * Martín Rocha
-    * Matias Ariel Ré Medina
-    * Matias Lang
+    * Matías Ariel Ré Medina
+    * Matías Lang
     * Micaela Ranea Sánchez
     * Nahuel Alonso
-    * Nicolas Rebagliati
-    * Sebastian Brachi
-    * Sebastian Kulesz
+    * Nicolás Rebagliati
+    * Sebastián Brachi
+    * Sebastián Kulesz
     * Zoe Marino
-    * Gonzalo Martinez
-    * Ignacio Feijo
-    * David Kraus
 
 Project contributors
 
     * Alejandro Parodi
     * Andrés López Luksenberg
-    * Andres Tarantini
+    * Andrés Tarantini
     * Brice Samulenok
     * Buanzo
     * csk
     * dmknght
     * Elian Gidoni
     * Endrigo Antonini
-    * Federico Fernandez
+    * Federico Fernández
     * James Jara
-    * Javier aguinaga
+    * Javier Aguinaga
     * Juan Urbano
     * Korantin Auguste
     * logdot
-    * Martin Tartarelli
+    * Martín Tartarelli
     * Mike Zhong (go bears)
     * Necrose99
     * Roberto Focke

CHANGELOG/5.1.0/community.md

@@ -0,0 +1,5 @@
+ * [MOD] Modify analytics type enum. #7615
+ * [ADD] Performance improved in `assets` views making several vulnerabilities stats statics in asset's model. #7634
+ * [FIX] Fix references. #7648
+ * [ADD] Now `custom fields` are available for filtering vulnerabilities. Also add `date` type for custom fields. #7625
+ * [MOD] We changed the order in which we set the path constant of `faraday_home` in order to fix a bug with faraday_manage when is installed by deb/rpm. #7653

CHANGELOG/5.1.0/date.md

@@ -0,0 +1 @@
+Feb 8th, 2024

RELEASE.md

@@ -1,6 +1,14 @@
 New features in the latest update
 =====================================
 
+5.1.0 [Feb 8th, 2024]:
+---
+ * [ADD] Performance improved in `assets` views making several vulnerabilities stats statics in asset's model. #7634
+ * [ADD] Now `custom fields` are available for filtering vulnerabilities. Also add `date` type for custom fields. #7625
+ * [MOD] We changed the order in which we set the path constant of `faraday_home` in order to fix a bug with faraday_manage when is installed by deb/rpm. #7653
+ * [MOD] Modify analytics type enum. #7615
+ * [FIX] Fix references. #7648
+
 5.0.1 [Jan 2nd, 2024]:
 ---
  * [MOD] Code refactor.

docker/server.ini

@@ -6,8 +6,8 @@ api_token_expiration = 604800
 session_timeout = 24
 delete_report_after_process = true
 #celery_enabled = false
-#celery_broker_url = localhost
-#celery_backend_url = localhost
+celery_broker_url = redis
+celery_backend_url = redis
 #redis_session_storage = XXX
 
 [storage]

faraday/__init__.py

@@ -4,5 +4,5 @@
 See the file 'doc/LICENSE' for the license information
 """
 
-__version__ = '5.0.1'
+__version__ = '5.1.0'
 __license_version__ = __version__

faraday/manage.py

@@ -21,7 +21,8 @@
 from sqlalchemy.exc import ProgrammingError, OperationalError
 
 import faraday.server.config
-from faraday.server.app import get_app
+from faraday.server.app import get_app, create_app
+from faraday.server.commands.sync_hosts_stats import _sync_hosts_stats
 from faraday.server.config import FARADAY_BASE
 from faraday.server.commands.initdb import InitDB
 from faraday.server.commands.faraday_schema_display import DatabaseSchema
@@ -304,6 +305,13 @@ def move_references(all_workspaces, workspace_name):
         _move_references(all_workspaces=all_workspaces, workspace_name=workspace_name)
 
 
+@click.command(help="Synchronize vulnerability severity stats in asset")
+def sync_hosts_stats():
+    app = create_app()
+    with app.app_context():
+        _sync_hosts_stats()
+
+
 cli.add_command(show_urls)
 cli.add_command(initdb)
 cli.add_command(database_schema)
@@ -321,6 +329,7 @@ def move_references(all_workspaces, workspace_name):
 cli.add_command(import_vulnerability_templates)
 cli.add_command(settings)
 cli.add_command(move_references)
+cli.add_command(sync_hosts_stats)
 
 
 if __name__ == '__main__':

faraday/migrations/versions/257f6d0ad43f_add_fields_to_kb.py

@@ -0,0 +1,31 @@
+"""add fields to KB
+
+Revision ID: 257f6d0ad43f
+Revises: b87b1de2f348
+Create Date: 2023-11-10 21:47:36.973846+00:00
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '257f6d0ad43f'
+down_revision = 'b87b1de2f348'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('vulnerability_template',
+                  sa.Column('cve', sa.Text(), default='', server_default='', nullable=True))
+    op.add_column('vulnerability_template',
+                  sa.Column('_cvss2_vector_string', sa.Text(), default='', server_default='', nullable=True))
+    op.add_column('vulnerability_template',
+                  sa.Column('_cvss3_vector_string', sa.Text(), default='', server_default='', nullable=True))
+
+
+def downgrade():
+    op.drop_column('vulnerability_template', 'cve')
+    op.drop_column('vulnerability_template', '_cvss2_vector_string')
+    op.drop_column('vulnerability_template', '_cvss3_vector_string')

faraday/migrations/versions/51e533d41312_add_host_stats_static_columns.py

@@ -0,0 +1,37 @@
+"""Add host stats static columns
+
+Revision ID: 51e533d41312
+Revises: 257f6d0ad43f
+Create Date: 2024-01-12 20:12:43.408035+00:00
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '51e533d41312'
+down_revision = '257f6d0ad43f'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('host', sa.Column('vulnerability_critical_generic_count', sa.Integer(), server_default=sa.text('0'), nullable=True))
+    op.add_column('host', sa.Column('vulnerability_high_generic_count', sa.Integer(), server_default=sa.text('0'), nullable=True))
+    op.add_column('host', sa.Column('vulnerability_medium_generic_count', sa.Integer(), server_default=sa.text('0'), nullable=True))
+    op.add_column('host', sa.Column('vulnerability_low_generic_count', sa.Integer(), server_default=sa.text('0'), nullable=True))
+    op.add_column('host', sa.Column('vulnerability_info_generic_count', sa.Integer(), server_default=sa.text('0'), nullable=True))
+    op.add_column('host', sa.Column('vulnerability_unclassified_generic_count', sa.Integer(), server_default=sa.text('0'), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('host', 'vulnerability_unclassified_generic_count')
+    op.drop_column('host', 'vulnerability_info_generic_count')
+    op.drop_column('host', 'vulnerability_low_generic_count')
+    op.drop_column('host', 'vulnerability_medium_generic_count')
+    op.drop_column('host', 'vulnerability_high_generic_count')
+    op.drop_column('host', 'vulnerability_critical_generic_count')
+    # ### end Alembic commands ###

faraday/migrations/versions/b87b1de2f348_add_delete_workspace_missing_cascades.py

@@ -0,0 +1,87 @@
+"""add delete workspace missing cascades
+
+Revision ID: b87b1de2f348
+Revises: d0a6105fdef1
+Create Date: 2023-10-18 19:30:09.640602+00:00
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = 'b87b1de2f348'
+down_revision = 'd0a6105fdef1'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute('alter table workspace_permission_association drop constraint workspace_permission_association_workspace_id_fkey;')
+    op.execute('alter table workspace_permission_association add constraint workspace_permission_association_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE;')
+    op.execute('alter table severities_histogram drop constraint severities_histogram_workspace_id_fkey;')
+    op.execute('alter table severities_histogram add constraint severities_histogram_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE;')
+    op.execute('alter table reference drop constraint reference_workspace_id_fkey;')
+    op.execute('alter table reference add constraint reference_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE;')
+    op.execute('alter table policy_violation drop constraint policy_violation_workspace_id_fkey;')
+    op.execute('alter table policy_violation add constraint policy_violation_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE;')
+    op.execute('alter table command_object drop constraint command_object_workspace_id_fkey;')
+    op.execute('alter table command_object add constraint command_object_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE;')
+
+    op.execute('alter table vulnerability_hit_count drop constraint vulnerability_hit_count_workspace_id_fkey;')
+    op.execute('alter table vulnerability_hit_count add constraint vulnerability_hit_count_workspace_id_fkey  FOREIGN KEY (workspace_id) REFERENCES workspace(id) on delete CASCADE;')
+    op.execute('alter table reference_vulnerability_association drop constraint reference_vulnerability_association_reference_id_fkey;')
+    op.execute('alter table reference_vulnerability_association add constraint reference_vulnerability_association_reference_id_fkey  FOREIGN KEY (reference_id) REFERENCES reference(id) on delete CASCADE;')
+    op.execute('alter table websocket_notification drop constraint websocket_notification_id_fkey;')
+    op.execute('alter table websocket_notification add constraint websocket_notification_id_fkey  FOREIGN KEY (id) REFERENCES notification_base(id) on delete CASCADE;')
+    op.execute('alter table notification_base drop constraint notification_base_notification_event_id_fkey;')
+    op.execute('alter table notification_base add constraint notification_base_notification_event_id_fkey  FOREIGN KEY (notification_event_id) REFERENCES notification_event(id) on delete CASCADE;')
+    op.execute('alter table notification_event drop constraint notification_event_workspace_id_fkey;')
+    op.execute('alter table notification_event add constraint notification_event_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id) on delete CASCADE;')
+    op.execute('alter table comment drop constraint comment_workspace_id_fkey;')
+    op.execute('alter table comment add constraint comment_workspace_id_fkey  FOREIGN KEY (workspace_id) REFERENCES workspace(id) on delete CASCADE;')
+    op.execute('alter table scope drop constraint scope_workspace_id_fkey;')
+    op.execute('alter table scope add constraint scope_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id) on delete CASCADE;')
+
+    op.execute('alter table executive_report drop constraint executive_report_workspace_id_fkey;')
+    op.execute('alter table executive_report add constraint executive_report_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id) on delete CASCADE;')
+    op.execute('alter table pipeline drop constraint pipeline_workspace_id_fkey;')
+    op.execute('alter table pipeline add constraint pipeline_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id) on delete SET NULL;')
+    op.execute('alter table agents_schedule_workspace_table drop constraint agents_schedule_workspace_table_workspace_id_fkey;')
+    op.execute('alter table agents_schedule_workspace_table add constraint agents_schedule_workspace_table_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id) on delete CASCADE;')
+
+
+# workspace_permission_association,  policy_violation,   websocket_notification
+
+
+def downgrade():
+    # perform downgrade of the previews actions
+    op.execute('alter table workspace_permission_association drop constraint workspace_permission_association_workspace_id_fkey;')
+    op.execute('alter table workspace_permission_association add constraint workspace_permission_association_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id);')
+    op.execute('alter table severities_histogram drop constraint severities_histogram_workspace_id_fkey;')
+    op.execute('alter table severities_histogram add constraint severities_histogram_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id);')
+    op.execute('alter table reference drop constraint reference_workspace_id_fkey;')
+    op.execute('alter table reference add constraint reference_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id);')
+    op.execute('alter table policy_violation drop constraint policy_violation_workspace_id_fkey;')
+    op.execute('alter table policy_violation add constraint policy_violation_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id);')
+    op.execute('alter table command_object drop constraint command_object_workspace_id_fkey;')
+    op.execute('alter table command_object add constraint command_object_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id);')
+    op.execute('alter table vulnerability_hit_count drop constraint vulnerability_hit_count_workspace_id_fkey;')
+    op.execute('alter table vulnerability_hit_count add constraint vulnerability_hit_count_workspace_id_fkey  FOREIGN KEY (workspace_id) REFERENCES workspace(id);')
+    op.execute('alter table reference_vulnerability_association drop constraint reference_vulnerability_association_reference_id_fkey;')
+    op.execute('alter table reference_vulnerability_association add constraint reference_vulnerability_association_reference_id_fkey  FOREIGN KEY (reference_id) REFERENCES reference(id);')
+    op.execute('alter table websocket_notification drop constraint websocket_notification_id_fkey;')
+    op.execute('alter table websocket_notification add constraint websocket_notification_id_fkey  FOREIGN KEY (id) REFERENCES notification_base(id);')
+    op.execute('alter table notification_base drop constraint notification_base_notification_event_id_fkey;')
+    op.execute('alter table notification_base add constraint notification_base_notification_event_id_fkey  FOREIGN KEY (notification_event_id) REFERENCES notification_event(id);')
+    op.execute('alter table notification_event drop constraint notification_event_workspace_id_fkey;')
+    op.execute('alter table notification_event add constraint notification_event_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id);')
+    op.execute('alter table comment drop constraint comment_workspace_id_fkey;')
+    op.execute('alter table comment add constraint comment_workspace_id_fkey  FOREIGN KEY (workspace_id) REFERENCES workspace(id);')
+    op.execute('alter table scope drop constraint scope_workspace_id_fkey;')
+    op.execute('alter table scope add constraint scope_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id);')
+    op.execute('alter table executive_report drop constraint executive_report_workspace_id_fkey;')
+    op.execute('alter table executive_report add constraint executive_report_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id);')
+    op.execute('alter table pipeline drop constraint pipeline_workspace_id_fkey;')
+    op.execute('alter table pipeline add constraint pipeline_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id);')
+    op.execute('alter table agents_schedule_workspace_table drop constraint agents_schedule_workspace_table_workspace_id_fkey;')
+    op.execute('alter table agents_schedule_workspace_table add constraint agents_schedule_workspace_table_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspace(id);')

faraday/migrations/versions/d0a6105fdef1_modify_analytics_type_enum.py

@@ -0,0 +1,26 @@
+"""modify analytics type enum
+
+Revision ID: d0a6105fdef1
+Revises: 443a136bb5f2
+Create Date: 2023-11-14 20:48:56.645881+00:00
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = 'd0a6105fdef1'
+down_revision = '443a136bb5f2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # add entry "vulnerabilities_by_risk_score" to enum "analytics_types"
+    with op.get_context().autocommit_block():
+        op.execute("ALTER TYPE analytics_types ADD VALUE IF NOT EXISTS 'vulnerabilities_by_risk_score'")
+
+
+def downgrade():
+    # CANT REMOVE ENUM VALUES, NOT SUPPORTED BY POSTGRES
+    pass