Merge branch 'main' into fix-nvt-indexes

CMakeLists.txt

@@ -20,7 +20,7 @@ cmake_minimum_required (VERSION 3.0)
 message ("-- Configuring Greenbone Vulnerability Manager...")
 
 project (gvm
-         VERSION 23.7.1
+         VERSION 23.8.2
          LANGUAGES C)
 
 if (POLICY CMP0005)

doc/gvmd.8

@@ -121,9 +121,15 @@ Maximum size of user-defined message text in alert emails, in bytes.
 \fB--max-ips-per-target=\fINUMBER\fB\f1
 Maximum number of IPs per target.
 .TP
+\fB--mem-wait-retries=\fINUMBER\fB\f1
+How often to try waiting for available memory. Default: 30. Each retry will wait for 10 seconds. 
+.TP
 \fB-m, --migrate\f1
 Migrate the database and exit.
 .TP
+\fB--min-mem-feed-update=\fINUMBER\fB\f1
+Minimum memory in MiB for feed updates. Default: 0. Feed updates are skipped if less physical memory is available. 
+.TP
 \fB--modify-scanner=\fISCANNER-UUID\fB\f1
 Modify scanner SCANNER-UUID and exit.
 .TP

doc/gvmd.8.xml

@@ -287,12 +287,30 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
         <p>Maximum number of IPs per target.</p>
       </optdesc>
     </option>
+    <option>
+      <p><opt>--mem-wait-retries=<arg>NUMBER</arg></opt></p>
+      <optdesc>
+        <p>
+          How often to try waiting for available memory. Default: 30.
+          Each retry will wait for 10 seconds.
+        </p>
+      </optdesc>
+    </option>
     <option>
       <p><opt>-m, --migrate</opt></p>
       <optdesc>
         <p>Migrate the database and exit.</p>
       </optdesc>
     </option>
+    <option>
+      <p><opt>--min-mem-feed-update=<arg>NUMBER</arg></opt></p>
+      <optdesc>
+        <p>
+          Minimum memory in MiB for feed updates. Default: 0.
+          Feed updates are skipped if less physical memory is available.
+        </p>
+      </optdesc>
+    </option>
     <option>
       <p><opt>--modify-scanner=<arg>SCANNER-UUID</arg></opt></p>
       <optdesc>

doc/gvmd.html

@@ -242,12 +242,30 @@ <h2>Options</h2>
 
 
 
+      <p><b>--mem-wait-retries=<em>NUMBER</em></b></p>
+
+        <p>
+          How often to try waiting for available memory. Default: 30.
+           Each retry will wait for 10 seconds.
+        </p>
+
+
+
       <p><b>-m, --migrate</b></p>
 
         <p>Migrate the database and exit.</p>
 
 
 
+      <p><b>--min-mem-feed-update=<em>NUMBER</em></b></p>
+
+        <p>
+          Minimum memory in MiB for feed updates. Default: 0.
+          Feed updates are skipped if less physical memory is available.
+        </p>
+
+
+
       <p><b>--modify-scanner=<em>SCANNER-UUID</em></b></p>
 
         <p>Modify scanner SCANNER-UUID and exit.</p>

src/gvmd.c

@@ -1895,6 +1895,8 @@ gvmd (int argc, char** argv, char *env[])
   static gchar *broker_address = NULL;
   static gchar *feed_lock_path = NULL;
   static int feed_lock_timeout = 0;
+  static int mem_wait_retries = 30;
+  static int min_mem_feed_update = 0;
   static int vt_ref_insert_size = VT_REF_INSERT_SIZE_DEFAULT;
   static int vt_sev_insert_size = VT_SEV_INSERT_SIZE_DEFAULT;
   static gchar *vt_verification_collation = NULL;
@@ -2088,10 +2090,20 @@ gvmd (int argc, char** argv, char *env[])
           &max_ips_per_target,
           "Maximum number of IPs per target.",
           "<number>" },
+        { "mem-wait-retries", '\0', 0, G_OPTION_ARG_INT,
+          &mem_wait_retries,
+          "How often to try waiting for available memory. Default: 30."
+          " Each retry will wait for 10 seconds.",
+          "<number>" },
         { "migrate", 'm', 0, G_OPTION_ARG_NONE,
           &migrate_database,
           "Migrate the database and exit.",
           NULL },
+        { "min-mem-feed-update", '\0', 0, G_OPTION_ARG_INT,
+          &min_mem_feed_update,
+          "Minimum memory in MiB for feed updates. Default: 0."
+          " Feed updates are skipped if less physical memory is available.",
+          "<number>" },
         { "modify-scanner", '\0', 0, G_OPTION_ARG_STRING,
           &modify_scanner,
           "Modify scanner <scanner-uuid> and exit.",
@@ -2447,6 +2459,12 @@ gvmd (int argc, char** argv, char *env[])
       }
   }
 
+  /* Set number of retries waiting for memory */
+  set_mem_wait_retries (mem_wait_retries);
+
+  /* Set minimum memory for feed updates */
+  set_min_mem_feed_update (min_mem_feed_update);
+
   /* Set relay mapper */
   if (relay_mapper)
     {

src/manage.c

@@ -184,6 +184,16 @@ static gchar *feed_lock_path = NULL;
  */
 static int feed_lock_timeout = 0;
 
+/**
+ * @brief Retries for waiting for memory to be available.
+ */
+static int mem_wait_retries = 0;
+
+/**
+ * @brief Minimum available memory in MiB for running a feed update.
+ */
+static int min_mem_feed_update = 0;
+
 /**
  * @brief Path to the relay mapper executable, NULL to disable relays.
  */
@@ -5050,7 +5060,52 @@ feed_sync_required ()
   return FALSE;
 }
 
+/**
+ * @brief Wait for memory
+ * 
+ * @param[in]  check_func  Function to check memory, should return 1 if enough.
+ * @param[in]  retries     Number of retries.
+ * @param[in]  min_mem     Minimum memory in MiB, for logging only
+ * @param[in]  action      Short descriptor of action waiting for memory.
+ * 
+ * @return 0 if enough memory is available, 1 gave up
+ */
+static int
+wait_for_mem (int check_func(),
+              int retries,
+              int min_mem,
+              const char *action)
+{
+  int retry_number = 0;
+  while (check_func () == 0)
+    {
+      if (retry_number == 0)
+        {
+          g_info ("%s: not enough memory for %s"
+                  " (%lld / %d) MiB",
+                  __func__,
+                  action,
+                  phys_mem_available () / 1048576llu,
+                  min_mem);
+        }
+      else
+        {
+          g_debug ("%s: waiting for memory for %s"
+                   " (%lld / %d) MiB",
+                   __func__,
+                   action,
+                   phys_mem_available () / 1048576llu,
+                   min_mem);
+        }
+
+      retry_number ++;
+      if (retry_number > retries)
+        return 1;
 
+      sleep (SCHEDULE_PERIOD);
+    }
+  return 0;
+}
 
 /**
  * @brief Perform any syncing that is due.
@@ -5074,7 +5129,11 @@ manage_sync (sigset_t *sigmask_current,
 
   if (feed_sync_required ())
     {
-      if (feed_lockfile_lock (&lockfile) == 0)
+      if (wait_for_mem (check_min_mem_feed_update,
+                        mem_wait_retries,
+                        min_mem_feed_update,
+                        "SecInfo feed sync") == 0
+          && feed_lockfile_lock (&lockfile) == 0)
         {
           pid_t nvts_pid, scap_pid, cert_pid;
           nvts_pid = manage_sync_nvts (fork_update_nvt_cache);
@@ -5096,7 +5155,11 @@ manage_sync (sigset_t *sigmask_current,
           || should_sync_port_lists ()
           || should_sync_report_formats ()))
     {
-      if (feed_lockfile_lock (&lockfile) == 0)
+      if (wait_for_mem (check_min_mem_feed_update,
+                        mem_wait_retries,
+                        min_mem_feed_update,
+                        "data objects feed sync") == 0
+          && feed_lockfile_lock (&lockfile) == 0)
         {
           manage_sync_configs ();
           manage_sync_port_lists ();
@@ -6352,6 +6415,82 @@ set_feed_lock_timeout (int new_timeout)
     feed_lock_timeout = new_timeout;
 }
 
+/**
+ * @brief Get the number of retries when waiting for memory to be available.
+ *
+ * @return The current number of retries.
+ */
+int
+get_mem_wait_retries()
+{
+  return mem_wait_retries;
+}
+
+/**
+ * @brief Set the number of retries when waiting for memory to be available.
+ *
+ * @param[in]  new_retries The new number of retries.
+ */
+void
+set_mem_wait_retries (int new_retries)
+{
+  if (new_retries < 0)
+    min_mem_feed_update = 0;
+  else
+    min_mem_feed_update = new_retries;
+}
+
+/**
+ * @brief Check if the minimum memory for feed updates is available
+ * 
+ * @return 1 if minimum memory amount is available, 0 if not
+ */
+int
+check_min_mem_feed_update ()
+{
+  if (min_mem_feed_update)
+    {
+      guint64 min_mem_bytes = (guint64)min_mem_feed_update * 1048576llu;
+      return phys_mem_available () >= min_mem_bytes ? 1 : 0;
+    }
+  return 1;
+}
+
+/**
+ * @brief Get the minimum memory for feed updates.
+ *
+ * @return The current minimum memory for feed updates in MiB.
+ */
+int
+get_min_mem_feed_update ()
+{
+  return min_mem_feed_update;
+}
+
+/**
+ * @brief Get the minimum memory for feed updates.
+ *
+ * @param[in]  new_min_mem The new minimum memory for feed updates in MiB.
+ */
+void
+set_min_mem_feed_update (int new_min_mem)
+{
+  guint64 min_mem_bytes = (guint64)new_min_mem * 1048576llu;
+  if (new_min_mem < 0)
+    min_mem_feed_update = 0;
+  else if (min_mem_bytes > phys_mem_total ())
+    {
+      g_warning ("%s: requested feed minimum memory limit (%d MiB)"
+                 " exceeds total physical memory (%lld MiB)."
+                 " The setting is ignored.",
+                 __func__,
+                 new_min_mem,
+                 phys_mem_total () / 1048576llu);
+    }
+  else
+    min_mem_feed_update = new_min_mem;
+}
+
 /**
  * @brief Write start time to sync lock file.
  *

src/manage.h

@@ -3870,6 +3870,21 @@ get_feed_lock_timeout ();
 void
 set_feed_lock_timeout (int);
 
+int
+get_mem_wait_retries ();
+
+void
+set_mem_wait_retries (int);
+
+int
+check_min_mem_feed_update ();
+
+int
+get_min_mem_feed_update ();
+
+void
+set_min_mem_feed_update (int);
+
 void
 write_sync_start (int);
 

src/manage_pg.c

@@ -1849,9 +1849,11 @@ create_view_result_vt_epss ()
          "    epss_score,"
          "    epss_percentile,"
          "    epss_cve,"
+         "    epss_severity,"
          "    max_epss_score,"
          "    max_epss_percentile,"
-         "    max_epss_cve"
+         "    max_epss_cve,"
+         "    max_epss_severity"
          "  FROM nvts);");
 
   sql ("SELECT create_index ('result_vt_epss_by_vt_id',"

src/manage_sql_nvts.c

@@ -1979,6 +1979,7 @@ update_nvts_from_vts (element_t *get_vts_response,
   if (rebuild) {
     sql ("DROP VIEW IF EXISTS results_autofp;");
     sql ("DROP VIEW vulns;");
+    sql ("DROP MATERIALIZED VIEW IF EXISTS result_vt_epss;");
     sql ("DROP TABLE nvts, nvt_preferences, vt_refs, vt_severities;");
 
     sql ("ALTER TABLE vt_refs_rebuild RENAME TO vt_refs;");

src/utils.c

@@ -959,3 +959,26 @@ wait_for_pid (pid_t pid, const char *context)
         }
     }
 }
+
+/**
+ * @brief Get the available physical memory in bytes.
+ * 
+ * @return The available memory
+ */
+guint64
+phys_mem_available ()
+{
+  return (unsigned long long)(sysconf(_SC_AVPHYS_PAGES))
+    * sysconf(_SC_PAGESIZE);
+}
+
+/**
+ * @brief Get the total physical memory in bytes.
+ * 
+ * @return The total memory
+ */
+guint64
+phys_mem_total ()
+{
+  return sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE);
+}

src/utils.h

@@ -103,4 +103,10 @@ fork_with_handlers ();
 void
 wait_for_pid (pid_t, const char *);
 
+guint64
+phys_mem_available ();
+
+guint64
+phys_mem_total ();
+
 #endif /* not _GVMD_UTILS_H */