Nightlyversions part 2

bin/lib/installable/installable.py

@@ -3,10 +3,12 @@
 import logging
 import os
 import re
+import socket
 import subprocess
 from functools import partial
 from pathlib import Path
 from typing import Optional, Callable, Dict, List, Any, Union
+from lib.nightly_versions import NightlyVersions
 
 from lib.installation_context import InstallationContext
 from lib.library_build_config import LibraryBuildConfig
@@ -16,6 +18,10 @@
 
 _LOGGER = logging.getLogger(__name__)
 
+running_on_admin_node = socket.gethostname() == "admin-node"
+
+nightlies: NightlyVersions = NightlyVersions(_LOGGER)
+
 
 class Installable:
     _check_link: Optional[Callable[[], bool]]
@@ -103,9 +109,29 @@ def install(self) -> None:
                 dependee.install()
         self._logger.debug("Dependees installed")
 
-    # pylint: disable=unused-argument
     def save_version(self, exe: str, res_call: str):
-        return
+        if not running_on_admin_node:
+            self._logger.warning("Not running on admin node - not saving compiler version info to AWS")
+            return
+
+        if not self.nightly_like:
+            return
+
+        # exe is something like "gcc-trunk-20231008/bin/g++" here
+        #  but we need the actual symlinked path ("/opt/compiler-explorer/gcc-snapshot/bin/g++")
+
+        # note: NightlyInstallable also has "path_name_symlink", so this function is overridden there
+
+        relative_exe = "/".join(exe.split("/")[1:])
+        if self.install_path_symlink:
+            fullpath = self.install_context.destination / self.install_path_symlink / relative_exe
+        else:
+            fullpath = self.install_context.destination / exe
+
+        stat = fullpath.stat()
+        modified = stat.st_mtime
+
+        nightlies.update_version(fullpath.as_posix(), str(modified), res_call.split("\n", 1)[0], res_call)
 
     def is_installed(self) -> bool:
         if self.check_file is None and not self.check_call:
@@ -157,7 +183,7 @@ def sort_key(self):
 
     @property
     def nightly_like(self) -> bool:
-        return False
+        return self.target_name in ["nightly", "trunk", "master", "main"]
 
     def build(self, buildfor, popular_compilers_only):
         if not self.is_library:

bin/lib/installable/rust.py

@@ -2,12 +2,10 @@
 
 import functools
 import os
-import socket
 import subprocess
 from datetime import datetime, timedelta
 from pathlib import Path
 from typing import Dict, Any
-from lib.nightly_versions import NightlyVersions
 
 from lib.amazon import list_s3_artifacts
 from lib.installable.installable import Installable
@@ -18,10 +16,6 @@
 
 _LOGGER = logging.getLogger(__name__)
 
-running_on_admin_node = socket.gethostname() == "admin-node"
-
-nightlies: NightlyVersions = NightlyVersions(_LOGGER)
-
 
 @functools.lru_cache(maxsize=512)
 def s3_available_rust_artifacts(prefix):
@@ -107,27 +101,6 @@ def install(self) -> None:
             self.stage(staging)
             self.install_context.move_from_staging(staging, self.install_path)
 
-    def save_version(self, exe: str, res_call: str):
-        if not self.nightly_like:
-            return
-
-        if not running_on_admin_node:
-            self._logger.warning("Not running on admin node - not saving compiler version info to AWS")
-            return
-
-        # exe is something like "gcc-trunk-20231008/bin/g++" here
-        #  but we need the actual symlinked path ("/opt/compiler-explorer/gcc-snapshot/bin/g++")
-        relative_exe = "/".join(exe.split("/")[1:])
-        if self.install_path_symlink:
-            fullpath = self.install_context.destination / self.install_path_symlink / relative_exe
-        else:
-            fullpath = self.install_context.destination / exe
-
-        stat = fullpath.stat()
-        modified = stat.st_mtime
-
-        nightlies.update_version(fullpath.as_posix(), str(modified), res_call.split("\n", 1)[0], res_call)
-
     def __repr__(self) -> str:
         return f"RustInstallable({self.name}, {self.install_path})"
 

bin/lib/installation_context.py

@@ -253,20 +253,32 @@ def check_output(self, args: List[str], env: Optional[dict] = None, stderr_on_st
         args[0] = str(self.destination / args[0])
         _LOGGER.debug("Executing %s in %s", args, self.destination)
         if not is_windows():
-            return subprocess.check_output(
+            output = subprocess.run(
                 args,
                 cwd=str(self.destination),
                 env=env,
                 stdin=subprocess.DEVNULL,
                 stderr=subprocess.STDOUT if stderr_on_stdout else None,
-            ).decode("utf-8")
+                capture_output=True,
+                check=True,
+            )
         else:
-            return subprocess.check_output(
+            output = subprocess.run(
                 args,
                 cwd=str(self.destination),
                 stdin=subprocess.DEVNULL,
                 stderr=subprocess.STDOUT if stderr_on_stdout else None,
-            ).decode("utf-8")
+                capture_output=True,
+                check=True,
+            )
+
+        fulloutput = ""
+        if output.stdout != None:
+            fulloutput += output.stdout.decode("utf-8")
+        if output.stderr != None:
+            fulloutput += output.stderr.decode("utf-8")
+
+        return fulloutput
 
     def check_call(self, args: List[str], env: Optional[dict] = None) -> None:
         args = args[:]

bin/lib/nightly_versions.py

@@ -1,5 +1,5 @@
 from collections import defaultdict
-from typing import Any, Dict
+from typing import Any, Dict, Set
 from lib.amazon import dynamodb_client
 from lib.amazon_properties import get_properties_compilers_and_libraries
 
@@ -12,6 +12,8 @@ class NightlyVersions:
     c: Dict[str, Dict[str, Any]] = defaultdict(lambda: {})
     fortran: Dict[str, Dict[str, Any]] = defaultdict(lambda: {})
     rust: Dict[str, Dict[str, Any]] = defaultdict(lambda: {})
+    swift: Dict[str, Dict[str, Any]] = defaultdict(lambda: {})
+    clean: Dict[str, Dict[str, Any]] = defaultdict(lambda: {})
 
     def __init__(self, logger):
         self.logger = logger
@@ -22,6 +24,8 @@ def load_ce_properties(self):
             [self.c, _] = get_properties_compilers_and_libraries("c", self.logger)
             [self.fortran, _] = get_properties_compilers_and_libraries("fortran", self.logger)
             [self.rust, _] = get_properties_compilers_and_libraries("rust", self.logger)
+            [self.swift, _] = get_properties_compilers_and_libraries("swift", self.logger)
+            [self.clean, _] = get_properties_compilers_and_libraries("clean", self.logger)
             self.props_loaded = True
 
     def as_c_compiler(self, exe: str):
@@ -36,37 +40,36 @@ def as_fortran_compiler(self, exe: str):
             return exe[:-3] + "gfortran"
         return exe
 
+    def collect_compiler_ids_for(self, ids: set, exe: str, compilers: Dict[str, Dict[str, Any]]):
+        for compiler_id in compilers:
+            compiler = compilers[compiler_id]
+            if exe == compiler["exe"]:
+                ids.add(compiler_id)
+
     def get_compiler_ids(self, exe: str):
         self.load_ce_properties()
 
-        ids = set()
+        ids: Set = set()
 
-        for compiler_id in self.cpp:
-            compiler = self.cpp[compiler_id]
-            if exe == compiler["exe"]:
-                ids.add(compiler_id)
-
-        for compiler_id in self.rust:
-            compiler = self.rust[compiler_id]
-            if exe == compiler["exe"]:
-                ids.add(compiler_id)
+        self.collect_compiler_ids_for(ids, exe, self.cpp)
+        self.collect_compiler_ids_for(ids, exe, self.rust)
+        self.collect_compiler_ids_for(ids, exe, self.swift)
+        self.collect_compiler_ids_for(ids, exe, self.clean)
 
         cexe = self.as_c_compiler(exe)
-        for compiler_id in self.c:
-            compiler = self.c[compiler_id]
-            if cexe == compiler["exe"]:
-                ids.add(compiler_id)
+        self.collect_compiler_ids_for(ids, cexe, self.c)
 
         fortranexe = self.as_fortran_compiler(exe)
-        for compiler_id in self.fortran:
-            compiler = self.fortran[compiler_id]
-            if fortranexe == compiler["exe"]:
-                ids.add(compiler_id)
+        self.collect_compiler_ids_for(ids, fortranexe, self.fortran)
 
         return ids
 
     def update_version(self, exe: str, modified: str, version: str, full_version: str):
         compiler_ids = self.get_compiler_ids(exe)
+        if len(compiler_ids) == 0:
+            self.logger.warning(f"No compiler ids found for {exe} - not saving compiler version info to AWS")
+            return
+
         dynamodb_client.put_item(
             TableName=self.version_table_name,
             Item={