Apply %DEPx% processing more uniformly (#1144)

bin/lib/ce_install.py

@@ -47,9 +47,7 @@ def get_installables(self, args_filter: List[str]) -> List[Installable]:
                 yaml_doc = yaml.load(yaml_file, Loader=ConfigSafeLoader)
             for installer in installers_for(self.installation_context, yaml_doc, self.enabled):
                 installables.append(installer)
-        installables_by_name = {installable.name: installable for installable in installables}
-        for installable in installables:
-            installable.link(installables_by_name)
+        Installable.resolve(installables)
         installables = sorted(
             filter(lambda installable: filter_aggregate(args_filter, installable, self.filter_match_all), installables),
             key=lambda x: x.sort_key,

bin/lib/installable/archives.py

@@ -59,7 +59,6 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any])
         else:
             raise RuntimeError(f"Unknown compression {compression}")
         self.strip = self.config_get("strip", False)
-        self._setup_check_exe(self.install_path)
 
     def fetch_and_pipe_to(self, staging: StagingDir, s3_path: str, command: list[str]) -> None:
         # Extension point for subclasses
@@ -113,8 +112,6 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any])
         self.compiler_pattern = os.path.join(self.subdir, f"{path_name_prefix}-*")
         self.path_name_symlink = self.config_get("symlink", os.path.join(self.subdir, f"{path_name_prefix}"))
         self.num_to_keep = self.config_get("num_to_keep", 5)
-        self._setup_check_exe(self.install_path)
-        self._setup_check_link(self.install_path, self.path_name_symlink)
 
     @property
     def nightly_like(self) -> bool:
@@ -203,9 +200,6 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any])
         if extract_only:
             self.tar_cmd += [extract_only]
         self.strip = self.config_get("strip", False)
-        self._setup_check_exe(self.install_path)
-        if self.install_path_symlink:
-            self._setup_check_link(self.install_path, self.install_path_symlink)
         self.remove_older_pattern = self.config_get("remove_older_pattern", "")
         self.num_to_keep = self.config_get("num_to_keep", 5)
 
@@ -259,10 +253,6 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any])
         today = datetime.today().strftime("%Y%m%d")
         self.install_path = f"{self.install_path}-{today}"
 
-        # redo exe checks
-        self._setup_check_exe(self.install_path)
-        self._setup_check_link(self.install_path, self.install_path_symlink)
-
     def should_install(self) -> bool:
         return True
 
@@ -283,9 +273,6 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any])
         self.folder_to_rename = self.config_get("folder", None if not self.extract_into_folder else "tmp")
         self.configure_command = command_config(self.config_get("configure_command", []))
         self.strip = self.config_get("strip", False)
-        self._setup_check_exe(self.install_path)
-        if self.install_path_symlink:
-            self._setup_check_link(self.install_path, self.install_path_symlink)
 
     def stage(self, staging: StagingDir) -> None:
         # Unzip does not support stdin piping so we need to create a file

bin/lib/installable/edg.py

@@ -80,7 +80,6 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any])
         self._compiler_type = self.config_get("compiler_type")
         self._shim_shell_func = _SHIM_SHELL_FUNCS[self._compiler_type]
         self.install_path = self.config_get("path_name")
-        self._setup_check_exe(self.install_path)
 
     def stage(self, staging: StagingDir) -> None:
         super().stage(staging)

bin/lib/installable/installable.py

@@ -17,6 +17,7 @@
 from lib.staging import StagingDir
 
 _LOGGER = logging.getLogger(__name__)
+_DEP_RE = re.compile("%DEP([0-9]+)%")
 
 running_on_admin_node = socket.gethostname() == "admin-node"
 
@@ -26,7 +27,7 @@
 class Installable:
     _check_link: Optional[Callable[[], bool]]
     check_env: Dict
-    check_file: Optional[str]
+    check_file: str
     check_call: List[str]
 
     def __init__(self, install_context: InstallationContext, config: Dict[str, Any]):
@@ -39,47 +40,60 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any])
         self.language = False
         if len(self.context) > 0:
             self.is_library = self.context[0] == "libraries"
+        if len(self.context) > 1:
             self.language = self.context[1]
         self.depends_by_name = self.config.get("depends", [])
         self.depends: List[Installable] = []
         self.install_always = self.config.get("install_always", False)
         self._check_link = None
         self.build_config = LibraryBuildConfig(config)
         self.check_env = {}
-        self.check_file = None
+        self.check_file = self.config_get("check_file", "")
         self.check_call = []
+        check_exe = self.config_get("check_exe", "")
+        if check_exe:
+            self.check_call = command_config(check_exe)
         self.check_stderr_on_stdout = self.config.get("check_stderr_on_stdout", False)
         self.install_path = ""
         self.after_stage_script = self.config_get("after_stage_script", [])
         self._logger = logging.getLogger(self.name)
         self.install_path_symlink = self.config_get("symlink", False)
 
-    def _setup_check_exe(self, path_name: str) -> None:
-        self.check_env = dict([x.replace("%PATH%", path_name).split("=", 1) for x in self.config_get("check_env", [])])
-
-        check_file = self.config_get("check_file", "")
-        if check_file:
-            self.check_file = os.path.join(path_name, check_file)
-        else:
-            self.check_call = command_config(self.config_get("check_exe"))
-            self.check_call[0] = os.path.join(path_name, self.check_call[0])
-
-    def _setup_check_link(self, source: str, link: str) -> None:
-        self._check_link = partial(self.install_context.check_link, source, link)
-
-    def link(self, all_installables: Dict[str, Installable]):
+    def _resolve(self, all_installables: Dict[str, Installable]):
         try:
             self.depends = [all_installables[dep] for dep in self.depends_by_name]
         except KeyError as ke:
             self._logger.error("Unable to find dependency %s in %s", ke, all_installables)
             raise
-        dep_re = re.compile("%DEP([0-9]+)%")
 
         def dep_n(match):
             return str(self.install_context.destination / self.depends[int(match.group(1))].install_path)
 
-        for k in self.check_env.keys():
-            self.check_env[k] = dep_re.sub(dep_n, self.check_env[k])
+        def resolve_deps(s: str) -> str:
+            return _DEP_RE.sub(dep_n, s)
+
+        self.check_env = dict(
+            [x.replace("%PATH%", self.install_path).split("=", 1) for x in self.config_get("check_env", [])]
+        )
+        self.check_env = {key: resolve_deps(value) for key, value in self.check_env.items()}
+
+        self.after_stage_script = [resolve_deps(line) for line in self.after_stage_script]
+        self.check_file = resolve_deps(self.check_file)
+        if self.check_file:
+            self.check_file = os.path.join(self.install_path, self.check_file)
+
+        self.check_call = [resolve_deps(arg) for arg in self.check_call]
+        if self.check_call:
+            self.check_call[0] = os.path.join(self.install_path, self.check_call[0])
+
+        if self.install_path_symlink:
+            self._check_link = partial(self.install_context.check_link, self.install_path, self.install_path_symlink)
+
+    @staticmethod
+    def resolve(installables: list[Installable]) -> None:
+        installables_by_name = {installable.name: installable for installable in installables}
+        for installable in installables:
+            installable._resolve(installables_by_name)  # pylint: disable=protected-access
 
     def find_dependee(self, name: str) -> Installable:
         for i in self.depends:
@@ -134,7 +148,7 @@ def save_version(self, exe: str, res_call: str):
         nightlies.update_version(fullpath.as_posix(), str(modified), res_call.split("\n", 1)[0], res_call)
 
     def is_installed(self) -> bool:
-        if self.check_file is None and not self.check_call:
+        if not self.check_file and not self.check_call:
             return True
 
         if self._check_link and not self._check_link():
@@ -244,7 +258,6 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any])
         self.install_path = self.config_get("dir")
         self.url = self.config_get("url")
         self.filename = self.config_get("filename")
-        self._setup_check_exe(self.install_path)
 
     def stage(self, staging: StagingDir) -> None:
         out_path = staging.path / self.install_path

bin/lib/installable/python.py

@@ -13,7 +13,6 @@ class PipInstallable(Installable):
     def __init__(self, install_context: InstallationContext, config: Dict[str, Any]):
         super().__init__(install_context, config)
         self.install_path = self.config_get("dir")
-        self._setup_check_exe(self.install_path)
         self.package = self.config_get("package")
         self.python = self.config_get("python")
 

bin/lib/installable/rust.py

@@ -31,7 +31,6 @@ class RustInstallable(Installable):
     def __init__(self, install_context: InstallationContext, config: Dict[str, Any]):
         super().__init__(install_context, config)
         self.install_path = self.config_get("dir")
-        self._setup_check_exe(self.install_path)
         self.base_package = self.config_get("base_package")
         self.nightly_install_days = self.config_get("nightly_install_days", 0)
         self.patchelf = self.config_get("patchelf")

bin/lib/installable/script.py

@@ -17,9 +17,6 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any])
         self.fetch = self.config_get("fetch")
         self.script = self.config_get("script")
         self.strip = self.config_get("strip", False)
-        self._setup_check_exe(self.install_path)
-        if self.install_path_symlink:
-            self._setup_check_link(self.install_path, self.install_path_symlink)
 
     def stage(self, staging: StagingDir) -> None:
         for url in self.fetch:

bin/lib/installable/solidity.py

@@ -23,7 +23,6 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any])
             raise RuntimeError(f"Unable to find solidity {self.target_name}")
         self.url = f"{self.url}/{release_path}"
         self.filename = self.config_get("filename")
-        self._setup_check_exe(self.install_path)
 
     def __repr__(self) -> str:
         return f"SolidityInstallable({self.name}, {self.install_path})"

bin/test/installation_test.py

@@ -1,8 +1,13 @@
+from pathlib import Path
+
 import pytest
 import yaml
 
 from lib.config_safe_loader import ConfigSafeLoader
+from lib.installable.installable import Installable
 from lib.installation import targets_from
+from lib.installation_context import InstallationContext
+from unittest import mock
 
 
 def parse_targets(string_config, enabled=None):
@@ -167,3 +172,39 @@ def test_jinja_expansion_with_filters_refering_forward():
       """
     )
     assert target["url"] == "https://dl.bintray.com/boostorg/release/1.64.0/source/boost_1_64_0.tar.bz2"
+
+
+def test_after_stage_script_dep():
+    ic = mock.Mock(spec_set=InstallationContext)
+    ic.destination = Path("/some/install/dir")
+    installation_a = Installable(
+        ic,
+        {
+            "context": ["compilers"],
+            "name": "a",
+            "after_stage_script": ["echo hello", "echo %DEP0%", "moo"],
+            "depends": ["compilers b"],
+        },
+    )
+    installation_b = Installable(ic, {"context": ["compilers"], "name": "b"})
+    installation_b.install_path = "pathy"
+    Installable.resolve([installation_a, installation_b])
+    assert installation_a.after_stage_script == ["echo hello", "echo /some/install/dir/pathy", "moo"]
+
+
+def test_check_exe_dep():
+    ic = mock.Mock(spec_set=InstallationContext)
+    ic.destination = Path("/some/install/dir")
+    installation_a = Installable(
+        ic,
+        {
+            "context": ["compilers"],
+            "name": "a",
+            "check_exe": "%DEP0%/bin/java --jar path/to/jar",
+            "depends": ["compilers b"],
+        },
+    )
+    installation_b = Installable(ic, {"context": ["compilers"], "name": "b"})
+    installation_b.install_path = "pathy"
+    Installable.resolve([installation_a, installation_b])
+    assert installation_a.check_call == ["/some/install/dir/pathy/bin/java", "--jar", "path/to/jar"]