diff --git a/bin/lib/ce_install.py b/bin/lib/ce_install.py index 6e76d8ad3..2c6f7fdae 100644 --- a/bin/lib/ce_install.py +++ b/bin/lib/ce_install.py @@ -47,9 +47,7 @@ def get_installables(self, args_filter: List[str]) -> List[Installable]: yaml_doc = yaml.load(yaml_file, Loader=ConfigSafeLoader) for installer in installers_for(self.installation_context, yaml_doc, self.enabled): installables.append(installer) - installables_by_name = {installable.name: installable for installable in installables} - for installable in installables: - installable.link(installables_by_name) + Installable.resolve(installables) installables = sorted( filter(lambda installable: filter_aggregate(args_filter, installable, self.filter_match_all), installables), key=lambda x: x.sort_key, diff --git a/bin/lib/installable/archives.py b/bin/lib/installable/archives.py index 6ca34d4bd..9e434e3f0 100644 --- a/bin/lib/installable/archives.py +++ b/bin/lib/installable/archives.py @@ -59,7 +59,6 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any]) else: raise RuntimeError(f"Unknown compression {compression}") self.strip = self.config_get("strip", False) - self._setup_check_exe(self.install_path) def fetch_and_pipe_to(self, staging: StagingDir, s3_path: str, command: list[str]) -> None: # Extension point for subclasses @@ -113,8 +112,6 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any]) self.compiler_pattern = os.path.join(self.subdir, f"{path_name_prefix}-*") self.path_name_symlink = self.config_get("symlink", os.path.join(self.subdir, f"{path_name_prefix}")) self.num_to_keep = self.config_get("num_to_keep", 5) - self._setup_check_exe(self.install_path) - self._setup_check_link(self.install_path, self.path_name_symlink) @property def nightly_like(self) -> bool: @@ -203,9 +200,6 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any]) if extract_only: self.tar_cmd += [extract_only] self.strip = self.config_get("strip", False) - self._setup_check_exe(self.install_path) - if self.install_path_symlink: - self._setup_check_link(self.install_path, self.install_path_symlink) self.remove_older_pattern = self.config_get("remove_older_pattern", "") self.num_to_keep = self.config_get("num_to_keep", 5) @@ -259,10 +253,6 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any]) today = datetime.today().strftime("%Y%m%d") self.install_path = f"{self.install_path}-{today}" - # redo exe checks - self._setup_check_exe(self.install_path) - self._setup_check_link(self.install_path, self.install_path_symlink) - def should_install(self) -> bool: return True @@ -283,9 +273,6 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any]) self.folder_to_rename = self.config_get("folder", None if not self.extract_into_folder else "tmp") self.configure_command = command_config(self.config_get("configure_command", [])) self.strip = self.config_get("strip", False) - self._setup_check_exe(self.install_path) - if self.install_path_symlink: - self._setup_check_link(self.install_path, self.install_path_symlink) def stage(self, staging: StagingDir) -> None: # Unzip does not support stdin piping so we need to create a file diff --git a/bin/lib/installable/edg.py b/bin/lib/installable/edg.py index 2f4864831..647c19e6b 100644 --- a/bin/lib/installable/edg.py +++ b/bin/lib/installable/edg.py @@ -80,7 +80,6 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any]) self._compiler_type = self.config_get("compiler_type") self._shim_shell_func = _SHIM_SHELL_FUNCS[self._compiler_type] self.install_path = self.config_get("path_name") - self._setup_check_exe(self.install_path) def stage(self, staging: StagingDir) -> None: super().stage(staging) diff --git a/bin/lib/installable/installable.py b/bin/lib/installable/installable.py index 514feb75a..963c66b01 100644 --- a/bin/lib/installable/installable.py +++ b/bin/lib/installable/installable.py @@ -17,6 +17,7 @@ from lib.staging import StagingDir _LOGGER = logging.getLogger(__name__) +_DEP_RE = re.compile("%DEP([0-9]+)%") running_on_admin_node = socket.gethostname() == "admin-node" @@ -26,7 +27,7 @@ class Installable: _check_link: Optional[Callable[[], bool]] check_env: Dict - check_file: Optional[str] + check_file: str check_call: List[str] def __init__(self, install_context: InstallationContext, config: Dict[str, Any]): @@ -39,6 +40,7 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any]) self.language = False if len(self.context) > 0: self.is_library = self.context[0] == "libraries" + if len(self.context) > 1: self.language = self.context[1] self.depends_by_name = self.config.get("depends", []) self.depends: List[Installable] = [] @@ -46,40 +48,52 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any]) self._check_link = None self.build_config = LibraryBuildConfig(config) self.check_env = {} - self.check_file = None + self.check_file = self.config_get("check_file", "") self.check_call = [] + check_exe = self.config_get("check_exe", "") + if check_exe: + self.check_call = command_config(check_exe) self.check_stderr_on_stdout = self.config.get("check_stderr_on_stdout", False) self.install_path = "" self.after_stage_script = self.config_get("after_stage_script", []) self._logger = logging.getLogger(self.name) self.install_path_symlink = self.config_get("symlink", False) - def _setup_check_exe(self, path_name: str) -> None: - self.check_env = dict([x.replace("%PATH%", path_name).split("=", 1) for x in self.config_get("check_env", [])]) - - check_file = self.config_get("check_file", "") - if check_file: - self.check_file = os.path.join(path_name, check_file) - else: - self.check_call = command_config(self.config_get("check_exe")) - self.check_call[0] = os.path.join(path_name, self.check_call[0]) - - def _setup_check_link(self, source: str, link: str) -> None: - self._check_link = partial(self.install_context.check_link, source, link) - - def link(self, all_installables: Dict[str, Installable]): + def _resolve(self, all_installables: Dict[str, Installable]): try: self.depends = [all_installables[dep] for dep in self.depends_by_name] except KeyError as ke: self._logger.error("Unable to find dependency %s in %s", ke, all_installables) raise - dep_re = re.compile("%DEP([0-9]+)%") def dep_n(match): return str(self.install_context.destination / self.depends[int(match.group(1))].install_path) - for k in self.check_env.keys(): - self.check_env[k] = dep_re.sub(dep_n, self.check_env[k]) + def resolve_deps(s: str) -> str: + return _DEP_RE.sub(dep_n, s) + + self.check_env = dict( + [x.replace("%PATH%", self.install_path).split("=", 1) for x in self.config_get("check_env", [])] + ) + self.check_env = {key: resolve_deps(value) for key, value in self.check_env.items()} + + self.after_stage_script = [resolve_deps(line) for line in self.after_stage_script] + self.check_file = resolve_deps(self.check_file) + if self.check_file: + self.check_file = os.path.join(self.install_path, self.check_file) + + self.check_call = [resolve_deps(arg) for arg in self.check_call] + if self.check_call: + self.check_call[0] = os.path.join(self.install_path, self.check_call[0]) + + if self.install_path_symlink: + self._check_link = partial(self.install_context.check_link, self.install_path, self.install_path_symlink) + + @staticmethod + def resolve(installables: list[Installable]) -> None: + installables_by_name = {installable.name: installable for installable in installables} + for installable in installables: + installable._resolve(installables_by_name) # pylint: disable=protected-access def find_dependee(self, name: str) -> Installable: for i in self.depends: @@ -134,7 +148,7 @@ def save_version(self, exe: str, res_call: str): nightlies.update_version(fullpath.as_posix(), str(modified), res_call.split("\n", 1)[0], res_call) def is_installed(self) -> bool: - if self.check_file is None and not self.check_call: + if not self.check_file and not self.check_call: return True if self._check_link and not self._check_link(): @@ -244,7 +258,6 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any]) self.install_path = self.config_get("dir") self.url = self.config_get("url") self.filename = self.config_get("filename") - self._setup_check_exe(self.install_path) def stage(self, staging: StagingDir) -> None: out_path = staging.path / self.install_path diff --git a/bin/lib/installable/python.py b/bin/lib/installable/python.py index 0bea398aa..a99d9a6a7 100644 --- a/bin/lib/installable/python.py +++ b/bin/lib/installable/python.py @@ -13,7 +13,6 @@ class PipInstallable(Installable): def __init__(self, install_context: InstallationContext, config: Dict[str, Any]): super().__init__(install_context, config) self.install_path = self.config_get("dir") - self._setup_check_exe(self.install_path) self.package = self.config_get("package") self.python = self.config_get("python") diff --git a/bin/lib/installable/rust.py b/bin/lib/installable/rust.py index 0c6dea87a..961199680 100644 --- a/bin/lib/installable/rust.py +++ b/bin/lib/installable/rust.py @@ -31,7 +31,6 @@ class RustInstallable(Installable): def __init__(self, install_context: InstallationContext, config: Dict[str, Any]): super().__init__(install_context, config) self.install_path = self.config_get("dir") - self._setup_check_exe(self.install_path) self.base_package = self.config_get("base_package") self.nightly_install_days = self.config_get("nightly_install_days", 0) self.patchelf = self.config_get("patchelf") diff --git a/bin/lib/installable/script.py b/bin/lib/installable/script.py index a99185d35..a47dac232 100644 --- a/bin/lib/installable/script.py +++ b/bin/lib/installable/script.py @@ -17,9 +17,6 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any]) self.fetch = self.config_get("fetch") self.script = self.config_get("script") self.strip = self.config_get("strip", False) - self._setup_check_exe(self.install_path) - if self.install_path_symlink: - self._setup_check_link(self.install_path, self.install_path_symlink) def stage(self, staging: StagingDir) -> None: for url in self.fetch: diff --git a/bin/lib/installable/solidity.py b/bin/lib/installable/solidity.py index 9ea47b799..60b6cd076 100644 --- a/bin/lib/installable/solidity.py +++ b/bin/lib/installable/solidity.py @@ -23,7 +23,6 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any]) raise RuntimeError(f"Unable to find solidity {self.target_name}") self.url = f"{self.url}/{release_path}" self.filename = self.config_get("filename") - self._setup_check_exe(self.install_path) def __repr__(self) -> str: return f"SolidityInstallable({self.name}, {self.install_path})" diff --git a/bin/test/installation_test.py b/bin/test/installation_test.py index e7ef7b33d..f93e46ac9 100644 --- a/bin/test/installation_test.py +++ b/bin/test/installation_test.py @@ -1,8 +1,13 @@ +from pathlib import Path + import pytest import yaml from lib.config_safe_loader import ConfigSafeLoader +from lib.installable.installable import Installable from lib.installation import targets_from +from lib.installation_context import InstallationContext +from unittest import mock def parse_targets(string_config, enabled=None): @@ -167,3 +172,39 @@ def test_jinja_expansion_with_filters_refering_forward(): """ ) assert target["url"] == "https://dl.bintray.com/boostorg/release/1.64.0/source/boost_1_64_0.tar.bz2" + + +def test_after_stage_script_dep(): + ic = mock.Mock(spec_set=InstallationContext) + ic.destination = Path("/some/install/dir") + installation_a = Installable( + ic, + { + "context": ["compilers"], + "name": "a", + "after_stage_script": ["echo hello", "echo %DEP0%", "moo"], + "depends": ["compilers b"], + }, + ) + installation_b = Installable(ic, {"context": ["compilers"], "name": "b"}) + installation_b.install_path = "pathy" + Installable.resolve([installation_a, installation_b]) + assert installation_a.after_stage_script == ["echo hello", "echo /some/install/dir/pathy", "moo"] + + +def test_check_exe_dep(): + ic = mock.Mock(spec_set=InstallationContext) + ic.destination = Path("/some/install/dir") + installation_a = Installable( + ic, + { + "context": ["compilers"], + "name": "a", + "check_exe": "%DEP0%/bin/java --jar path/to/jar", + "depends": ["compilers b"], + }, + ) + installation_b = Installable(ic, {"context": ["compilers"], "name": "b"}) + installation_b.install_path = "pathy" + Installable.resolve([installation_a, installation_b]) + assert installation_a.check_call == ["/some/install/dir/pathy/bin/java", "--jar", "path/to/jar"]