diff --git a/recipe/meta.yaml b/recipe/meta.yaml index adf3c0210..67a39cdae 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -122,7 +122,9 @@ outputs: - python run: - {{ pin_subpackage('yggdrasil.c', exact=True) }} - - {{ compiler('fortran') }} + - m2w64-gcc-fortran # [win] + - m2w64-toolchain_win-64 # [win] + - {{ compiler('fortran') }} # [not win] - name: yggdrasil.r build: string: py{{ PY_VER_MAJOR }}{{ PY_VER_MINOR }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }} diff --git a/tests/conftest.py b/tests/conftest.py index 1bfce5234..159ca8250 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -836,6 +836,12 @@ def project_dir(): return os.path.abspath(os.path.dirname(yggdrasil.__file__)) +@pytest.fixture(scope="session") +def external_dir(): + r"""Directory outside of the local yggdrasil installation.""" + return os.path.dirname(os.getcwd()) + + @pytest.fixture(scope="session") def logger(): r"""Package logger.""" @@ -1064,7 +1070,7 @@ def check_service_manager_settings_w(service_type, partial_commtype=None): @pytest.fixture(scope="session") def running_service(pytestconfig, check_service_manager_settings, - project_dir): + project_dir, external_dir, logger): r"""Context manager to run and clean-up an integration service.""" manager = pytestconfig.pluginmanager plugin_class = manager.get_plugin('pytest_cov').CovPlugin @@ -1095,7 +1101,7 @@ def running_service_w(service_type, partial_commtype=None, args.append("--track-memory") if debug: args.append("--debug") - process_kws = {} + process_kws = {'cwd': external_dir} if with_coverage: script_path = os.path.expanduser(os.path.join('~', 'run_server.py')) process_kws['cwd'] = project_dir @@ -1131,6 +1137,7 @@ def running_service_w(service_type, partial_commtype=None, assert cli.service_type == 'flask' assert not cli.is_running p = subprocess.Popen(args, **process_kws) + logger.info(f"Started service manager via {args} ({process_kws})") try: cli.wait_for_server() yield cli diff --git a/utils/requirements/requirements.json b/utils/requirements/requirements.json index 97a1e223d..0f9d32e25 100644 --- a/utils/requirements/requirements.json +++ b/utils/requirements/requirements.json @@ -235,12 +235,22 @@ "gfortran": [ { "name": "{{ compiler('fortran') }}", - "method": "conda_recipe" + "method": "conda_recipe", + "os": "unix" }, { "name": "fortran-compiler", + "os": "unix", "method": "conda" }, + { + "name": "m2w64-gcc-fortran", + "os": "win", + "method": "conda", + "add": [ + "m2w64-toolchain_win-64" + ] + }, { "name": "gfortran", "method": "apt" diff --git a/utils/requirements/requirements.yaml b/utils/requirements/requirements.yaml index 09a3c2368..e2b60c6d6 100644 --- a/utils/requirements/requirements.yaml +++ b/utils/requirements/requirements.yaml @@ -144,14 +144,14 @@ extras: - gfortran: - name: "{{ compiler('fortran') }}" method: conda_recipe - # os: unix + os: unix - name: fortran-compiler - # os: unix + os: unix method: conda - # - name: m2w64-gcc-fortran - # os: win - # method: conda - # add: [m2w64-toolchain_win-64] + - name: m2w64-gcc-fortran + os: win + method: conda + add: [m2w64-toolchain_win-64] - name: gfortran method: apt - name: gfortran diff --git a/utils/requirements/requirements_condaonly.txt b/utils/requirements/requirements_condaonly.txt index 44a574704..6ad92e28e 100644 --- a/utils/requirements/requirements_condaonly.txt +++ b/utils/requirements/requirements_condaonly.txt @@ -1,11 +1,13 @@ cmake; platform_system != 'Windows' # [conda,c] compiler-rt; platform_system == 'Darwin' # [conda,c] czmq # [conda,zmq] -fortran-compiler # [conda,fortran] +fortran-compiler; platform_system != 'Windows' # [conda,fortran] ghostscript # [conda,images] git # [conda] juliaup # [conda,julia] lz4 # [conda,zmq] +m2w64-gcc-fortran; platform_system == 'Windows' # [conda,fortran] +m2w64-toolchain_win-64; platform_system == 'Windows' # [conda,fortran] make; platform_system != 'Windows' # [conda,c] matplotlib-base # [conda] mpich; platform_system == 'Linux' # [conda,mpi] diff --git a/yggdrasil/config.py b/yggdrasil/config.py index d496c9be7..73296a66e 100644 --- a/yggdrasil/config.py +++ b/yggdrasil/config.py @@ -381,10 +381,10 @@ def update_language_config(languages=None, skip_warnings=False, ygg_cfg.reload() if not skip_warnings: for sect, opt, desc in miss: # pragma: windows - warnings.warn(("Could not set option %s in section %s. " - + "Please set this in %s to: %s") - % (opt, sect, ygg_cfg_usr.file_to_update, desc), - RuntimeWarning) + warnings.warn( + f"Could not set option {opt} in section {sect}. Please " + f"set this in {ygg_cfg_usr.file_to_update} to: {desc}", + RuntimeWarning) if verbose: with open(usr_config_file, 'r') as fd: print(fd.read()) diff --git a/yggdrasil/drivers/CModelDriver.py b/yggdrasil/drivers/CModelDriver.py index f375cf2bd..09ddab31c 100755 --- a/yggdrasil/drivers/CModelDriver.py +++ b/yggdrasil/drivers/CModelDriver.py @@ -107,6 +107,19 @@ class GCCCompiler(CCompilerBase): 'specialization': 'with_asan'}, } + @classmethod + def is_alias(cls): + r"""Determine if the tool is actually an alias for another tool. + + Returns: + bool, str: False if this tool is not an alias, otherwise + return the name of the aliased tool. + + """ + if cls.is_clang(): + return 'clang' + return super(GCCCompiler, cls).is_alias() + @classmethod def is_clang(cls): r"""Determine if this tool is actually an alias for clang. diff --git a/yggdrasil/drivers/CPPModelDriver.py b/yggdrasil/drivers/CPPModelDriver.py index b35f27c86..b99246d08 100644 --- a/yggdrasil/drivers/CPPModelDriver.py +++ b/yggdrasil/drivers/CPPModelDriver.py @@ -64,6 +64,19 @@ class GPPCompiler(CPPCompilerBase, GCCCompiler): standard_library = 'stdc++' libraries = {} + @classmethod + def is_alias(cls): + r"""Determine if the tool is actually an alias for another tool. + + Returns: + bool, str: False if this tool is not an alias, otherwise + return the name of the aliased tool. + + """ + if cls.is_clang(): + return 'clang++' + return super(GPPCompiler, cls).is_alias() + @classmethod def get_flags(cls, skip_standard_flag=False, **kwargs): r"""Get a list of compiler flags. diff --git a/yggdrasil/drivers/CompiledModelDriver.py b/yggdrasil/drivers/CompiledModelDriver.py index 595014103..28ab510aa 100644 --- a/yggdrasil/drivers/CompiledModelDriver.py +++ b/yggdrasil/drivers/CompiledModelDriver.py @@ -121,6 +121,9 @@ def _toolnames(self, tooltype, toolname): if platform._is_win: out += [x.lower() for x in out.copy()] out = list(set([self._check_toolname(tooltype, x) for x in out])) + out_aliased = [self.tooltype[tooltype][x].is_alias() for x in out + if x in self.tooltype[tooltype]] + out = [k for k in out_aliased if k] + out return out def _check_tooltype(self, tooltype, driver=None): @@ -604,6 +607,9 @@ def tool(self, tooltype, toolname=None, language=None, for x in self._toolnames(tooltype, toolname): if x in self.tooltype[tooltype]: out = self.tooltype[tooltype][x] + aliased = out.is_alias() + if aliased: + out = self.tooltype[tooltype][aliased] break return self._check_return(tooltype, out, default=default, **sorting_kws) @@ -925,6 +931,26 @@ def get_group(self, origin): return {k: v for k, v in self.libraries.items() if v.origin == origin} + def remove(self, key, cfg=None): + r"""Remove a dependency from the registry. + + Args: + key (str): Name of dependency to remove. + cfg (CisConfigParser, optional): Set of configuration options + that the dependency's options should also be removed from. + + Returns: + bool: True if the dependency was present (and removed), False + otherwise. + + """ + if key in self: + if cfg is not None: + self[key].clear_cache(cfg) + del self.libraries[key] + return True + return False + @classmethod def splitext(cls, fname): r"""Split a file extension, taking special care for the .dll.a @@ -973,9 +999,40 @@ def basetool(self): self._basetool = False return self._basetool + def remove_compiler_libraries(self, basetool=None, cfg=None): + r"""Reset the libraries associated with the compiler. + + Args: + basetool (CompilationToolBase, optional): Compilation base + tool that owns the libraries that should be reset. If not + provided, self.basetool will be used. + cfg (CisConfigParser, optional): Set of configuration options + that libraries should be removed from. + + Returns: + bool: True if any librares are removed, False otherwise. + + """ + if basetool is None: + basetool = self.basetool + libs_removed = False + if basetool: + stdlib = basetool.standard_library + if stdlib and self.remove(stdlib, cfg=cfg): + libs_removed = True + for k, v in basetool.libraries.items(): + if self.remove(v.get("name", k), cfg=cfg): + libs_removed = True + return libs_removed + def add_compiler_libraries(self, basetool=None): r"""Add the standard library for the compiler. + Args: + basetool (CompilationToolBase, optional): Compilation base + tool that owns the libraries that should be added. If not + provided, self.basetool will be used. + Returns: bool: True if any libraries are added, False otherwise. @@ -2647,6 +2704,28 @@ def from_cache(self, cfg): for kk, v in cached.items()} self.files.update(cached) + def clear_cache(self, cfg): + r"""Update the configuration file to remove paths associated with + this dependency. + + Args: + cfg (CisConfigParser): Configuration class containing cached + file paths. + + """ + if self.is_internal: + return + options = [f'{self.name}_generated'] + for k, v in self.files.items(): + if not (v and isinstance(k, str) + and k.startswith(tuple(self.cached_files))): + continue + assert '_' not in self.name # So that key can be loaded + options.append(f"{self.name}_{k}") + for k in options: + if cfg.has_option(self.language, k): + cfg.remove_option(self.language, k) + def update_cache(self, cfg): r"""Update the configuration file with library file paths. @@ -2724,7 +2803,7 @@ def generate(self, filetype=None, overwrite=False, **kwargs): tool = self.tool(tooltype, **kwargs) out = None if tool: - out = tool.get_executable(full_path=True) + out = tool.get_executable(full_path=True, cfg=self.cfg) elif filetype in [f'{lang}_{k}' for lang, k in itertools.product( constants.LANGUAGES['compiled'], @@ -2745,7 +2824,7 @@ def generate(self, filetype=None, overwrite=False, **kwargs): out = None tool = self.get(tooltype, None, **kwargs) if tool: - out = tool.get_executable(full_path=True) + out = tool.get_executable(full_path=True, cfg=self.cfg) elif filetype in [f'{k}_env' for k in DependencySpecialization.tooltypes]: out = self._build_env(filetype, **kwargs) @@ -3647,7 +3726,8 @@ def _build_env(self, key, to_update=None, if self.parameters.get('flags_in_env', False): kenv = self.parameters.get( f'env_{k}', tool.default_executable_env) - out[kenv] = tool.get_executable(full_path=True) + out[kenv] = tool.get_executable(full_path=True, + cfg=self.cfg) if self.parameters.get('build_driver', False): out[kenv] = self.parameters['build_driver'].fix_path( out[kenv], for_env=True) @@ -4150,6 +4230,17 @@ def before_registration(cls): globals()[stage_cls.__name__] = stage_cls del stage_cls + @classmethod + def is_alias(cls): + r"""Determine if the tool is actually an alias for another tool. + + Returns: + bool, str: False if this tool is not an alias, otherwise + return the name of the aliased tool. + + """ + return False + @classmethod def get_tool(cls, tooltype, allow_uninstalled=False, force_simultaneous_next_stage=False, **kwargs): @@ -4518,15 +4609,20 @@ def create_flag(cls, key, value): return out @classmethod - def is_installed(cls): + def is_installed(cls, cfg=None): r"""Determine if this tool is installed by looking for the executable. + Args: + cfg (CisConfigParser, optional): Configuration options that + should be checked for a tool executable path. If not + provided, yggdrasil.config.ygg_cfg will be used. + Returns: bool: True if the tool is installed, False otherwise. """ try: - cls.get_executable() + cls.get_executable(cfg=cfg) return True except InvalidCompilationTool: return False @@ -4788,12 +4884,15 @@ def get_flags(cls, flags=None, outfile=None, libtype=None, return out @classmethod - def get_executable(cls, full_path=False): + def get_executable(cls, full_path=False, cfg=None): r"""Determine the executable that should be used to call this tool. Args: full_path (bool, optional): If True the full path to the executable file will be returned. Defaults to False. + cfg (CisConfigParser, optional): Configuration options that + should be checked for a tool executable path. If not + provided, yggdrasil.config.ygg_cfg will be used. Returns: str: Name of (or path to) the tool executable. @@ -4801,12 +4900,13 @@ def get_executable(cls, full_path=False): """ out = getattr(cls, 'executable', None) if out is None: - from yggdrasil.config import ygg_cfg out = cls.default_executable if cls.languages: - out = ygg_cfg.get(cls.languages[0], - f'{cls.toolname}_executable', - out) + if cfg is None: + from yggdrasil.config import ygg_cfg as cfg + out = cfg.get(cls.languages[0], + f'{cls.toolname}_executable', + out) if out is None or not (os.path.isfile(out) or shutil.which(out)): raise InvalidCompilationTool(f"Executable invalid for " f"{cls.tooltype} " @@ -4846,8 +4946,7 @@ def get_search_path(cls, env_only=False, libtype=None, cfg=None, """ if cfg is None: - from yggdrasil.config import ygg_cfg - cfg = ygg_cfg + from yggdrasil.config import ygg_cfg as cfg if (cls.search_path_flags is None) and (cls.search_path_envvar is None): raise NotImplementedError("get_search_path method not implemented for " "%s tool '%s'" % (cls.tooltype, cls.toolname)) @@ -4857,7 +4956,7 @@ def get_search_path(cls, env_only=False, libtype=None, cfg=None, suffix = 'lib' paths = [] # Add path based on executable - exec_file = cls.get_executable(full_path=True) + exec_file = cls.get_executable(full_path=True, cfg=cfg) if exec_file is not None: prefix, exec_dir = os.path.split(os.path.dirname(exec_file)) if exec_dir == 'bin': @@ -5584,7 +5683,7 @@ def find_component(cls, component, component_types=None, and cls.toolset in ['llvm', 'gnu'])): lib = os.path.basename(lib) lib_file = subprocess.check_output( - [cls.get_executable(), + [cls.get_executable(cfg=cfg), f'-print-file-name={lib}'] ).decode('utf-8').strip() if lib_file: @@ -6270,12 +6369,17 @@ def after_registration(cls, **kwargs): f'an alternative .') setattr(cls, f'default_{k}', None) if not kwargs.get('second_pass', False): - cls.libraries = DependencyRegistry( - cls.language, - internal=cls.internal_libraries, - external=cls.external_libraries, - standard=cls.standard_libraries, - cfg=cls.cfg, driver=cls, in_driver_registration=True) + CompiledModelDriver._reset_libraries( + cls, in_driver_registration=True) + + @staticmethod + def _reset_libraries(cls, **kwargs): + cls.libraries = DependencyRegistry( + cls.language, + internal=cls.internal_libraries, + external=cls.external_libraries, + standard=cls.standard_libraries, + cfg=cls.cfg, driver=cls, **kwargs) def parse_arguments(self, args, **kwargs): r"""Sort model arguments to determine which one is the executable @@ -6796,8 +6900,15 @@ def configure(cls, cfg, **kwargs): raise InvalidCompilationTool( f"Could not locate a {k} tool '{v}'.") cfg.set(cls.language, k, vtool.toolname) + setattr(cls, f'default_{k}', vtool.toolname) if os.path.isfile(v): cfg.set(cls.language, f'{vtool.toolname}_executable', v) + # Clear dependencies that should be set based on tool + if kwargs: + cls.cleanup_dependencies() + cls.libraries.remove_compiler_libraries(cfg=cfg) + cls.cfg = cfg + CompiledModelDriver._reset_libraries(cls) # Call __func__ to avoid direct invoking of class which dosn't # exist in after_registration where this is called return ModelDriver.configure.__func__(cls, cfg)