From 322de87a50c5ec6eaaeb4f6b12b8e8eab1afc35a Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Fri, 12 Apr 2024 22:19:05 -0500 Subject: [PATCH] Add conda_build.yaml --- conda_build/build.py | 14 ++--- conda_build/exceptions.py | 3 ++ conda_build/metadata.py | 61 +++++----------------- conda_build/render.py | 66 ++++++++++-------------- conda_build/skeletons/cran.py | 12 +---- conda_build/variants.py | 4 +- conda_build/yaml.py | 94 ++++++++++++++++++++++++++++++++++ tests/cli/test_main_inspect.py | 5 +- tests/cli/test_main_render.py | 3 +- tests/test_api_build.py | 22 ++------ tests/test_api_render.py | 5 +- tests/test_api_skeleton.py | 6 +-- tests/test_variants.py | 25 ++++----- 13 files changed, 168 insertions(+), 152 deletions(-) create mode 100644 conda_build/yaml.py diff --git a/conda_build/build.py b/conda_build/build.py index d0c939d9e8..fabe1c3579 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -23,7 +23,6 @@ from pathlib import Path import conda_package_handling.api -import yaml from bs4 import UnicodeDammit from conda import __version__ as conda_version from conda.base.context import context, reset_context @@ -31,6 +30,8 @@ from conda.exceptions import CondaError, NoPackagesFoundError, UnsatisfiableError from conda.models.channel import Channel +from conda_build import yaml + from . import __version__ as conda_build_version from . import environ, noarch_python, source, tarcheck, utils from .conda_interface import ( @@ -882,7 +883,7 @@ def copy_recipe(m): # dump the full variant in use for this package to the recipe folder with open(os.path.join(recipe_dir, "conda_build_config.yaml"), "w") as f: - yaml.dump(m.config.variant, f) + yaml.safe_dump(m.config.variant, f) def copy_readme(m): @@ -918,11 +919,12 @@ def jsonify_info_yamls(m): except: pass with open(file) as i, open(dst, "w") as o: - import yaml - - yaml = yaml.full_load(i) json.dump( - yaml, o, sort_keys=True, indent=2, separators=(",", ": ") + yaml.safe_load(i), + o, + sort_keys=True, + indent=2, + separators=(",", ": "), ) res.append( join(os.path.basename(m.config.info_dir), ijd, bn + ".json") diff --git a/conda_build/exceptions.py b/conda_build/exceptions.py index f38706786a..49ef9e351c 100644 --- a/conda_build/exceptions.py +++ b/conda_build/exceptions.py @@ -2,6 +2,8 @@ # SPDX-License-Identifier: BSD-3-Clause import textwrap +from .deprecations import deprecated + SEPARATOR = "-" * 70 indent = lambda s: textwrap.fill(textwrap.dedent(s)) @@ -42,6 +44,7 @@ def indented_exception(self): return f"Error Message:\n--> {indent(orig)}\n\n" +@deprecated("24.5", "24.7", addendum="Unused.") class UnableToParseMissingJinja2(UnableToParse): def error_body(self): return "\n".join( diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 07425e404e..41e77d90a0 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -20,9 +20,10 @@ from conda.gateways.disk.read import compute_sum from frozendict import deepfreeze -from . import exceptions, utils, variants +from . import exceptions, utils, variants, yaml from .conda_interface import MatchSpec from .config import Config, get_or_merge_config +from .deprecations import deprecated from .features import feature_list from .license_family import ensure_valid_license_family from .utils import ( @@ -34,50 +35,19 @@ insert_variant_versions, on_win, ) +from .yaml import _StringifyNumbersLoader if TYPE_CHECKING: from typing import Literal -try: - import yaml -except ImportError: - sys.exit( - "Error: could not import yaml (required to read meta.yaml " - "files of conda recipes)" - ) - -try: - Loader = yaml.CLoader -except AttributeError: - Loader = yaml.Loader - - -class StringifyNumbersLoader(Loader): - @classmethod - def remove_implicit_resolver(cls, tag): - if "yaml_implicit_resolvers" not in cls.__dict__: - cls.yaml_implicit_resolvers = { - k: v[:] for k, v in cls.yaml_implicit_resolvers.items() - } - for ch in tuple(cls.yaml_implicit_resolvers): - resolvers = [(t, r) for t, r in cls.yaml_implicit_resolvers[ch] if t != tag] - if resolvers: - cls.yaml_implicit_resolvers[ch] = resolvers - else: - del cls.yaml_implicit_resolvers[ch] - - @classmethod - def remove_constructor(cls, tag): - if "yaml_constructors" not in cls.__dict__: - cls.yaml_constructors = cls.yaml_constructors.copy() - if tag in cls.yaml_constructors: - del cls.yaml_constructors[tag] - +deprecated.constant( + "24.5", + "24.7", + "StringifyNumbersLoader", + _StringifyNumbersLoader, + addendum="Use `conda_build.yaml._StringifyNumbersLoader` instead.", +) -StringifyNumbersLoader.remove_implicit_resolver("tag:yaml.org,2002:float") -StringifyNumbersLoader.remove_implicit_resolver("tag:yaml.org,2002:int") -StringifyNumbersLoader.remove_constructor("tag:yaml.org,2002:float") -StringifyNumbersLoader.remove_constructor("tag:yaml.org,2002:int") # arches that don't follow exact names in the subdir need to be mapped here ARCH_MAP = {"32": "x86", "64": "x86_64"} @@ -305,15 +275,8 @@ def select_lines(data, namespace, variants_in_place): def yamlize(data): try: - return yaml.load(data, Loader=StringifyNumbersLoader) - except yaml.error.YAMLError as e: - if "{{" in data: - try: - import jinja2 - - jinja2 # Avoid pyflakes failure: 'jinja2' imported but unused - except ImportError: - raise exceptions.UnableToParseMissingJinja2(original=e) + return yaml.safe_load(data, stringify_numbers=True) + except yaml.YAMLError as e: print("Problematic recipe:", file=sys.stderr) print(data, file=sys.stderr) raise exceptions.UnableToParse(original=e) diff --git a/conda_build/render.py b/conda_build/render.py index be17eaa461..239141d16d 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -3,7 +3,6 @@ from __future__ import annotations import json -import os import random import re import string @@ -15,7 +14,6 @@ from contextlib import contextmanager from functools import lru_cache from os.path import ( - dirname, isabs, isdir, isfile, @@ -25,13 +23,14 @@ from pathlib import Path from typing import TYPE_CHECKING -import yaml +import yaml as pyyaml from conda.base.context import context from conda.core.package_cache_data import ProgressiveFetchExtract from conda.exceptions import UnsatisfiableError -from . import environ, exceptions, source, utils +from . import environ, exceptions, source, utils, yaml from .conda_interface import PackageRecord, TemporaryDirectory, specs_from_url +from .deprecations import deprecated from .exceptions import DependencyNeedsBuildingError from .index import get_build_index from .metadata import MetaData, combine_top_level_metadata_with_output @@ -47,20 +46,17 @@ ) if TYPE_CHECKING: + import os from typing import Iterator from .config import Config +@deprecated("24.5", "24.7") def odict_representer(dumper, data): return dumper.represent_dict(data.items()) -yaml.add_representer(set, yaml.representer.SafeRepresenter.represent_list) -yaml.add_representer(tuple, yaml.representer.SafeRepresenter.represent_list) -yaml.add_representer(OrderedDict, odict_representer) - - def bldpkg_path(m): """ Returns path to built package's tarball given its ``Metadata``. @@ -1026,6 +1022,7 @@ def render_recipe( # Next bit of stuff is to support YAML output in the order we expect. # http://stackoverflow.com/a/17310199/1170370 +@deprecated("24.5", "24.7") class _MetaYaml(dict): fields = FIELDS @@ -1033,16 +1030,19 @@ def to_omap(self): return [(field, self[field]) for field in _MetaYaml.fields if field in self] +@deprecated("24.5", "24.7") def _represent_omap(dumper, data): return dumper.represent_mapping("tag:yaml.org,2002:map", data.to_omap()) +@deprecated("24.5", "24.7") def _unicode_representer(dumper, uni): - node = yaml.ScalarNode(tag="tag:yaml.org,2002:str", value=uni) + node = pyyaml.ScalarNode(tag="tag:yaml.org,2002:str", value=uni) return node -class _IndentDumper(yaml.Dumper): +@deprecated("24.5", "24.7") +class _IndentDumper(pyyaml.Dumper): def increase_indent(self, flow=False, indentless=False): return super().increase_indent(flow, False) @@ -1050,33 +1050,23 @@ def ignore_aliases(self, data): return True -yaml.add_representer(_MetaYaml, _represent_omap) -yaml.add_representer(str, _unicode_representer) -unicode = None # silence pyflakes about unicode not existing in py3 +def output_yaml( + metadata: MetaData, + filename: str | os.PathLike | Path | None = None, + suppress_outputs: bool = False, +) -> str: + meta = metadata.meta + # create a manually ordered copy of the meta dict + meta = {field: meta[field] for field in FIELDS if field in meta} + if suppress_outputs and metadata.is_output and "outputs" in meta: + del meta["outputs"] + output = yaml.safe_dump(meta) -def output_yaml(metadata, filename=None, suppress_outputs=False): - local_metadata = metadata.copy() - if ( - suppress_outputs - and local_metadata.is_output - and "outputs" in local_metadata.meta - ): - del local_metadata.meta["outputs"] - output = yaml.dump( - _MetaYaml(local_metadata.meta), - Dumper=_IndentDumper, - default_flow_style=False, - indent=2, - ) - if filename: - if any(sep in filename for sep in ("\\", "/")): - try: - os.makedirs(dirname(filename)) - except OSError: - pass - with open(filename, "w") as f: - f.write(output) - return "Wrote yaml to %s" % filename - else: + if not filename: return output + + filename = Path(filename) + filename.parent.mkdir(parents=True, exist_ok=True) + filename.write_text(output) + return "Wrote yaml to %s" % filename diff --git a/conda_build/skeletons/cran.py b/conda_build/skeletons/cran.py index 7140c9a89f..1e2ff563ec 100755 --- a/conda_build/skeletons/cran.py +++ b/conda_build/skeletons/cran.py @@ -29,18 +29,10 @@ realpath, relpath, ) +from typing import TYPE_CHECKING import requests import yaml - -# try to import C dumper -try: - from yaml import CSafeDumper as SafeDumper -except ImportError: - from yaml import SafeDumper - -from typing import TYPE_CHECKING - from conda.common.io import dashlist from .. import source @@ -564,7 +556,7 @@ def yaml_quote_string(string): Note that this function is NOT general. """ return ( - yaml.dump(string, indent=True, Dumper=SafeDumper) + yaml.safe_dump(string, indent=True) .replace("\n...\n", "") .replace("\n", "\n ") .rstrip("\n ") diff --git a/conda_build/variants.py b/conda_build/variants.py index c5bbe9a41e..e91696872d 100644 --- a/conda_build/variants.py +++ b/conda_build/variants.py @@ -11,9 +11,9 @@ from functools import lru_cache from itertools import product -import yaml from conda.base.context import context +from . import yaml from .conda_interface import cc_conda_build from .utils import ensure_list, get_logger, islist, on_win, trim_empty_keys from .version import _parse as parse_version @@ -136,7 +136,7 @@ def parse_config_file(path, config): with open(path) as f: contents = f.read() contents = select_lines(contents, get_selectors(config), variants_in_place=False) - content = yaml.load(contents, Loader=yaml.loader.BaseLoader) or {} + content = yaml.safe_load(contents) or {} trim_empty_keys(content) return content diff --git a/conda_build/yaml.py b/conda_build/yaml.py new file mode 100644 index 0000000000..63fd0da968 --- /dev/null +++ b/conda_build/yaml.py @@ -0,0 +1,94 @@ +# Copyright (C) 2014 Anaconda, Inc +# SPDX-License-Identifier: BSD-3-Clause +from __future__ import annotations + +from collections import OrderedDict +from typing import TYPE_CHECKING, overload + +from frozendict import frozendict +from yaml import dump as _dump +from yaml import load as _load +from yaml.error import YAMLError # noqa: F401 + +try: + from yaml import CSafeDumper as _SafeDumper + from yaml import CSafeLoader as _SafeLoader +except ImportError: + from yaml import SafeDumper as _SafeDumper # type: ignore[assignment] + from yaml import SafeLoader as _SafeLoader # type: ignore[assignment] + +if TYPE_CHECKING: + from typing import Any, Self, TextIO + + +class _NoAliasesDumper(_SafeDumper): + def ignore_aliases(self: Self, data: Any) -> bool: + return True + + +_NoAliasesDumper.add_representer(set, _NoAliasesDumper.represent_list) +_NoAliasesDumper.add_representer(tuple, _NoAliasesDumper.represent_list) +_NoAliasesDumper.add_representer(OrderedDict, _NoAliasesDumper.represent_dict) +_NoAliasesDumper.add_representer(frozendict, _NoAliasesDumper.represent_dict) + + +@overload +def safe_dump(data: Any, stream: None = None, **kwargs) -> str: ... + + +@overload +def safe_dump(data: Any, stream: TextIO, **kwargs) -> None: ... + + +def safe_dump( + data: Any, + stream: TextIO | None = None, + *, + default_flow_style: bool = False, # always serialize in the block style + indent: int = 2, + sort_keys: bool = False, # prefer the manual ordering + **kwargs, +) -> str | None: + return _dump( + data, + stream, + Dumper=_NoAliasesDumper, + default_flow_style=default_flow_style, + indent=indent, + **kwargs, + ) + + +class _StringifyNumbersLoader(_SafeLoader): + @classmethod + def remove_implicit_resolver(cls, tag): + if "yaml_implicit_resolvers" not in cls.__dict__: + cls.yaml_implicit_resolvers = { + k: v[:] for k, v in cls.yaml_implicit_resolvers.items() + } + for ch in tuple(cls.yaml_implicit_resolvers): + resolvers = [(t, r) for t, r in cls.yaml_implicit_resolvers[ch] if t != tag] + if resolvers: + cls.yaml_implicit_resolvers[ch] = resolvers + else: + del cls.yaml_implicit_resolvers[ch] + + @classmethod + def remove_constructor(cls, tag): + if "yaml_constructors" not in cls.__dict__: + cls.yaml_constructors = cls.yaml_constructors.copy() + if tag in cls.yaml_constructors: + del cls.yaml_constructors[tag] + + +_StringifyNumbersLoader.remove_implicit_resolver("tag:yaml.org,2002:float") +_StringifyNumbersLoader.remove_implicit_resolver("tag:yaml.org,2002:int") +_StringifyNumbersLoader.remove_constructor("tag:yaml.org,2002:float") +_StringifyNumbersLoader.remove_constructor("tag:yaml.org,2002:int") + + +def safe_load(stream: str | TextIO, *, stringify_numbers: bool = False) -> Any: + return _load( + stream, + _StringifyNumbersLoader if stringify_numbers else _SafeLoader, + ) diff --git a/tests/cli/test_main_inspect.py b/tests/cli/test_main_inspect.py index b8931b5220..9d5c046192 100644 --- a/tests/cli/test_main_inspect.py +++ b/tests/cli/test_main_inspect.py @@ -5,9 +5,8 @@ import sys import pytest -import yaml -from conda_build import api +from conda_build import api, yaml from conda_build.cli import main_inspect from conda_build.utils import on_win @@ -77,7 +76,7 @@ def test_inspect_hash_input(testing_metadata, testing_workdir, capfd): api.output_yaml(testing_metadata, "meta.yaml") output = api.build(testing_workdir, notest=True)[0] with open(os.path.join(testing_workdir, "conda_build_config.yaml"), "w") as f: - yaml.dump({"zlib": ["1.2.11"]}, f) + yaml.safe_dump({"zlib": ["1.2.11"]}, f) args = ["hash-inputs", output] main_inspect.execute(args) output, error = capfd.readouterr() diff --git a/tests/cli/test_main_render.py b/tests/cli/test_main_render.py index 59fff7901c..d69af016a0 100644 --- a/tests/cli/test_main_render.py +++ b/tests/cli/test_main_render.py @@ -4,9 +4,8 @@ import sys import pytest -import yaml -from conda_build import api +from conda_build import api, yaml from conda_build.cli import main_render from conda_build.conda_interface import TemporaryDirectory diff --git a/tests/test_api_build.py b/tests/test_api_build.py index 5932bf4f1a..1ae56f2461 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -24,7 +24,6 @@ # for version import conda import pytest -import yaml from binstar_client.commands import remove, show from binstar_client.errors import NotFound from conda.base.context import context, reset_context @@ -32,7 +31,7 @@ from conda.exceptions import ClobberError, CondaError, CondaMultiError, LinkError from conda_index.api import update_index -from conda_build import __version__, api, exceptions +from conda_build import __version__, api, exceptions, yaml from conda_build.conda_interface import url_path from conda_build.config import Config from conda_build.exceptions import ( @@ -72,21 +71,6 @@ from conda_build.metadata import MetaData -def represent_ordereddict(dumper, data): - value = [] - - for item_key, item_value in data.items(): - node_key = dumper.represent_data(item_key) - node_value = dumper.represent_data(item_value) - - value.append((node_key, node_value)) - - return yaml.nodes.MappingNode("tag:yaml.org,2002:map", value) - - -yaml.add_representer(OrderedDict, represent_ordereddict) - - class AnacondaClientArgs: def __init__( self, specs, token=None, site=None, log_level=logging.INFO, force=False @@ -782,7 +766,7 @@ def test_relative_git_url_submodule_clone(testing_workdir, testing_config, monke } with open(filename, "w") as outfile: - outfile.write(yaml.dump(data, default_flow_style=False, width=999999999)) + outfile.write(yaml.safe_dump(data, width=999999999)) # Reset the path because our broken, dummy `git` would cause `render_recipe` # to fail, while no `git` will cause the build_dependencies to be installed. monkeypatch.undo() @@ -804,7 +788,7 @@ def test_noarch(testing_workdir): ] ) with open(filename, "w") as outfile: - outfile.write(yaml.dump(data, default_flow_style=False, width=999999999)) + outfile.write(yaml.safe_dump(data, width=999999999)) output = api.get_output_file_paths(testing_workdir)[0] assert os.path.sep + "noarch" + os.path.sep in output or not noarch assert os.path.sep + "noarch" + os.path.sep not in output or noarch diff --git a/tests/test_api_render.py b/tests/test_api_render.py index 7849daa01c..291a0cd123 100644 --- a/tests/test_api_render.py +++ b/tests/test_api_render.py @@ -10,11 +10,10 @@ from itertools import count, islice import pytest -import yaml from conda.base.context import context from conda.common.compat import on_win -from conda_build import api, render +from conda_build import api, render, yaml from conda_build.conda_interface import cc_conda_build from conda_build.variants import validate_spec @@ -219,7 +218,7 @@ def test_setting_condarc_vars_with_env_var_expansion(testing_workdir): python_versions = ["2.6", "3.4", "3.11"] config = {"python": python_versions, "bzip2": ["0.9", "1.0"]} with open(os.path.join("config", "conda_build_config.yaml"), "w") as f: - yaml.dump(config, f, default_flow_style=False) + yaml.safe_dump(config, f) cc_conda_build_backup = cc_conda_build.copy() # hacky equivalent of changing condarc diff --git a/tests/test_api_skeleton.py b/tests/test_api_skeleton.py index a8273492b0..fdb1f019b0 100644 --- a/tests/test_api_skeleton.py +++ b/tests/test_api_skeleton.py @@ -9,9 +9,8 @@ from typing import TYPE_CHECKING import pytest -import ruamel.yaml -from conda_build import api +from conda_build import api, yaml from conda_build.skeletons.pypi import ( clean_license_name, convert_to_flat_list, @@ -484,8 +483,7 @@ def test_pypi_section_order_preserved(tmp_path: Path): if not line.startswith("{%") ] - # The loader below preserves the order of entries... - recipe = ruamel.yaml.load("\n".join(lines), Loader=ruamel.yaml.RoundTripLoader) + recipe = yaml.safe_load("\n".join(lines)) major_sections = list(recipe.keys()) # Blank fields are omitted when skeletonizing, so prune any missing ones diff --git a/tests/test_variants.py b/tests/test_variants.py index 50e9cea4f2..ec8d86e3e0 100644 --- a/tests/test_variants.py +++ b/tests/test_variants.py @@ -8,10 +8,9 @@ from pathlib import Path import pytest -import yaml from conda.common.compat import on_mac -from conda_build import api, exceptions +from conda_build import api, exceptions, yaml from conda_build.utils import ensure_list, package_has_file from conda_build.variants import ( combine_specs, @@ -66,7 +65,7 @@ def test_python_variants(testing_workdir, testing_config, as_yaml): # write variants to disk if as_yaml: variants_path = Path(testing_workdir, "variant_example.yaml") - variants_path.write_text(yaml.dump(variants, default_flow_style=False)) + variants_path.write_text(yaml.safe_dump(variants)) testing_config.variant_config_files = [str(variants_path)] # render the metadata @@ -501,19 +500,15 @@ def test_numpy_used_variable_looping(): def test_exclusive_config_files(): with open("conda_build_config.yaml", "w") as f: - yaml.dump({"abc": ["someval"], "cwd": ["someval"]}, f, default_flow_style=False) + yaml.safe_dump({"abc": ["someval"], "cwd": ["someval"]}, f) os.makedirs("config_dir") with open(os.path.join("config_dir", "config-0.yaml"), "w") as f: - yaml.dump( - {"abc": ["super_0"], "exclusive_0": ["0"], "exclusive_both": ["0"]}, - f, - default_flow_style=False, + yaml.safe_dump( + {"abc": ["super_0"], "exclusive_0": ["0"], "exclusive_both": ["0"]}, f ) with open(os.path.join("config_dir", "config-1.yaml"), "w") as f: - yaml.dump( - {"abc": ["super_1"], "exclusive_1": ["1"], "exclusive_both": ["1"]}, - f, - default_flow_style=False, + yaml.safe_dump( + {"abc": ["super_1"], "exclusive_1": ["1"], "exclusive_both": ["1"]}, f ) exclusive_config_files = ( os.path.join("config_dir", "config-0.yaml"), @@ -538,12 +533,10 @@ def test_exclusive_config_files(): def test_exclusive_config_file(): with open("conda_build_config.yaml", "w") as f: - yaml.dump({"abc": ["someval"], "cwd": ["someval"]}, f, default_flow_style=False) + yaml.safe_dump({"abc": ["someval"], "cwd": ["someval"]}, f) os.makedirs("config_dir") with open(os.path.join("config_dir", "config.yaml"), "w") as f: - yaml.dump( - {"abc": ["super"], "exclusive": ["someval"]}, f, default_flow_style=False - ) + yaml.safe_dump({"abc": ["super"], "exclusive": ["someval"]}, f) output = api.render( os.path.join(variants_dir, "exclusive_config_file"), exclusive_config_file=os.path.join("config_dir", "config.yaml"),