From bd63936d9baf364287516947b87ba89e3fac974e Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 30 Aug 2022 06:15:08 -0700 Subject: [PATCH 01/19] Initial pass at implementing #4 --- dcargs/__init__.py | 3 +- dcargs/_argparse_formatter.py | 2 +- dcargs/_calling.py | 8 +- dcargs/_fields.py | 17 ++++- dcargs/_parsers.py | 44 ++++++++--- dcargs/_resolver.py | 58 ++++++++++++-- dcargs/_strings.py | 11 +++ dcargs/extras/__init__.py | 6 +- dcargs/extras/_base_configs.py | 75 +++++++++++++++++++ dcargs/metadata/__init__.py | 16 ++++ dcargs/metadata/_subcommands.py | 55 ++++++++++++++ ..._unions.rst => 06_literals_and_unions.rst} | 6 +- ...tional_args.rst => 07_positional_args.rst} | 10 +-- .../{09_subparsers.rst => 08_subparsers.rst} | 22 +++--- ...parsers.rst => 09_multiple_subparsers.rst} | 18 ++--- ...6_base_configs.rst => 10_base_configs.rst} | 55 +++++--------- ...nd_unions.py => 06_literals_and_unions.py} | 2 +- ...sitional_args.py => 07_positional_args.py} | 4 +- .../{09_subparsers.py => 08_subparsers.py} | 10 +-- ...ubparsers.py => 09_multiple_subparsers.py} | 8 +- ...{06_base_configs.py => 10_base_configs.py} | 43 +++-------- tests/test_helptext.py | 8 +- tests/test_nested.py | 16 ++++ 23 files changed, 357 insertions(+), 140 deletions(-) create mode 100644 dcargs/extras/_base_configs.py create mode 100644 dcargs/metadata/__init__.py create mode 100644 dcargs/metadata/_subcommands.py rename docs/source/examples/{07_literals_and_unions.rst => 06_literals_and_unions.rst} (92%) rename docs/source/examples/{08_positional_args.rst => 07_positional_args.rst} (88%) rename docs/source/examples/{09_subparsers.rst => 08_subparsers.rst} (68%) rename docs/source/examples/{10_multiple_subparsers.rst => 09_multiple_subparsers.rst} (79%) rename docs/source/examples/{06_base_configs.rst => 10_base_configs.rst} (65%) rename examples/{07_literals_and_unions.py => 06_literals_and_unions.py} (96%) rename examples/{08_positional_args.py => 07_positional_args.py} (93%) rename examples/{09_subparsers.py => 08_subparsers.py} (73%) rename examples/{10_multiple_subparsers.py => 09_multiple_subparsers.py} (84%) rename examples/{06_base_configs.py => 10_base_configs.py} (66%) diff --git a/dcargs/__init__.py b/dcargs/__init__.py index 4022ff6a..0b317d4f 100644 --- a/dcargs/__init__.py +++ b/dcargs/__init__.py @@ -1,10 +1,11 @@ -from . import extras +from . import extras, metadata from ._cli import cli, generate_parser from ._fields import MISSING_PUBLIC as MISSING from ._instantiators import UnsupportedTypeAnnotationError __all__ = [ "extras", + "metadata", "cli", "generate_parser", "MISSING", diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index 3a59b9b6..bfc30292 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -136,7 +136,7 @@ def _format_action(self, action): ) ) # - parts.append("%*s%s\n" % (indent_first, "", help_lines[0])) + parts.append("%*s%s\n" % (indent_first, "", help_lines[0])) # type: ignore for line in help_lines[1:]: parts.append("%*s%s\n" % (help_position, "", line)) diff --git a/dcargs/_calling.py b/dcargs/_calling.py index 958b587f..cfab0e48 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -56,11 +56,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: prefixed_field_name = _strings.make_field_name([field_name_prefix, field.name]) # Resolve field type. - field_type = ( - type_from_typevar[field.typ] # type: ignore - if field.typ in type_from_typevar - else field.typ - ) + field_type = type_from_typevar.get(field.typ, field.typ) # type: ignore if prefixed_field_name in arg_from_prefixed_field_name: assert prefixed_field_name not in consumed_keywords @@ -148,7 +144,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: value = None else: options = map( - lambda x: x if x not in type_from_typevar else type_from_typevar[x], + lambda x: type_from_typevar.get(x, x), get_args(field_type), ) chosen_f = None diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 412e84b3..a61de72d 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -145,6 +145,14 @@ def _try_field_list_from_callable( f: Union[Callable, Type], default_instance: _DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: + from . import metadata as _metadata + + f, subcommand_config = _resolver.unwrap_annotated( + f, _metadata._subcommands._SubcommandConfiguration + ) + if subcommand_config is not None: + default_instance = subcommand_config.default + # Unwrap generics. f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.narrow_type(f, default_instance) @@ -156,7 +164,7 @@ def _try_field_list_from_callable( if isinstance(f, type): cls = f f = cls.__init__ # type: ignore - f_origin: Callable = cls + f_origin: Callable = cls # type: ignore f_origin = _resolver.unwrap_origin(f) # Try special cases. @@ -191,6 +199,7 @@ def _try_field_list_from_callable( " default to infer from." ) assert isinstance(default_instance, Iterable) + contained_type = next(iter(default_instance)) else: (contained_type,) = get_args(f) f_origin = list if f_origin is typing.Sequence else f_origin # type: ignore @@ -232,7 +241,7 @@ def _try_field_list_from_typeddict( and default_instance is not EXCLUDE_FROM_CALL ) assert not valid_default_instance or isinstance(default_instance, dict) - for name, typ in get_type_hints(cls).items(): + for name, typ in get_type_hints(cls, include_extras=True).items(): if valid_default_instance: default = default_instance.get(name, MISSING_PROP) # type: ignore elif getattr(cls, "__total__") is False: @@ -268,7 +277,7 @@ def _try_field_list_from_namedtuple( field_defaults = getattr(cls, "_field_defaults") # Note that _field_types is removed in Python 3.9. - for name, typ in _resolver.get_type_hints(cls).items(): + for name, typ in get_type_hints(cls, include_extras=True).items(): # Get default, with priority for `default_instance`. default = field_defaults.get(name, MISSING_NONPROP) if hasattr(default_instance, name): @@ -483,7 +492,7 @@ def _field_list_from_params( # This will throw a type error for torch.device, typing.Dict, etc. try: - hints = get_type_hints(f) + hints = get_type_hints(f, include_extras=True) except TypeError: return UnsupportedNestedTypeMessage(f"Could not get hints for {f}!") diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 46d6965c..5277d4a3 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -18,6 +18,7 @@ _resolver, _strings, ) +from . import metadata as _metadata T = TypeVar("T") @@ -255,18 +256,38 @@ def from_field( # Add subparser for each option. parser_from_name: Dict[str, ParserSpecification] = {} for option in options_no_none: - subparser_name = _strings.subparser_name_from_type(prefix, option) - parser_from_name[subparser_name] = ParserSpecification.from_callable( + name = _strings.subparser_name_from_type(prefix, option) + option, subcommand_config = _resolver.unwrap_annotated( + option, _metadata._subcommands._SubcommandConfiguration + ) + if subcommand_config is None: + subcommand_config = _metadata._subcommands._SubcommandConfiguration( + "unused", + description=None, + default=( + field.default + if type(field.default) is _resolver.unwrap_origin(option) + else _fields.MISSING_NONPROP + ), + ) + + subparser = ParserSpecification.from_callable( option, - description=None, + description=subcommand_config.description, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, - default_instance=field.default - if type(field.default) == _resolver.unwrap_origin(option) # type: ignore - else _fields.MISSING_NONPROP, + default_instance=subcommand_config.default, prefix=prefix, avoid_subparsers=avoid_subparsers, ) + subparser = dataclasses.replace( + subparser, + helptext_from_nested_class_field_name={ + _strings.make_field_name([field.name, k]): v + for k, v in subparser.helptext_from_nested_class_field_name.items() + }, + ) + parser_from_name[name] = subparser # Optional if: type hint is Optional[], or a default instance is provided. required = True @@ -286,9 +307,14 @@ def from_field( "Default values for generic subparsers are not supported." ) - default_parser = parser_from_name[ - _strings.subparser_name_from_type(prefix, type(field.default)) - ] + default_name = _strings.subparser_name_from_type( + prefix, type(field.default) + ) + assert default_name in parser_from_name, ( + "Default with type {type(field.default)} was passed in, but no matching" + " subparser." + ) + default_parser = parser_from_name[default_name] if any(map(lambda arg: arg.lowered.required, default_parser.args)): required = True if any( diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index f6598cf9..88e3fe03 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -2,18 +2,30 @@ import copy import dataclasses -from typing import Any, Callable, Dict, List, Tuple, Type, TypeVar, Union, cast +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + Tuple, + Type, + TypeVar, + Union, + cast, +) from typing_extensions import get_args, get_origin, get_type_hints TypeOrCallable = TypeVar("TypeOrCallable", Type, Callable) -def unwrap_origin(tp: TypeOrCallable) -> TypeOrCallable: - """Returns the origin of tp if it exists. Otherwise, returns tp.""" - origin = get_origin(tp) +def unwrap_origin(typ: TypeOrCallable) -> TypeOrCallable: + """Returns the origin of typ if it exists. Otherwise, returns typ.""" + typ, _ = unwrap_annotated(typ) + origin = get_origin(typ) if origin is None: - return tp + return typ else: return origin @@ -57,7 +69,7 @@ def resolved_fields(cls: Type) -> List[dataclasses.Field]: assert dataclasses.is_dataclass(cls) fields = [] - annotations = get_type_hints(cls) + annotations = get_type_hints(cls, include_extras=True) for field in dataclasses.fields(cls): # Avoid mutating original field. field = copy.copy(field) @@ -108,3 +120,37 @@ def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: except TypeError: pass return typ + + +MetadataType = TypeVar("MetadataType") + + +def unwrap_annotated( + typ: TypeOrCallable, search_type: Optional[Type[MetadataType]] = None +) -> Tuple[TypeOrCallable, Optional[MetadataType]]: + """Helper for parsing typing.Annotated types. + + Examples: + - int, int => (int, ()) + - Annotated[int, 1], int => (int, 1) + - Annotated[int, "1"], int => (int, None) + """ + if not hasattr(typ, "__metadata__"): + return typ, None + + args = get_args(typ) + assert len(args) >= 2 + + # Don't search for a specific metadata type if `None` is passed in. + if search_type is None: + return args[0], None + + # Look through metadata for desired metadata type. + targets = tuple(x for x in args[1:] if isinstance(x, search_type)) + if len(targets) == 0: + return args[0], None + else: + assert ( + len(targets) == 1 + ), f"Found two instances of {search_type} in metadata, but only expected one." + return args[0], targets[0] diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 7ac05c46..1cc781ee 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -54,7 +54,18 @@ def hyphen_separated_from_camel_case(name: str) -> str: def _subparser_name_from_type(cls: Type) -> str: + from .metadata import _subcommands # Prevent circular imports + cls, type_from_typevar = _resolver.resolve_generic_types(cls) + cls, subcommand_config = _resolver.unwrap_annotated( + cls, _subcommands._SubcommandConfiguration + ) + + # Subparser name from `dcargs.metadata.subcommand()`. + if subcommand_config is not None: + return subcommand_config.name + + # Subparser name from class name. if len(type_from_typevar) == 0: assert hasattr(cls, "__name__") return hyphen_separated_from_camel_case(cls.__name__) # type: ignore diff --git a/dcargs/extras/__init__.py b/dcargs/extras/__init__.py index 71ab7651..dd38ce86 100644 --- a/dcargs/extras/__init__.py +++ b/dcargs/extras/__init__.py @@ -1,3 +1,7 @@ +"""The :mod:`dcargs.extras` submodule contains helpers that complement :func:`dcargs.cli()`, but +aren't considered part of the core interface.""" + +from ._base_configs import union_type_from_mapping from ._serialization import from_yaml, to_yaml -__all__ = ["to_yaml", "from_yaml"] +__all__ = ["union_type_from_mapping", "to_yaml", "from_yaml"] diff --git a/dcargs/extras/_base_configs.py b/dcargs/extras/_base_configs.py new file mode 100644 index 00000000..cd2bb5ee --- /dev/null +++ b/dcargs/extras/_base_configs.py @@ -0,0 +1,75 @@ +from typing import Mapping, Type, TypeVar, Union + +from typing_extensions import Annotated + +from ..metadata import subcommand + +T = TypeVar("T") + + +def union_type_from_mapping(base_mapping: Mapping[str, T]) -> Type[T]: + """Returns a Union type for defining subcommands that choose between nested types. + + For example, when `base_mapping` is set to: + + ```python + { + "small": Config(...), + "big": Config(...), + } + ``` + + We return: + + ```python + Union[ + Annotated[ + Config, + dcargs.metadata.subcommand("small", default=Config(...)) + ], + Annotated[ + Config, + dcargs.metadata.subcommand("big", default=Config(...)) + ] + ] + ``` + + This can be used directly in dcargs.cli: + + ```python + config = dcargs.cli(union_from_base_mapping(base_mapping)) + reveal_type(config) # Should be correct! + ``` + + Or to generate annotations for functions: + + ```python + SelectableConfig = union_from_base_mapping(base_mapping) + + def train( + config: SelectableConfig, + checkpoint_path: Optional[pathlib.Path] = None, + ) -> None: + ... + + dcargs.cli(train) + ``` + + Note that Pyright understands the latter case, but mypy does not. If mypy support is + necessary we can work around this with an `if TYPE_CHECKING` guard: + + ```python + if TYPE_CHECKING: + SelectableConfig = ExperimentConfig + else: + SelectableConfig = union_from_base_mapping(base_mapping) + ``` + """ + return Union.__getitem__( # type: ignore + tuple( + Annotated.__class_getitem__( # type: ignore + (type(v), subcommand(k, default=v)) + ) + for k, v in base_mapping.items() + ) + ) diff --git a/dcargs/metadata/__init__.py b/dcargs/metadata/__init__.py new file mode 100644 index 00000000..6f2156b9 --- /dev/null +++ b/dcargs/metadata/__init__.py @@ -0,0 +1,16 @@ +"""The :mod:`dcargs.metadata` submodule contains helpers for attaching parsing-specific +metadata to types. For the forseeable future, this is limited to subcommand configuration. + +Features here are supported, but generally contradict the core design ethos of +:func:`dcargs.cli()`. + +As such: +1. Usage of existing functionality should be avoided unless absolutely necessary. +2. Introduction of new functionality should be avoided unless it (a) cannot be + reproduced with standard type annotations and (b) meaningfully improves the + usefulness of the library. +""" + +from ._subcommands import subcommand + +__all__ = ["subcommand"] diff --git a/dcargs/metadata/_subcommands.py b/dcargs/metadata/_subcommands.py new file mode 100644 index 00000000..eee8f2cb --- /dev/null +++ b/dcargs/metadata/_subcommands.py @@ -0,0 +1,55 @@ +import dataclasses +from typing import Any, Optional + +from .._fields import MISSING_NONPROP + + +@dataclasses.dataclass(frozen=True) +class _SubcommandConfiguration: + # Things we could potentially add: + # - `description` + # - `avoid_subparsers` + name: str + description: Optional[str] + default: Any + + +def subcommand( + name: str, + *, + description: Optional[str] = None, + default: Any = MISSING_NONPROP, +) -> Any: + """Returns a metadata object for configuring subcommands with `typing.Annotated`. + Use of this function is supported but discouraged unless absolutely necessary. + + --- + + Consider the standard approach for creating subcommands: + + ```python + dcargs.cli( + Union[NestedTypeA, NestedTypeB] + ) + ``` + + This will create two subcommands: nested-type-a and nested-type-b. + + + Annotating each type with `dcargs.metadata.subcommand()` allows us to override for + each subcommand the (a) name and (b) defaults. + + ```python + dcargs.cli( + Union[ + Annotated[ + NestedTypeA, subcommand("a", default=NestedTypeA(...)) + ], + Annotated[ + NestedTypeA, subcommand("b", default=NestedTypeA(...)) + ], + ] + ) + ``` + """ + return _SubcommandConfiguration(name, description, default) diff --git a/docs/source/examples/07_literals_and_unions.rst b/docs/source/examples/06_literals_and_unions.rst similarity index 92% rename from docs/source/examples/07_literals_and_unions.rst rename to docs/source/examples/06_literals_and_unions.rst index 6f43fa83..f5bbf58c 100644 --- a/docs/source/examples/07_literals_and_unions.rst +++ b/docs/source/examples/06_literals_and_unions.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -7. Literals And Unions +6. Literals And Unions ========================================== @@ -60,6 +60,6 @@ .. raw:: html - python 07_literals_and_unions.py --help + python 06_literals_and_unions.py --help -.. program-output:: python ../../examples/07_literals_and_unions.py --help +.. program-output:: python ../../examples/06_literals_and_unions.py --help diff --git a/docs/source/examples/08_positional_args.rst b/docs/source/examples/07_positional_args.rst similarity index 88% rename from docs/source/examples/08_positional_args.rst rename to docs/source/examples/07_positional_args.rst index 6dd19374..270c9b9d 100644 --- a/docs/source/examples/08_positional_args.rst +++ b/docs/source/examples/07_positional_args.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -8. Positional Args +7. Positional Args ========================================== @@ -69,14 +69,14 @@ Positional-only arguments in functions are converted to positional CLI arguments .. raw:: html - python 08_positional_args.py --help + python 07_positional_args.py --help -.. program-output:: python ../../examples/08_positional_args.py --help +.. program-output:: python ../../examples/07_positional_args.py --help ------------ .. raw:: html - python 08_positional_args.py ./a ./b --optimizer.learning-rate 1e-5 + python 07_positional_args.py ./a ./b --optimizer.learning-rate 1e-5 -.. program-output:: python ../../examples/08_positional_args.py ./a ./b --optimizer.learning-rate 1e-5 +.. program-output:: python ../../examples/07_positional_args.py ./a ./b --optimizer.learning-rate 1e-5 diff --git a/docs/source/examples/09_subparsers.rst b/docs/source/examples/08_subparsers.rst similarity index 68% rename from docs/source/examples/09_subparsers.rst rename to docs/source/examples/08_subparsers.rst index f41c03a4..bc1c20f6 100644 --- a/docs/source/examples/09_subparsers.rst +++ b/docs/source/examples/08_subparsers.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -9. Subparsers +8. Subparsers ========================================== @@ -49,38 +49,38 @@ Unions over nested types (classes or dataclasses) are populated using subparsers .. raw:: html - python 09_subparsers.py --help + python 08_subparsers.py --help -.. program-output:: python ../../examples/09_subparsers.py --help +.. program-output:: python ../../examples/08_subparsers.py --help ------------ .. raw:: html - python 09_subparsers.py cmd:commit --help + python 08_subparsers.py cmd:commit --help -.. program-output:: python ../../examples/09_subparsers.py cmd:commit --help +.. program-output:: python ../../examples/08_subparsers.py cmd:commit --help ------------ .. raw:: html - python 09_subparsers.py cmd:commit --cmd.message hello --cmd.all + python 08_subparsers.py cmd:commit --cmd.message hello --cmd.all -.. program-output:: python ../../examples/09_subparsers.py cmd:commit --cmd.message hello --cmd.all +.. program-output:: python ../../examples/08_subparsers.py cmd:commit --cmd.message hello --cmd.all ------------ .. raw:: html - python 09_subparsers.py cmd:checkout --help + python 08_subparsers.py cmd:checkout --help -.. program-output:: python ../../examples/09_subparsers.py cmd:checkout --help +.. program-output:: python ../../examples/08_subparsers.py cmd:checkout --help ------------ .. raw:: html - python 09_subparsers.py cmd:checkout --cmd.branch main + python 08_subparsers.py cmd:checkout --cmd.branch main -.. program-output:: python ../../examples/09_subparsers.py cmd:checkout --cmd.branch main +.. program-output:: python ../../examples/08_subparsers.py cmd:checkout --cmd.branch main diff --git a/docs/source/examples/10_multiple_subparsers.rst b/docs/source/examples/09_multiple_subparsers.rst similarity index 79% rename from docs/source/examples/10_multiple_subparsers.rst rename to docs/source/examples/09_multiple_subparsers.rst index ee2fd733..0af826b7 100644 --- a/docs/source/examples/10_multiple_subparsers.rst +++ b/docs/source/examples/09_multiple_subparsers.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -10. Multiple Subparsers +9. Multiple Subparsers ========================================== @@ -75,30 +75,30 @@ Multiple unions over nested types are populated using a series of subparsers. .. raw:: html - python 10_multiple_subparsers.py + python 09_multiple_subparsers.py -.. program-output:: python ../../examples/10_multiple_subparsers.py +.. program-output:: python ../../examples/09_multiple_subparsers.py ------------ .. raw:: html - python 10_multiple_subparsers.py --help + python 09_multiple_subparsers.py --help -.. program-output:: python ../../examples/10_multiple_subparsers.py --help +.. program-output:: python ../../examples/09_multiple_subparsers.py --help ------------ .. raw:: html - python 10_multiple_subparsers.py dataset:mnist --help + python 09_multiple_subparsers.py dataset:mnist --help -.. program-output:: python ../../examples/10_multiple_subparsers.py dataset:mnist --help +.. program-output:: python ../../examples/09_multiple_subparsers.py dataset:mnist --help ------------ .. raw:: html - python 10_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 + python 09_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 -.. program-output:: python ../../examples/10_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 +.. program-output:: python ../../examples/09_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 diff --git a/docs/source/examples/06_base_configs.rst b/docs/source/examples/10_base_configs.rst similarity index 65% rename from docs/source/examples/06_base_configs.rst rename to docs/source/examples/10_base_configs.rst index f28c0041..a78b030c 100644 --- a/docs/source/examples/06_base_configs.rst +++ b/docs/source/examples/10_base_configs.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -6. Base Configs +10. Base Configs ========================================== @@ -20,11 +20,11 @@ avoid fussing with ``sys.argv`` by using a ``BASE_CONFIG`` environment variable. .. code-block:: python :linenos: - import sys from dataclasses import dataclass - from typing import Callable, Dict, Literal, Tuple, TypeVar, Union + from typing import Callable, Literal, Mapping, Tuple, Type, TypeVar, Union from torch import nn + from typing_extensions import Annotated, reveal_type import dcargs @@ -95,71 +95,52 @@ avoid fussing with ``sys.argv`` by using a ``BASE_CONFIG`` environment variable. } - T = TypeVar("T") - - - def cli_from_base_configs(base_library: Dict[str, T]) -> T: - """Populate an instance of `cls`, where the first positional argument is used to - select from a library of named base configs.""" - # Get base configuration name from the first positional argument. - if len(sys.argv) < 2 or sys.argv[1] not in base_library: - valid_usages = map(lambda k: f"{sys.argv[0]} {k} --help", base_library.keys()) - raise SystemExit("usage:\n " + "\n ".join(valid_usages)) - - # Get base configuration from our library, and use it for default CLI parameters. - default_instance = base_library[sys.argv[1]] - return dcargs.cli( - type(default_instance), - prog=" ".join(sys.argv[:2]), - args=sys.argv[2:], - default_instance=default_instance, + if __name__ == "__main__": + config = dcargs.cli( + dcargs.extras.union_type_from_mapping(base_configs), # `avoid_subparsers` will avoid making a subparser for unions when a default is - # provided; in this case, it simplifies our CLI but makes it less expressive - # (cannot switch away from the base optimizer types). + # provided; it simplifies our CLI but makes it less expressive. avoid_subparsers=True, ) - - - if __name__ == "__main__": - config = cli_from_base_configs(base_configs) + reveal_type(config) # Should ExperimentConfig, both staticaly and dynamically. print(config) ------------ .. raw:: html - python 06_base_configs.py + python 10_base_configs.py -.. program-output:: python ../../examples/06_base_configs.py +.. program-output:: python ../../examples/10_base_configs.py ------------ .. raw:: html - python 06_base_configs.py small --help + python 10_base_configs.py small --help -.. program-output:: python ../../examples/06_base_configs.py small --help +.. program-output:: python ../../examples/10_base_configs.py small --help ------------ .. raw:: html - python 06_base_configs.py small --seed 94720 + python 10_base_configs.py small --seed 94720 -.. program-output:: python ../../examples/06_base_configs.py small --seed 94720 +.. program-output:: python ../../examples/10_base_configs.py small --seed 94720 ------------ .. raw:: html - python 06_base_configs.py big --help + python 10_base_configs.py big --help -.. program-output:: python ../../examples/06_base_configs.py big --help +.. program-output:: python ../../examples/10_base_configs.py big --help ------------ .. raw:: html - python 06_base_configs.py big --seed 94720 + python 10_base_configs.py big --seed 94720 -.. program-output:: python ../../examples/06_base_configs.py big --seed 94720 +.. program-output:: python ../../examples/10_base_configs.py big --seed 94720 diff --git a/examples/07_literals_and_unions.py b/examples/06_literals_and_unions.py similarity index 96% rename from examples/07_literals_and_unions.py rename to examples/06_literals_and_unions.py index b8231ae1..a46fff65 100644 --- a/examples/07_literals_and_unions.py +++ b/examples/06_literals_and_unions.py @@ -2,7 +2,7 @@ `typing.Union[]` can be used to restrict inputs to a fixed set of types. Usage: -`python ./07_literals_and_unions.py --help` +`python ./06_literals_and_unions.py --help` """ import dataclasses diff --git a/examples/08_positional_args.py b/examples/07_positional_args.py similarity index 93% rename from examples/08_positional_args.py rename to examples/07_positional_args.py index 713d2aab..317eccaa 100644 --- a/examples/08_positional_args.py +++ b/examples/07_positional_args.py @@ -1,8 +1,8 @@ """Positional-only arguments in functions are converted to positional CLI arguments. Usage: -`python ./08_positional_args.py --help` -`python ./08_positional_args.py ./a ./b --optimizer.learning-rate 1e-5` +`python ./07_positional_args.py --help` +`python ./07_positional_args.py ./a ./b --optimizer.learning-rate 1e-5` """ from __future__ import annotations diff --git a/examples/09_subparsers.py b/examples/08_subparsers.py similarity index 73% rename from examples/09_subparsers.py rename to examples/08_subparsers.py index aa0ec883..d0156da2 100644 --- a/examples/09_subparsers.py +++ b/examples/08_subparsers.py @@ -1,11 +1,11 @@ """Unions over nested types (classes or dataclasses) are populated using subparsers. Usage: -`python ./09_subparsers.py --help` -`python ./09_subparsers.py cmd:commit --help` -`python ./09_subparsers.py cmd:commit --cmd.message hello --cmd.all` -`python ./09_subparsers.py cmd:checkout --help` -`python ./09_subparsers.py cmd:checkout --cmd.branch main` +`python ./08_subparsers.py --help` +`python ./08_subparsers.py cmd:commit --help` +`python ./08_subparsers.py cmd:commit --cmd.message hello --cmd.all` +`python ./08_subparsers.py cmd:checkout --help` +`python ./08_subparsers.py cmd:checkout --cmd.branch main` """ from __future__ import annotations diff --git a/examples/10_multiple_subparsers.py b/examples/09_multiple_subparsers.py similarity index 84% rename from examples/10_multiple_subparsers.py rename to examples/09_multiple_subparsers.py index fd474660..512d61df 100644 --- a/examples/10_multiple_subparsers.py +++ b/examples/09_multiple_subparsers.py @@ -1,10 +1,10 @@ """Multiple unions over nested types are populated using a series of subparsers. Usage: -`python ./10_multiple_subparsers.py` -`python ./10_multiple_subparsers.py --help` -`python ./10_multiple_subparsers.py dataset:mnist --help` -`python ./10_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4` +`python ./09_multiple_subparsers.py` +`python ./09_multiple_subparsers.py --help` +`python ./09_multiple_subparsers.py dataset:mnist --help` +`python ./09_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4` """ from __future__ import annotations diff --git a/examples/06_base_configs.py b/examples/10_base_configs.py similarity index 66% rename from examples/06_base_configs.py rename to examples/10_base_configs.py index 0e721b5f..98a0cd37 100644 --- a/examples/06_base_configs.py +++ b/examples/10_base_configs.py @@ -9,18 +9,18 @@ avoid fussing with `sys.argv` by using a `BASE_CONFIG` environment variable. Usage: -`python ./06_base_configs.py` -`python ./06_base_configs.py small --help` -`python ./06_base_configs.py small --seed 94720` -`python ./06_base_configs.py big --help` -`python ./06_base_configs.py big --seed 94720` +`python ./10_base_configs.py` +`python ./10_base_configs.py small --help` +`python ./10_base_configs.py small --seed 94720` +`python ./10_base_configs.py big --help` +`python ./10_base_configs.py big --seed 94720` """ -import sys from dataclasses import dataclass -from typing import Callable, Dict, Literal, Tuple, TypeVar, Union +from typing import Callable, Literal, Mapping, Tuple, Type, TypeVar, Union from torch import nn +from typing_extensions import Annotated, reveal_type import dcargs @@ -91,31 +91,12 @@ class ExperimentConfig: } -T = TypeVar("T") - - -def cli_from_base_configs(base_library: Dict[str, T]) -> T: - """Populate an instance of `cls`, where the first positional argument is used to - select from a library of named base configs.""" - # Get base configuration name from the first positional argument. - if len(sys.argv) < 2 or sys.argv[1] not in base_library: - valid_usages = map(lambda k: f"{sys.argv[0]} {k} --help", base_library.keys()) - raise SystemExit("usage:\n " + "\n ".join(valid_usages)) - - # Get base configuration from our library, and use it for default CLI parameters. - default_instance = base_library[sys.argv[1]] - return dcargs.cli( - type(default_instance), - prog=" ".join(sys.argv[:2]), - args=sys.argv[2:], - default_instance=default_instance, +if __name__ == "__main__": + config = dcargs.cli( + dcargs.extras.union_type_from_mapping(base_configs), # `avoid_subparsers` will avoid making a subparser for unions when a default is - # provided; in this case, it simplifies our CLI but makes it less expressive - # (cannot switch away from the base optimizer types). + # provided; it simplifies our CLI but makes it less expressive. avoid_subparsers=True, ) - - -if __name__ == "__main__": - config = cli_from_base_configs(base_configs) + reveal_type(config) # Should ExperimentConfig, both staticaly and dynamically. print(config) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 2160c40b..eaee76bf 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -8,7 +8,7 @@ import pytest import torch.nn as nn -from typing_extensions import Literal +from typing_extensions import Annotated, Literal import dcargs import dcargs._argparse_formatter @@ -39,7 +39,7 @@ class Helptext: x: int # Documentation 1 # Documentation 2 - y: int + y: Annotated[int, "ignored"] z: int = 3 """Documentation 3""" @@ -525,9 +525,9 @@ def main(x: Any = Struct()): helptext = _get_helptext(main) assert "--x {fixed}" in helptext - def main(x: Callable = nn.ReLU): + def main2(x: Callable = nn.ReLU): pass - helptext = _get_helptext(main) + helptext = _get_helptext(main2) assert "--x {fixed}" in helptext assert "(fixed to: Date: Tue, 30 Aug 2022 07:22:49 -0700 Subject: [PATCH 02/19] Tests, improvements for generics --- dcargs/_arguments.py | 4 +- dcargs/_calling.py | 4 +- dcargs/_parsers.py | 11 ++- dcargs/_resolver.py | 29 +++++- dcargs/metadata/_subcommands.py | 3 + examples/10_base_configs.py | 6 +- tests/test_nested.py | 162 +++++++++++++++++++++++++++++++ tests/test_union_from_mapping.py | 48 +++++++++ 8 files changed, 250 insertions(+), 17 deletions(-) create mode 100644 tests/test_union_from_mapping.py diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index d10df3c6..dc9d64be 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -23,7 +23,7 @@ import termcolor -from . import _fields, _instantiators, _strings +from . import _fields, _instantiators, _resolver, _strings try: # Python >=3.8. @@ -132,7 +132,7 @@ def _rule_handle_boolean_flags( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, ) -> LoweredArgumentDefinition: - if arg.type_from_typevar.get(arg.field.typ, arg.field.typ) is not bool: # type: ignore + if _resolver.apply_type_from_typevar(arg.field.typ, arg.type_from_typevar) is not bool: # type: ignore return lowered if lowered.default is False and not arg.field.positional: diff --git a/dcargs/_calling.py b/dcargs/_calling.py index cfab0e48..be534d58 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -56,7 +56,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: prefixed_field_name = _strings.make_field_name([field_name_prefix, field.name]) # Resolve field type. - field_type = type_from_typevar.get(field.typ, field.typ) # type: ignore + field_type = _resolver.apply_type_from_typevar(field.typ, type_from_typevar) # type: ignore if prefixed_field_name in arg_from_prefixed_field_name: assert prefixed_field_name not in consumed_keywords @@ -144,7 +144,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: value = None else: options = map( - lambda x: type_from_typevar.get(x, x), + lambda x: _resolver.apply_type_from_typevar(x, type_from_typevar), get_args(field_type), ) chosen_f = None diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 5277d4a3..0a2642b9 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -80,9 +80,9 @@ def from_callable( field = dataclasses.replace( field, typ=_resolver.type_from_typevar_constraints( - type_from_typevar.get( # type: ignore - field.typ, + _resolver.apply_type_from_typevar( field.typ, + type_from_typevar, ) ), ) @@ -243,7 +243,10 @@ def from_field( return None # We don't use sets here to retain order of subcommands. - options = [type_from_typevar.get(typ, typ) for typ in get_args(field.typ)] + options = [ + _resolver.apply_type_from_typevar(typ, type_from_typevar) + for typ in get_args(field.typ) + ] options_no_none = [o for o in options if o != type(None)] # noqa if not all( [ @@ -283,7 +286,7 @@ def from_field( subparser = dataclasses.replace( subparser, helptext_from_nested_class_field_name={ - _strings.make_field_name([field.name, k]): v + _strings.make_field_name([prefix, k]): v for k, v in subparser.helptext_from_nested_class_field_name.items() }, ) diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index 88e3fe03..cbe9d8cc 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -1,5 +1,6 @@ """Utilities for resolving types and forward references.""" +import collections.abc import copy import dataclasses from typing import ( @@ -15,7 +16,7 @@ cast, ) -from typing_extensions import get_args, get_origin, get_type_hints +from typing_extensions import Annotated, get_args, get_origin, get_type_hints TypeOrCallable = TypeVar("TypeOrCallable", Type, Callable) @@ -43,13 +44,13 @@ def resolve_generic_types( origin_cls = get_origin(cls) - type_from_typevars = {} + type_from_typevar = {} if origin_cls is not None and hasattr(origin_cls, "__parameters__"): typevars = origin_cls.__parameters__ typevar_values = get_args(cls) assert len(typevars) == len(typevar_values) cls = origin_cls - type_from_typevars.update(dict(zip(typevars, typevar_values))) + type_from_typevar.update(dict(zip(typevars, typevar_values))) if hasattr(cls, "__orig_bases__"): bases = getattr(cls, "__orig_bases__") @@ -59,9 +60,9 @@ def resolve_generic_types( continue typevars = origin_base.__parameters__ typevar_values = get_args(base) - type_from_typevars.update(dict(zip(typevars, typevar_values))) + type_from_typevar.update(dict(zip(typevars, typevar_values))) - return cls, type_from_typevars + return cls, type_from_typevar def resolved_fields(cls: Type) -> List[dataclasses.Field]: @@ -154,3 +155,21 @@ def unwrap_annotated( len(targets) == 1 ), f"Found two instances of {search_type} in metadata, but only expected one." return args[0], targets[0] + + +def apply_type_from_typevar( + typ: TypeOrCallable, type_from_typevar: Dict[TypeVar, Type] +) -> TypeOrCallable: + if typ in type_from_typevar: + return type_from_typevar[typ] # type: ignore + + if len(get_args(typ)) > 0: + args = get_args(typ) + if get_origin(typ) is Annotated: + args = args[:1] + if get_origin(typ) is collections.abc.Callable: + assert isinstance(args[0], list) + args = tuple(args[0]) + args[1:] + return typ.copy_with(tuple(apply_type_from_typevar(x, type_from_typevar) for x in args)) # type: ignore + + return typ diff --git a/dcargs/metadata/_subcommands.py b/dcargs/metadata/_subcommands.py index eee8f2cb..c36b7f69 100644 --- a/dcargs/metadata/_subcommands.py +++ b/dcargs/metadata/_subcommands.py @@ -13,6 +13,9 @@ class _SubcommandConfiguration: description: Optional[str] default: Any + def __hash__(self) -> int: + return object.__hash__(self) + def subcommand( name: str, diff --git a/examples/10_base_configs.py b/examples/10_base_configs.py index 98a0cd37..d8f67c75 100644 --- a/examples/10_base_configs.py +++ b/examples/10_base_configs.py @@ -9,7 +9,7 @@ avoid fussing with `sys.argv` by using a `BASE_CONFIG` environment variable. Usage: -`python ./10_base_configs.py` +`python ./10_base_configs.py --help` `python ./10_base_configs.py small --help` `python ./10_base_configs.py small --seed 94720` `python ./10_base_configs.py big --help` @@ -17,10 +17,9 @@ """ from dataclasses import dataclass -from typing import Callable, Literal, Mapping, Tuple, Type, TypeVar, Union +from typing import Callable, Literal, Tuple, Union from torch import nn -from typing_extensions import Annotated, reveal_type import dcargs @@ -98,5 +97,4 @@ class ExperimentConfig: # provided; it simplifies our CLI but makes it less expressive. avoid_subparsers=True, ) - reveal_type(config) # Should ExperimentConfig, both staticaly and dynamically. print(config) diff --git a/tests/test_nested.py b/tests/test_nested.py index ffd366ef..22cbff7e 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -761,3 +761,165 @@ def main( assert hash(dcargs.cli(main, args="--x.num-epochs 10".split(" "))) == hash( frozendict({"num_epochs": 10, "batch_size": 64}) ) + + +def test_subparser_in_nested_with_metadata(): + @dataclasses.dataclass + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + @dataclasses.dataclass + class Nested2: + subcommand: Union[ + Annotated[A, dcargs.metadata.subcommand("command-a", default=A(7))], + Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], + ] + + @dataclasses.dataclass + class Nested1: + nested2: Nested2 + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-a".split(" "), + ) == Parent(Nested1(Nested2(A(7)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(A(3)))) + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-b".split(" "), + ) == Parent(Nested1(Nested2(B(9)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(B(7)))) + + +def test_subparser_in_nested_with_metadata_generic(): + @dataclasses.dataclass + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + T = TypeVar("T") + + @dataclasses.dataclass + class Nested2(Generic[T]): + subcommand: T + + @dataclasses.dataclass + class Nested1: + nested2: Nested2[ + Union[ + Annotated[A, dcargs.metadata.subcommand("command-a", default=A(7))], + Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], + ] + ] + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-a".split(" "), + ) == Parent(Nested1(Nested2(A(7)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(A(3)))) + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-b".split(" "), + ) == Parent(Nested1(Nested2(B(9)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(B(7)))) + + +def test_subparser_in_nested_with_metadata_generic_alt(): + @dataclasses.dataclass + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + T = TypeVar("T") + + @dataclasses.dataclass + class Nested2(Generic[T]): + subcommand: Union[ + Annotated[T, dcargs.metadata.subcommand("command-a", default=A(7))], + Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], + ] + + @dataclasses.dataclass + class Nested1: + nested2: Nested2[A] + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-a".split(" "), + ) == Parent(Nested1(Nested2(A(7)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(A(3)))) + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-b".split(" "), + ) == Parent(Nested1(Nested2(B(9)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(B(7)))) diff --git a/tests/test_union_from_mapping.py b/tests/test_union_from_mapping.py new file mode 100644 index 00000000..b82fd3f9 --- /dev/null +++ b/tests/test_union_from_mapping.py @@ -0,0 +1,48 @@ +import dataclasses +from typing import Optional + +import dcargs + + +@dataclasses.dataclass +class A: + x: int + + +def test_union_from_mapping(): + base_configs = { + "one": A(1), + "two": A(2), + "three": A(3), + } + ConfigUnion = dcargs.extras.union_type_from_mapping(base_configs) + + assert dcargs.cli(ConfigUnion, args="one".split(" ")) == A(1) + assert dcargs.cli(ConfigUnion, args="two".split(" ")) == A(2) + assert dcargs.cli(ConfigUnion, args="two --x 4".split(" ")) == A(4) + assert dcargs.cli(ConfigUnion, args="three".split(" ")) == A(3) + + +def test_union_from_mapping_in_function(): + base_configs = { + "one": A(1), + "two": A(2), + "three": A(3), + } + + # Hack for mypy. Not needed for pyright. + ConfigUnion = A + ConfigUnion = dcargs.extras.union_type_from_mapping(base_configs) # type: ignore + + def main(config: ConfigUnion, flag: bool = False) -> Optional[A]: + if flag: + return config + return None + + assert dcargs.cli(main, args="--flag config:one".split(" ")) == A(1) + assert dcargs.cli(main, args="--flag config:one --config.x 3".split(" ")) == A(3) + assert dcargs.cli(main, args="config:one --config.x 1".split(" ")) is None + + assert dcargs.cli(main, args="--flag config:two".split(" ")) == A(2) + assert dcargs.cli(main, args="--flag config:two --config.x 3".split(" ")) == A(3) + assert dcargs.cli(main, args="config:two --config.x 1".split(" ")) is None From f7dd1bdb721628c1c51174f1f6a018993cf531ad Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 31 Aug 2022 17:59:41 -0700 Subject: [PATCH 03/19] Minor stuff for shtab --- dcargs/_arguments.py | 5 ++++- dcargs/_parsers.py | 2 ++ dcargs/extras/_base_configs.py | 13 ++++++++++--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index dc9d64be..a015784b 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -175,7 +175,10 @@ def _rule_recursive_instantiator_from_type( ) except _instantiators.UnsupportedTypeAnnotationError as e: if arg.field.default in _fields.MISSING_SINGLETONS: - raise e + raise _instantiators.UnsupportedTypeAnnotationError( + "Unsupported type annotation for the field" + f" {_strings.make_field_name([arg.prefix, arg.field.name])}. To suppress this error, assign the field a default value." + ) from e else: # For fields with a default, we'll get by even if there's no instantiator # available. diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 0a2642b9..e76c715c 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -397,6 +397,7 @@ def apply( subparser = argparse_subparsers.add_parser( name=_strings.subparser_name_from_type(self.prefix, None), formatter_class=_argparse_formatter.make_formatter_class(0), + help="", ) subparser_tree_nodes.append(subparser) @@ -406,6 +407,7 @@ def apply( formatter_class=_argparse_formatter.make_formatter_class( len(subparser_def.args) ), + help=subparser_def.description, ) subparser_def.apply(subparser) diff --git a/dcargs/extras/_base_configs.py b/dcargs/extras/_base_configs.py index cd2bb5ee..7c58b84c 100644 --- a/dcargs/extras/_base_configs.py +++ b/dcargs/extras/_base_configs.py @@ -1,4 +1,4 @@ -from typing import Mapping, Type, TypeVar, Union +from typing import Mapping, Tuple, Type, TypeVar, Union from typing_extensions import Annotated @@ -7,7 +7,9 @@ T = TypeVar("T") -def union_type_from_mapping(base_mapping: Mapping[str, T]) -> Type[T]: +def union_type_from_mapping( + base_mapping: Mapping[Union[str, Tuple[str, str]], T] +) -> Type[T]: """Returns a Union type for defining subcommands that choose between nested types. For example, when `base_mapping` is set to: @@ -68,7 +70,12 @@ def train( return Union.__getitem__( # type: ignore tuple( Annotated.__class_getitem__( # type: ignore - (type(v), subcommand(k, default=v)) + ( + type(v), + subcommand(k, default=v) + if isinstance(k, str) + else subcommand(k[0], default=v, description=k[1]), + ) ) for k, v in base_mapping.items() ) From f445c895f420231d1805b3deeecb17d835e54375 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 31 Aug 2022 18:37:25 -0700 Subject: [PATCH 04/19] Fix bug for defaults set to dataclass types --- dcargs/_fields.py | 6 ++++-- tests/test_dcargs.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/dcargs/_fields.py b/dcargs/_fields.py index a61de72d..390192d7 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -539,7 +539,7 @@ def _ensure_dataclass_instance_used_as_default_is_frozen( frozen.""" assert dataclasses.is_dataclass(default_instance) cls = type(default_instance) - if not cls.__dataclass_params__.frozen: + if not cls.__dataclass_params__.frozen: # type: ignore warnings.warn( f"Mutable type {cls} is used as a default value for `{field.name}`. This is" " dangerous! Consider using `dataclasses.field(default_factory=...)` or" @@ -575,7 +575,9 @@ def _get_dataclass_field_default( # Try grabbing default from dataclass field. if field.default not in MISSING_SINGLETONS: default = field.default - if dataclasses.is_dataclass(default): + # Note that dataclasses.is_dataclass() will also return true for dataclass + # _types_, not just instances. + if type(default) is not type and dataclasses.is_dataclass(default): _ensure_dataclass_instance_used_as_default_is_frozen(field, default) return default diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index e01c86b6..6c474064 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -468,6 +468,19 @@ def main(x: Callable[[int], int] = lambda x: x * 2) -> Callable[[int], int]: dcargs.cli(main, args=["--x", "something"]) +def test_fixed_dataclass_type(): + @dataclasses.dataclass + class Dummy: + pass + + def main(x: Callable = Dummy) -> Callable: + return x + + assert dcargs.cli(main, args=[]) is Dummy + with pytest.raises(SystemExit): + dcargs.cli(main, args=["--x", "something"]) + + def test_missing_singleton(): assert dcargs.MISSING is copy.deepcopy(dcargs.MISSING) From 1e04ba39294ec8bce936f490edb8551fe60482f3 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 1 Sep 2022 00:54:04 -0700 Subject: [PATCH 05/19] Marker implementation: Fixed[], FlagsOff[], SubcommandsOff[], etc --- dcargs/_arguments.py | 37 +++- dcargs/_calling.py | 12 +- dcargs/_cli.py | 15 -- dcargs/_docstrings.py | 3 +- dcargs/_fields.py | 46 ++--- dcargs/_parsers.py | 83 +++++--- dcargs/_resolver.py | 29 ++- dcargs/_singleton.py | 13 ++ dcargs/_strings.py | 8 +- dcargs/extras/__init__.py | 4 +- dcargs/extras/_base_configs.py | 17 +- dcargs/metadata/__init__.py | 19 +- dcargs/metadata/_markers.py | 33 ++++ dcargs/metadata/_subcommands.py | 10 +- examples/10_base_configs.py | 88 ++++++--- tests/test_helptext.py | 4 - tests/test_metadata.py | 330 +++++++++++++++++++++++++++++++ tests/test_nested.py | 215 -------------------- tests/test_union_from_mapping.py | 4 +- 19 files changed, 581 insertions(+), 389 deletions(-) create mode 100644 dcargs/_singleton.py create mode 100644 dcargs/metadata/_markers.py create mode 100644 tests/test_metadata.py diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index a015784b..a1aca8f8 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -24,6 +24,7 @@ import termcolor from . import _fields, _instantiators, _resolver, _strings +from .metadata import _markers try: # Python >=3.8. @@ -95,9 +96,9 @@ class LoweredArgumentDefinition: def is_fixed(self) -> bool: """If the instantiator is set to `None`, even after all argument - transformations, it means that we weren't able to determine a valid instantiator - for an argument. We then mark the argument as 'fixed', with a value always equal - to the field default.""" + transformations, it means that we don't have a valid instantiator for an + argument. We then mark the argument as 'fixed', with a value always equal to the + field default.""" return self.instantiator is None # From here on out, all fields correspond 1:1 to inputs to argparse's @@ -135,23 +136,32 @@ def _rule_handle_boolean_flags( if _resolver.apply_type_from_typevar(arg.field.typ, arg.type_from_typevar) is not bool: # type: ignore return lowered - if lowered.default is False and not arg.field.positional: + if ( + arg.field.default in _fields.MISSING_SINGLETONS + or arg.field.positional + or _markers.FLAGS_OFF in arg.field.markers + ): + # Treat bools as a normal parameter. + return lowered + elif arg.field.default is False: # Default `False` => --flag passed in flips to `True`. return dataclasses.replace( lowered, action="store_true", instantiator=lambda x: x, # argparse will directly give us a bool! ) - elif lowered.default is True and not arg.field.positional: + elif arg.field.default is True: # Default `True` => --no-flag passed in flips to `False`. return dataclasses.replace( lowered, action="store_false", instantiator=lambda x: x, # argparse will directly give us a bool! ) - else: - # Treat bools as a normal parameter. - return lowered + + assert False, ( + "Expected a boolean as a default for {arg.field.name}, but got" + " {lowered.default}." + ) def _rule_recursive_instantiator_from_type( @@ -166,6 +176,14 @@ def _rule_recursive_instantiator_from_type( Conversions from strings to our desired types happen in the instantiator; this is a bit more flexible, and lets us handle more complex types like enums and multi-type tuples.""" + if _markers.FIXED in arg.field.markers: + return dataclasses.replace( + lowered, + instantiator=None, + metavar=termcolor.colored("{fixed}", color="red"), + required=False, + default=_fields.MISSING_PROP, + ) if lowered.instantiator is not None: return lowered try: @@ -177,7 +195,8 @@ def _rule_recursive_instantiator_from_type( if arg.field.default in _fields.MISSING_SINGLETONS: raise _instantiators.UnsupportedTypeAnnotationError( "Unsupported type annotation for the field" - f" {_strings.make_field_name([arg.prefix, arg.field.name])}. To suppress this error, assign the field a default value." + f" {_strings.make_field_name([arg.prefix, arg.field.name])}. To" + " suppress this error, assign the field a default value." ) from e else: # For fields with a default, we'll get by even if there's no instantiator diff --git a/dcargs/_calling.py b/dcargs/_calling.py index be534d58..d20656ca 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -5,7 +5,7 @@ from typing import Any, Callable, Dict, List, Sequence, Set, Tuple, TypeVar, Union -from typing_extensions import get_args, get_origin +from typing_extensions import get_args from . import _arguments, _fields, _parsers, _resolver, _strings @@ -24,7 +24,6 @@ def call_from_args( default_instance: Union[T, _fields.NonpropagatingMissingType], value_from_prefixed_field_name: Dict[str, Any], field_name_prefix: str, - avoid_subparsers: bool, ) -> Tuple[T, Set[str]]: """Call `f` with arguments specified by a dictionary of values from argparse. @@ -96,8 +95,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: in parser_definition.helptext_from_nested_class_field_name ): # Nested callable. - if get_origin(field_type) is Union: - assert avoid_subparsers + if _resolver.unwrap_origin_strip_extras(field_type) is Union: field_type = type(field.default) value, consumed_keywords_child = call_from_args( field_type, @@ -105,7 +103,6 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: field.default, value_from_prefixed_field_name, field_name_prefix=prefixed_field_name, - avoid_subparsers=avoid_subparsers, ) consumed_keywords |= consumed_keywords_child else: @@ -145,7 +142,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: else: options = map( lambda x: _resolver.apply_type_from_typevar(x, type_from_typevar), - get_args(field_type), + get_args(_resolver.unwrap_annotated(field_type)[0]), ) chosen_f = None for option in options: @@ -162,7 +159,6 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: field.default if type(field.default) is chosen_f else None, value_from_prefixed_field_name, field_name_prefix=prefixed_field_name, - avoid_subparsers=avoid_subparsers, ) consumed_keywords |= consumed_keywords_child @@ -174,7 +170,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: field.name if field.name_override is None else field.name_override ] = value - unwrapped_f = _resolver.unwrap_origin(f) + unwrapped_f = _resolver.unwrap_origin_strip_extras(f) unwrapped_f = list if unwrapped_f is Sequence else unwrapped_f # type: ignore unwrapped_f = _resolver.narrow_type(unwrapped_f, default_instance) if unwrapped_f in (tuple, list, set): diff --git a/dcargs/_cli.py b/dcargs/_cli.py index ee40c530..1376bce8 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -27,7 +27,6 @@ def cli( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[OutT] = None, - avoid_subparsers: bool = False, ) -> OutT: ... @@ -40,7 +39,6 @@ def cli( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[OutT] = None, - avoid_subparsers: bool = False, ) -> OutT: ... @@ -52,7 +50,6 @@ def cli( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[OutT] = None, - avoid_subparsers: bool = False, ) -> OutT: """Call `f(...)`, with arguments populated from an automatically generated CLI interface. @@ -101,8 +98,6 @@ def cli( if `T` is a dataclass, TypedDict, or NamedTuple. Helpful for merging CLI arguments with values loaded from elsewhere. (for example, a config object loaded from a yaml file) - avoid_subparsers: Avoid creating a subparser when defaults are provided for - unions over nested types. Generates cleaner but less expressive CLIs. Returns: The output of `f(...)`. @@ -114,7 +109,6 @@ def cli( description=description, args=args, default_instance=default_instance, - avoid_subparsers=avoid_subparsers, ) return out @@ -127,7 +121,6 @@ def generate_parser( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[OutT] = None, - avoid_subparsers: bool = False, ) -> argparse.ArgumentParser: ... @@ -140,7 +133,6 @@ def generate_parser( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[OutT] = None, - avoid_subparsers: bool = False, ) -> argparse.ArgumentParser: ... @@ -152,7 +144,6 @@ def generate_parser( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[OutT] = None, - avoid_subparsers: bool = False, ) -> argparse.ArgumentParser: """Returns the argparse parser that would be used under-the-hood if `dcargs.cli()` was called with the same arguments. @@ -166,7 +157,6 @@ def generate_parser( description=description, args=args, default_instance=default_instance, - avoid_subparsers=avoid_subparsers, ) assert isinstance(out, argparse.ArgumentParser) return out @@ -181,7 +171,6 @@ def _cli_impl( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[OutT] = None, - avoid_subparsers: bool = False, ) -> argparse.ArgumentParser: ... @@ -195,7 +184,6 @@ def _cli_impl( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[OutT] = None, - avoid_subparsers: bool = False, ) -> OutT: ... @@ -208,7 +196,6 @@ def _cli_impl( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[OutT] = None, - avoid_subparsers: bool = False, ) -> Union[OutT, argparse.ArgumentParser]: default_instance_internal: Union[_fields.NonpropagatingMissingType, OutT] = ( _fields.MISSING_NONPROP if default_instance is None else default_instance @@ -239,7 +226,6 @@ def _cli_impl( parent_type_from_typevar=None, # Used for recursive calls. default_instance=default_instance_internal, # Overrides for default values. prefix="", # Used for recursive calls. - avoid_subparsers=avoid_subparsers, ) # Generate parser! @@ -269,7 +255,6 @@ def _cli_impl( default_instance_internal, value_from_prefixed_field_name, field_name_prefix="", - avoid_subparsers=avoid_subparsers, ) except _calling.InstantiationError as e: # Emulate argparse's error behavior when invalid arguments are passed in. diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index dcca98ad..492620a6 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -254,7 +254,8 @@ def get_callable_description(f: Callable) -> str: these docstrings.""" f, _unused = _resolver.resolve_generic_types(f) - if _resolver.unwrap_origin(f) in _callable_description_blocklist: + f = _resolver.unwrap_origin_strip_extras(f) + if f in _callable_description_blocklist: return "" # Note inspect.getdoc() causes some corner cases with TypedDicts. diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 390192d7..6c5169fa 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -17,7 +17,8 @@ import typing_extensions from typing_extensions import get_args, get_type_hints, is_typeddict -from . import _docstrings, _instantiators, _resolver, _strings +from . import _docstrings, _instantiators, _resolver, _singleton, _strings +from .metadata import _markers @dataclasses.dataclass(frozen=True) @@ -27,37 +28,32 @@ class FieldDefinition: default: Any helptext: Optional[str] positional: bool + markers: List[_markers.Marker] = dataclasses.field(default_factory=list) # Override the name in our kwargs. Currently only used for dictionary types when # the key values aren't strings, but in the future could be used whenever the # user-facing argument name doesn't match the keyword expected by our callable. name_override: Optional[Any] = None - -class _Singleton: - # Singleton pattern. - # https://www.python.org/download/releases/2.2/descrintro/#__new__ - def __new__(cls, *args, **kwds): - it = cls.__dict__.get("__it__") - if it is not None: - return it - cls.__it__ = it = object.__new__(cls) - it.init(*args, **kwds) - return it - - def init(self, *args, **kwds): - pass + def __post_init__(self): # + # Auto-populate markers if unset; this is meant to not run when we do + # dataclasses.replace, etc. TODO: the whole markers design, handling of + # Annotated[], etc, should be revisited... + if len(self.markers) == 0: + self.markers.extend( + _resolver.unwrap_annotated(self.typ, _markers.Marker)[1] + ) -class PropagatingMissingType(_Singleton): +class PropagatingMissingType(_singleton.Singleton): pass -class NonpropagatingMissingType(_Singleton): +class NonpropagatingMissingType(_singleton.Singleton): pass -class ExcludeFromCallType(_Singleton): +class ExcludeFromCallType(_singleton.Singleton): pass @@ -147,11 +143,11 @@ def _try_field_list_from_callable( ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: from . import metadata as _metadata - f, subcommand_config = _resolver.unwrap_annotated( + f, found_subcommand_configs = _resolver.unwrap_annotated( f, _metadata._subcommands._SubcommandConfiguration ) - if subcommand_config is not None: - default_instance = subcommand_config.default + if len(found_subcommand_configs) > 0: + default_instance = found_subcommand_configs[0].default # Unwrap generics. f, type_from_typevar = _resolver.resolve_generic_types(f) @@ -165,7 +161,7 @@ def _try_field_list_from_callable( cls = f f = cls.__init__ # type: ignore f_origin: Callable = cls # type: ignore - f_origin = _resolver.unwrap_origin(f) + f_origin = _resolver.unwrap_origin_strip_extras(f) # Try special cases. if cls is not None and is_typeddict(cls): @@ -224,9 +220,9 @@ def _try_field_list_from_callable( return container_fields # General cases. - if (cls is not None and cls in _known_parsable_types) or _resolver.unwrap_origin( - f - ) in _known_parsable_types: + if ( + cls is not None and cls in _known_parsable_types + ) or _resolver.unwrap_origin_strip_extras(f) in _known_parsable_types: return UnsupportedNestedTypeMessage(f"{f} should be parsed directly!") else: return _try_field_list_from_general_callable(f, cls, default_instance) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index e76c715c..9cca1d71 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -43,11 +43,11 @@ def from_callable( T, _fields.PropagatingMissingType, _fields.NonpropagatingMissingType ], prefix: str, - avoid_subparsers: bool, ) -> ParserSpecification: """Create a parser definition from a callable.""" # Resolve generic types. + markers = _resolver.unwrap_annotated(f, _metadata._markers.Marker)[1] f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.narrow_type(f, default_instance) if parent_type_from_typevar is not None: @@ -86,36 +86,56 @@ def from_callable( ) ), ) + field.markers.extend(markers) # TODO: would be nice to avoid this mutation! + if isinstance(field.typ, TypeVar): raise _instantiators.UnsupportedTypeAnnotationError( f"Field {field.name} has an unbound TypeVar: {field.typ}." ) - # (1) Handle Unions over callables; these result in subparsers. + # (1) Handle fields marked as fixed. + if _metadata._markers.FIXED in field.markers: + args.append( + _arguments.ArgumentDefinition( + prefix=prefix, + field=field, + type_from_typevar=type_from_typevar, + ) + ) + continue + + # (2) Handle Unions over callables; these result in subparsers. subparsers_attempt = SubparsersSpecification.from_field( field, type_from_typevar=type_from_typevar, parent_classes=parent_classes, prefix=_strings.make_field_name([prefix, field.name]), - avoid_subparsers=avoid_subparsers, ) if subparsers_attempt is not None: if ( - not avoid_subparsers - # Required subparsers => must create a subparser. - or subparsers_attempt.required + not subparsers_attempt.required + and _metadata._markers.SUBCOMMANDS_OFF in field.markers ): + # Don't make a subparser. + field = dataclasses.replace(field, typ=type(field.default)) + else: subparsers_from_name[ _strings.make_field_name([prefix, subparsers_attempt.name]) ] = subparsers_attempt + + for subparser_def in subparsers_attempt.parser_from_name.values(): + for arg in subparser_def.args: + arg.field.markers.extend(field.markers) continue - else: - field = dataclasses.replace(field, typ=type(field.default)) - # (2) Handle nested callables. + # (3) Handle nested callables. if _fields.is_nested_type(field.typ, field.default): field = dataclasses.replace( - field, typ=_resolver.narrow_type(field.typ, field.default) + field, + typ=_resolver.narrow_type( + field.typ, + field.default, + ), ) nested_parser = ParserSpecification.from_callable( field.typ, @@ -124,10 +144,12 @@ def from_callable( parent_type_from_typevar=type_from_typevar, default_instance=field.default, prefix=_strings.make_field_name([prefix, field.name]), - avoid_subparsers=avoid_subparsers, ) args.extend(nested_parser.args) + for arg in nested_parser.args: + arg.field.markers.extend(field.markers) + # Include nested subparsers. subparsers_from_name.update(nested_parser.subparsers_from_name) @@ -145,10 +167,12 @@ def from_callable( ] = _docstrings.get_callable_description(field.typ) continue - # (3) Handle primitive types. These produce a single argument! + # (4) Handle primitive types. These produce a single argument! args.append( _arguments.ArgumentDefinition( - prefix=prefix, field=field, type_from_typevar=type_from_typevar + prefix=prefix, + field=field, + type_from_typevar=type_from_typevar, ) ) @@ -236,16 +260,16 @@ def from_field( type_from_typevar: Dict[TypeVar, Type], parent_classes: Set[Type], prefix: str, - avoid_subparsers: bool, ) -> Optional[SubparsersSpecification]: # Union of classes should create subparsers. - if get_origin(field.typ) is not Union: + typ = _resolver.unwrap_annotated(field.typ)[0] + if get_origin(typ) is not Union: return None # We don't use sets here to retain order of subcommands. options = [ _resolver.apply_type_from_typevar(typ, type_from_typevar) - for typ in get_args(field.typ) + for typ in get_args(typ) ] options_no_none = [o for o in options if o != type(None)] # noqa if not all( @@ -260,28 +284,31 @@ def from_field( parser_from_name: Dict[str, ParserSpecification] = {} for option in options_no_none: name = _strings.subparser_name_from_type(prefix, option) - option, subcommand_config = _resolver.unwrap_annotated( + option, found_subcommand_configs = _resolver.unwrap_annotated( option, _metadata._subcommands._SubcommandConfiguration ) - if subcommand_config is None: - subcommand_config = _metadata._subcommands._SubcommandConfiguration( - "unused", - description=None, - default=( - field.default - if type(field.default) is _resolver.unwrap_origin(option) - else _fields.MISSING_NONPROP + if len(found_subcommand_configs) == 0: + # Make a dummy subcommand config. + found_subcommand_configs = ( + _metadata._subcommands._SubcommandConfiguration( + "unused", + description=None, + default=( + field.default + if type(field.default) + is _resolver.unwrap_origin_strip_extras(option) + else _fields.MISSING_NONPROP + ), ), ) subparser = ParserSpecification.from_callable( option, - description=subcommand_config.description, + description=found_subcommand_configs[0].description, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, - default_instance=subcommand_config.default, + default_instance=found_subcommand_configs[0].default, prefix=prefix, - avoid_subparsers=avoid_subparsers, ) subparser = dataclasses.replace( subparser, diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index cbe9d8cc..46489672 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -21,8 +21,9 @@ TypeOrCallable = TypeVar("TypeOrCallable", Type, Callable) -def unwrap_origin(typ: TypeOrCallable) -> TypeOrCallable: - """Returns the origin of typ if it exists. Otherwise, returns typ.""" +def unwrap_origin_strip_extras(typ: TypeOrCallable) -> TypeOrCallable: + """Returns the origin, ignoring typing.Annotated, of typ if it exists. Otherwise, returns typ.""" + # TODO: Annotated[] handling should be revisited... typ, _ = unwrap_annotated(typ) origin = get_origin(typ) if origin is None: @@ -33,7 +34,7 @@ def unwrap_origin(typ: TypeOrCallable) -> TypeOrCallable: def is_dataclass(cls: Union[Type, Callable]) -> bool: """Same as `dataclasses.is_dataclass`, but also handles generic aliases.""" - return dataclasses.is_dataclass(unwrap_origin(cls)) + return dataclasses.is_dataclass(unwrap_origin_strip_extras(cls)) def resolve_generic_types( @@ -55,7 +56,7 @@ def resolve_generic_types( if hasattr(cls, "__orig_bases__"): bases = getattr(cls, "__orig_bases__") for base in bases: - origin_base = unwrap_origin(base) + origin_base = unwrap_origin_strip_extras(base) if origin_base is base or not hasattr(origin_base, "__parameters__"): continue typevars = origin_base.__parameters__ @@ -115,8 +116,12 @@ def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: should parse as Cat.""" try: potential_subclass = type(default_instance) - superclass = typ + superclass = unwrap_annotated(typ)[0] if superclass is Any or issubclass(potential_subclass, superclass): # type: ignore + if get_origin(typ) is Annotated: + return Annotated.__class_getitem__( # type: ignore + (potential_subclass,) + get_args(typ)[1:] + ) return cast(TypeT, potential_subclass) except TypeError: pass @@ -128,7 +133,7 @@ def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: def unwrap_annotated( typ: TypeOrCallable, search_type: Optional[Type[MetadataType]] = None -) -> Tuple[TypeOrCallable, Optional[MetadataType]]: +) -> Tuple[TypeOrCallable, Tuple[MetadataType, ...]]: """Helper for parsing typing.Annotated types. Examples: @@ -137,24 +142,18 @@ def unwrap_annotated( - Annotated[int, "1"], int => (int, None) """ if not hasattr(typ, "__metadata__"): - return typ, None + return typ, () args = get_args(typ) assert len(args) >= 2 # Don't search for a specific metadata type if `None` is passed in. if search_type is None: - return args[0], None + return args[0], () # Look through metadata for desired metadata type. targets = tuple(x for x in args[1:] if isinstance(x, search_type)) - if len(targets) == 0: - return args[0], None - else: - assert ( - len(targets) == 1 - ), f"Found two instances of {search_type} in metadata, but only expected one." - return args[0], targets[0] + return args[0], targets def apply_type_from_typevar( diff --git a/dcargs/_singleton.py b/dcargs/_singleton.py new file mode 100644 index 00000000..f52efefd --- /dev/null +++ b/dcargs/_singleton.py @@ -0,0 +1,13 @@ +class Singleton: + # Singleton pattern. + # https://www.python.org/download/releases/2.2/descrintro/#__new__ + def __new__(cls, *args, **kwds): + it = cls.__dict__.get("__it__") + if it is not None: + return it + cls.__it__ = it = object.__new__(cls) + it.init(*args, **kwds) + return it + + def init(self, *args, **kwds): + pass diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 1cc781ee..7b2ddf14 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -57,13 +57,13 @@ def _subparser_name_from_type(cls: Type) -> str: from .metadata import _subcommands # Prevent circular imports cls, type_from_typevar = _resolver.resolve_generic_types(cls) - cls, subcommand_config = _resolver.unwrap_annotated( + cls, found_subcommand_configs = _resolver.unwrap_annotated( cls, _subcommands._SubcommandConfiguration ) # Subparser name from `dcargs.metadata.subcommand()`. - if subcommand_config is not None: - return subcommand_config.name + if len(found_subcommand_configs) > 0: + return found_subcommand_configs[0].name # Subparser name from class name. if len(type_from_typevar) == 0: @@ -82,7 +82,7 @@ def subparser_name_from_type(prefix: str, cls: Union[Type, None]) -> str: suffix = _subparser_name_from_type(cls) if cls is not None else "None" if len(prefix) == 0: return suffix - return f"{prefix}:{suffix}" + return f"{prefix}:{suffix}".replace("_", "-") @functools.lru_cache(maxsize=None) diff --git a/dcargs/extras/__init__.py b/dcargs/extras/__init__.py index dd38ce86..9671a957 100644 --- a/dcargs/extras/__init__.py +++ b/dcargs/extras/__init__.py @@ -1,7 +1,7 @@ """The :mod:`dcargs.extras` submodule contains helpers that complement :func:`dcargs.cli()`, but aren't considered part of the core interface.""" -from ._base_configs import union_type_from_mapping +from ._base_configs import subcommand_union_from_mapping from ._serialization import from_yaml, to_yaml -__all__ = ["union_type_from_mapping", "to_yaml", "from_yaml"] +__all__ = ["subcommand_union_from_mapping", "to_yaml", "from_yaml"] diff --git a/dcargs/extras/_base_configs.py b/dcargs/extras/_base_configs.py index 7c58b84c..3a452888 100644 --- a/dcargs/extras/_base_configs.py +++ b/dcargs/extras/_base_configs.py @@ -1,4 +1,4 @@ -from typing import Mapping, Tuple, Type, TypeVar, Union +from typing import Mapping, Type, TypeVar, Union from typing_extensions import Annotated @@ -7,8 +7,8 @@ T = TypeVar("T") -def union_type_from_mapping( - base_mapping: Mapping[Union[str, Tuple[str, str]], T] +def subcommand_union_from_mapping( + defaults: Mapping[str, T], descriptions: Mapping[str, str] = {} ) -> Type[T]: """Returns a Union type for defining subcommands that choose between nested types. @@ -43,7 +43,7 @@ def union_type_from_mapping( reveal_type(config) # Should be correct! ``` - Or to generate annotations for functions: + Or to generate annotations for classes and functions: ```python SelectableConfig = union_from_base_mapping(base_mapping) @@ -70,13 +70,8 @@ def train( return Union.__getitem__( # type: ignore tuple( Annotated.__class_getitem__( # type: ignore - ( - type(v), - subcommand(k, default=v) - if isinstance(k, str) - else subcommand(k[0], default=v, description=k[1]), - ) + (type(v), subcommand(k, default=v, description=descriptions.get(k))) ) - for k, v in base_mapping.items() + for k, v in defaults.items() ) ) diff --git a/dcargs/metadata/__init__.py b/dcargs/metadata/__init__.py index 6f2156b9..9324a808 100644 --- a/dcargs/metadata/__init__.py +++ b/dcargs/metadata/__init__.py @@ -1,16 +1,15 @@ """The :mod:`dcargs.metadata` submodule contains helpers for attaching parsing-specific -metadata to types. For the forseeable future, this is limited to subcommand configuration. +metadata to types. -Features here are supported, but generally contradict the core design ethos of -:func:`dcargs.cli()`. - -As such: -1. Usage of existing functionality should be avoided unless absolutely necessary. -2. Introduction of new functionality should be avoided unless it (a) cannot be - reproduced with standard type annotations and (b) meaningfully improves the - usefulness of the library. +Features here are supported, but generally should be unnecessary. """ +from ._markers import Fixed, FlagsOff, SubcommandsOff from ._subcommands import subcommand -__all__ = ["subcommand"] +__all__ = [ + "Fixed", + "FlagsOff", + "SubcommandsOff", + "subcommand", +] diff --git a/dcargs/metadata/_markers.py b/dcargs/metadata/_markers.py new file mode 100644 index 00000000..0835bad3 --- /dev/null +++ b/dcargs/metadata/_markers.py @@ -0,0 +1,33 @@ +from typing import Type, TypeVar + +from typing_extensions import Annotated + +from .. import _singleton + + +class Marker(_singleton.Singleton): + pass + + +def _make_marker(description: str) -> Marker: + class _InnerMarker(Marker): + def __repr__(self): + return description + + return _InnerMarker() + + +T = TypeVar("T", bound=Type) + +FIXED = _make_marker("Fixed") +Fixed = Annotated[T, FIXED] +"""A type T can be annotated as Fixed[T] if we don't want dcargs to parse it. A default +value should be set instead.""" + +FLAGS_OFF = _make_marker("FlagsOff") +FlagsOff = Annotated[T, FLAGS_OFF] +"""Turn off flag conversion, which.""" + +SUBCOMMANDS_OFF = _make_marker("SubcommandsOff") +SubcommandsOff = Annotated[T, SUBCOMMANDS_OFF] +"""A boolean type can be annotated as NoFlag[bool] to turn off automatic flag generation.""" diff --git a/dcargs/metadata/_subcommands.py b/dcargs/metadata/_subcommands.py index c36b7f69..104e5524 100644 --- a/dcargs/metadata/_subcommands.py +++ b/dcargs/metadata/_subcommands.py @@ -6,9 +6,6 @@ @dataclasses.dataclass(frozen=True) class _SubcommandConfiguration: - # Things we could potentially add: - # - `description` - # - `avoid_subparsers` name: str description: Optional[str] default: Any @@ -24,9 +21,7 @@ def subcommand( default: Any = MISSING_NONPROP, ) -> Any: """Returns a metadata object for configuring subcommands with `typing.Annotated`. - Use of this function is supported but discouraged unless absolutely necessary. - - --- + This is useful but can make code harder to read, so usage is discouraged. Consider the standard approach for creating subcommands: @@ -36,8 +31,7 @@ def subcommand( ) ``` - This will create two subcommands: nested-type-a and nested-type-b. - + This will create two subcommands: `nested-type-a` and `nested-type-b`. Annotating each type with `dcargs.metadata.subcommand()` allows us to override for each subcommand the (a) name and (b) defaults. diff --git a/examples/10_base_configs.py b/examples/10_base_configs.py index d8f67c75..c85b72e8 100644 --- a/examples/10_base_configs.py +++ b/examples/10_base_configs.py @@ -1,9 +1,9 @@ """We can integrate `dcargs.cli()` into common configuration patterns: here, we select -one of multiple possible base configurations, and then use the CLI to either override -(existing) or fill in (missing) values. +one of multiple possible base configurations, create a subcommand for each one, and then +use the CLI to either override (existing) or fill in (missing) values. -Note that our interfaces don't prescribe any of the mechanics used for storing or -choosing between base configurations. A Hydra-style YAML approach could just as easily +Note that our interfaces don't prescribe any of the mechanics used for storing +base configurations. A Hydra-style YAML approach could just as easily be used for the config libary (although we generally prefer to avoid YAMLs; staying in Python is convenient for autocompletion and type checking). For selection, we could also avoid fussing with `sys.argv` by using a `BASE_CONFIG` environment variable. @@ -64,37 +64,61 @@ class ExperimentConfig: # Note that we could also define this library using separate YAML files (similar to # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type # checking + IDE support. -base_configs = { - "small": ExperimentConfig( - dataset="mnist", - optimizer=SgdOptimizer(), - batch_size=2048, - num_layers=4, - units=64, - train_steps=30_000, - # The dcargs.MISSING sentinel allows us to specify that the seed should have no - # default, and needs to be populated from the CLI. - seed=dcargs.MISSING, - activation=nn.ReLU, - ), - "big": ExperimentConfig( - dataset="imagenet-50", - optimizer=AdamOptimizer(), - batch_size=32, - num_layers=8, - units=256, - train_steps=100_000, - seed=dcargs.MISSING, - activation=nn.GELU, - ), -} +descriptions = {} +base_configs = {} + +descriptions["small"] = "Train a smaller model." +base_configs["small"] = ExperimentConfig( + dataset="mnist", + optimizer=SgdOptimizer(), + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + # The dcargs.MISSING sentinel allows us to specify that the seed should have no + # default, and needs to be populated from the CLI. + seed=dcargs.MISSING, + activation=nn.ReLU, +) + + +descriptions["big"] = "Train a bigger model." +base_configs["big"] = ExperimentConfig( + dataset="imagenet-50", + optimizer=AdamOptimizer(), + batch_size=32, + num_layers=8, + units=256, + train_steps=100_000, + seed=dcargs.MISSING, + activation=nn.GELU, +) if __name__ == "__main__": config = dcargs.cli( - dcargs.extras.union_type_from_mapping(base_configs), - # `avoid_subparsers` will avoid making a subparser for unions when a default is - # provided; it simplifies our CLI but makes it less expressive. - avoid_subparsers=True, + dcargs.extras.subcommand_union_from_mapping(base_configs, descriptions), ) + # Note that this is equivalent to: + # + # config = dcargs.cli( + # Union[ + # Annotated[ + # ExperimentConfig, + # dcargs.metadata.subcommand( + # "small", + # default=base_configs["small"], + # description=descriptions["small"], + # ), + # ], + # Annotated[ + # ExperimentConfig, + # dcargs.metadata.subcommand( + # "big", + # default=base_configs["big"], + # description=descriptions["big"], + # ), + # ], + # ] + # ) print(config) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index eaee76bf..485b70a1 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -104,8 +104,6 @@ def some_method(self) -> None: # noqa class ChildClass(ParentClass): """This docstring should be printed as a description.""" - pass - def main(x: ParentClass = ChildClass(x=5, y=5)) -> Any: return x @@ -127,7 +125,6 @@ def __init__(self, a: int): Args: a (int): Hello world! """ - pass def main_with_docstring(a: Inner) -> None: """main_with_docstring. @@ -138,7 +135,6 @@ def main_with_docstring(a: Inner) -> None: def main_no_docstring(a: Inner) -> None: """main_no_docstring.""" - pass helptext = _get_helptext(main_with_docstring) assert "Documented in function" in helptext and str(Inner.__doc__) not in helptext diff --git a/tests/test_metadata.py b/tests/test_metadata.py new file mode 100644 index 00000000..ca653cb7 --- /dev/null +++ b/tests/test_metadata.py @@ -0,0 +1,330 @@ +import dataclasses +from typing import Generic, TypeVar, Union + +import pytest +from typing_extensions import Annotated + +import dcargs + + +def test_avoid_subparser_with_default_instance(): + @dataclasses.dataclass + class DefaultInstanceHTTPServer: + y: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSMTPServer: + z: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSubparser: + x: int + bc: dcargs.metadata.SubcommandsOff[ + Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + ] + + assert ( + dcargs.cli( + DefaultInstanceSubparser, + args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "5"], + ) + == dcargs.cli( + DefaultInstanceSubparser, + args=["--x", "1", "--bc.y", "5"], + default_instance=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=3) + ), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) + ) + assert ( + dcargs.cli( + DefaultInstanceSubparser, + args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "8"], + ) + == dcargs.cli( + DefaultInstanceSubparser, + args=["--bc.y", "8"], + default_instance=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=7) + ), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) + ) + + +def test_avoid_subparser_with_default_instance_recursive(): + @dataclasses.dataclass + class DefaultInstanceHTTPServer: + y: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSMTPServer: + z: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSubparser: + x: int + bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + + assert ( + dcargs.cli( + DefaultInstanceSubparser, + args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "5"], + ) + == dcargs.cli( + dcargs.metadata.SubcommandsOff[DefaultInstanceSubparser], + args=["--x", "1", "--bc.y", "5"], + default_instance=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=3) + ), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) + ) + assert dcargs.cli( + DefaultInstanceSubparser, + args=["bc:default-instance-smtp-server", "--bc.z", "3"], + default_instance=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=5) + ), + ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceSMTPServer(z=3)) + assert ( + dcargs.cli( + dcargs.metadata.SubcommandsOff[DefaultInstanceSubparser], + args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "8"], + ) + == dcargs.cli( + dcargs.metadata.SubcommandsOff[DefaultInstanceSubparser], + args=["--bc.y", "8"], + default_instance=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=7) + ), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) + ) + + +def test_subparser_in_nested_with_metadata(): + @dataclasses.dataclass + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + @dataclasses.dataclass + class Nested2: + subcommand: Union[ + Annotated[A, dcargs.metadata.subcommand("command-a", default=A(7))], + Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], + ] + + @dataclasses.dataclass + class Nested1: + nested2: Nested2 + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-a".split(" "), + ) == Parent(Nested1(Nested2(A(7)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(A(3)))) + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-b".split(" "), + ) == Parent(Nested1(Nested2(B(9)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(B(7)))) + + +def test_subparser_in_nested_with_metadata_generic(): + @dataclasses.dataclass + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + T = TypeVar("T") + + @dataclasses.dataclass + class Nested2(Generic[T]): + subcommand: T + + @dataclasses.dataclass + class Nested1: + nested2: Nested2[ + Union[ + Annotated[A, dcargs.metadata.subcommand("command-a", default=A(7))], + Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], + ] + ] + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-a".split(" "), + ) == Parent(Nested1(Nested2(A(7)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(A(3)))) + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-b".split(" "), + ) == Parent(Nested1(Nested2(B(9)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(B(7)))) + + +def test_subparser_in_nested_with_metadata_generic_alt(): + @dataclasses.dataclass + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + T = TypeVar("T") + + @dataclasses.dataclass + class Nested2(Generic[T]): + subcommand: Union[ + Annotated[T, dcargs.metadata.subcommand("command-a", default=A(7))], + Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], + ] + + @dataclasses.dataclass + class Nested1: + nested2: Nested2[A] + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-a".split(" "), + ) == Parent(Nested1(Nested2(A(7)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(A(3)))) + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-b".split(" "), + ) == Parent(Nested1(Nested2(B(9)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(B(7)))) + + +def test_flag(): + """When boolean flags have no default value, they must be explicitly specified.""" + + @dataclasses.dataclass + class A: + x: bool + + assert dcargs.cli( + A, + args=["--x"], + default_instance=A(False), + ) == A(True) + + assert dcargs.cli( + dcargs.metadata.FlagsOff[A], + args=["--x", "True"], + default_instance=A(False), + ) == A(True) + + +def test_fixed(): + """When boolean flags have no default value, they must be explicitly specified.""" + + @dataclasses.dataclass + class A: + x: dcargs.metadata.Fixed[bool] + + assert dcargs.cli( + A, + args=[], + default_instance=A(True), + ) == A(True) + + with pytest.raises(SystemExit): + assert dcargs.cli( + dcargs.metadata.FlagsOff[A], + args=["--x", "True"], + default_instance=A(False), + ) == A(True) + + +def test_fixed_recursive(): + """When boolean flags have no default value, they must be explicitly specified.""" + + @dataclasses.dataclass + class A: + x: bool + + assert dcargs.cli( + A, + args=["--x"], + default_instance=A(False), + ) == A(True) + + with pytest.raises(SystemExit): + assert dcargs.cli( + dcargs.metadata.Fixed[ + dcargs.metadata.FlagsOff[A], + ], + args=["--x", "True"], + default_instance=A(False), + ) == A(True) diff --git a/tests/test_nested.py b/tests/test_nested.py index 22cbff7e..fc41f76a 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -306,59 +306,6 @@ class DefaultInstanceSubparser: dcargs.cli(DefaultInstanceSubparser, args=["--x", "1", "c", "--bc.y", "3"]) -def test_avoid_subparser_with_default_instance(): - @dataclasses.dataclass - class DefaultInstanceHTTPServer: - y: int = 0 - - @dataclasses.dataclass - class DefaultInstanceSMTPServer: - z: int = 0 - - @dataclasses.dataclass - class DefaultInstanceSubparser: - x: int - bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] - - assert ( - dcargs.cli( - DefaultInstanceSubparser, - args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "5"], - ) - == dcargs.cli( - DefaultInstanceSubparser, - args=["--x", "1", "--bc.y", "5"], - default_instance=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=3) - ), - avoid_subparsers=True, - ) - == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) - ) - assert dcargs.cli( - DefaultInstanceSubparser, - args=["bc:default-instance-smtp-server", "--bc.z", "3"], - default_instance=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=5) - ), - ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceSMTPServer(z=3)) - assert ( - dcargs.cli( - DefaultInstanceSubparser, - args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "8"], - ) - == dcargs.cli( - DefaultInstanceSubparser, - args=["--bc.y", "8"], - default_instance=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=7) - ), - avoid_subparsers=True, - ) - == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) - ) - - def test_optional_subparser(): @dataclasses.dataclass class OptionalHTTPServer: @@ -761,165 +708,3 @@ def main( assert hash(dcargs.cli(main, args="--x.num-epochs 10".split(" "))) == hash( frozendict({"num_epochs": 10, "batch_size": 64}) ) - - -def test_subparser_in_nested_with_metadata(): - @dataclasses.dataclass - class A: - a: int - - @dataclasses.dataclass - class B: - b: int - a: A = A(5) - - @dataclasses.dataclass - class Nested2: - subcommand: Union[ - Annotated[A, dcargs.metadata.subcommand("command-a", default=A(7))], - Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], - ] - - @dataclasses.dataclass - class Nested1: - nested2: Nested2 - - @dataclasses.dataclass - class Parent: - nested1: Nested1 - - assert dcargs.cli( - Parent, - args="nested1.nested2.subcommand:command-a".split(" "), - ) == Parent(Nested1(Nested2(A(7)))) - assert dcargs.cli( - Parent, - args=( - "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( - " " - ) - ), - ) == Parent(Nested1(Nested2(A(3)))) - - assert dcargs.cli( - Parent, - args="nested1.nested2.subcommand:command-b".split(" "), - ) == Parent(Nested1(Nested2(B(9)))) - assert dcargs.cli( - Parent, - args=( - "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( - " " - ) - ), - ) == Parent(Nested1(Nested2(B(7)))) - - -def test_subparser_in_nested_with_metadata_generic(): - @dataclasses.dataclass - class A: - a: int - - @dataclasses.dataclass - class B: - b: int - a: A = A(5) - - T = TypeVar("T") - - @dataclasses.dataclass - class Nested2(Generic[T]): - subcommand: T - - @dataclasses.dataclass - class Nested1: - nested2: Nested2[ - Union[ - Annotated[A, dcargs.metadata.subcommand("command-a", default=A(7))], - Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], - ] - ] - - @dataclasses.dataclass - class Parent: - nested1: Nested1 - - assert dcargs.cli( - Parent, - args="nested1.nested2.subcommand:command-a".split(" "), - ) == Parent(Nested1(Nested2(A(7)))) - assert dcargs.cli( - Parent, - args=( - "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( - " " - ) - ), - ) == Parent(Nested1(Nested2(A(3)))) - - assert dcargs.cli( - Parent, - args="nested1.nested2.subcommand:command-b".split(" "), - ) == Parent(Nested1(Nested2(B(9)))) - assert dcargs.cli( - Parent, - args=( - "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( - " " - ) - ), - ) == Parent(Nested1(Nested2(B(7)))) - - -def test_subparser_in_nested_with_metadata_generic_alt(): - @dataclasses.dataclass - class A: - a: int - - @dataclasses.dataclass - class B: - b: int - a: A = A(5) - - T = TypeVar("T") - - @dataclasses.dataclass - class Nested2(Generic[T]): - subcommand: Union[ - Annotated[T, dcargs.metadata.subcommand("command-a", default=A(7))], - Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], - ] - - @dataclasses.dataclass - class Nested1: - nested2: Nested2[A] - - @dataclasses.dataclass - class Parent: - nested1: Nested1 - - assert dcargs.cli( - Parent, - args="nested1.nested2.subcommand:command-a".split(" "), - ) == Parent(Nested1(Nested2(A(7)))) - assert dcargs.cli( - Parent, - args=( - "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( - " " - ) - ), - ) == Parent(Nested1(Nested2(A(3)))) - - assert dcargs.cli( - Parent, - args="nested1.nested2.subcommand:command-b".split(" "), - ) == Parent(Nested1(Nested2(B(9)))) - assert dcargs.cli( - Parent, - args=( - "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( - " " - ) - ), - ) == Parent(Nested1(Nested2(B(7)))) diff --git a/tests/test_union_from_mapping.py b/tests/test_union_from_mapping.py index b82fd3f9..14142f4b 100644 --- a/tests/test_union_from_mapping.py +++ b/tests/test_union_from_mapping.py @@ -15,7 +15,7 @@ def test_union_from_mapping(): "two": A(2), "three": A(3), } - ConfigUnion = dcargs.extras.union_type_from_mapping(base_configs) + ConfigUnion = dcargs.extras.subcommand_union_from_mapping(base_configs) assert dcargs.cli(ConfigUnion, args="one".split(" ")) == A(1) assert dcargs.cli(ConfigUnion, args="two".split(" ")) == A(2) @@ -32,7 +32,7 @@ def test_union_from_mapping_in_function(): # Hack for mypy. Not needed for pyright. ConfigUnion = A - ConfigUnion = dcargs.extras.union_type_from_mapping(base_configs) # type: ignore + ConfigUnion = dcargs.extras.subcommand_union_from_mapping(base_configs) # type: ignore def main(config: ConfigUnion, flag: bool = False) -> Optional[A]: if flag: From bd48a7cc8628325c0e8a71490513bf72a25fa00c Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 1 Sep 2022 01:41:49 -0700 Subject: [PATCH 06/19] `dcargs.metadata` => `dcargs.conf` --- dcargs/__init__.py | 4 +-- dcargs/_arguments.py | 4 +-- dcargs/_fields.py | 7 ++-- dcargs/_parsers.py | 12 +++---- dcargs/_strings.py | 2 +- dcargs/conf/__init__.py | 15 ++++++++ dcargs/conf/_markers.py | 44 +++++++++++++++++++++++ dcargs/{metadata => conf}/_subcommands.py | 0 dcargs/extras/_base_configs.py | 6 ++-- dcargs/metadata/__init__.py | 15 -------- dcargs/metadata/_markers.py | 33 ----------------- docs/source/goals_and_alternatives.md | 2 +- examples/10_base_configs.py | 4 +-- tests/test_metadata.py | 30 ++++++++-------- 14 files changed, 94 insertions(+), 84 deletions(-) create mode 100644 dcargs/conf/__init__.py create mode 100644 dcargs/conf/_markers.py rename dcargs/{metadata => conf}/_subcommands.py (100%) delete mode 100644 dcargs/metadata/__init__.py delete mode 100644 dcargs/metadata/_markers.py diff --git a/dcargs/__init__.py b/dcargs/__init__.py index 0b317d4f..466b4945 100644 --- a/dcargs/__init__.py +++ b/dcargs/__init__.py @@ -1,11 +1,11 @@ -from . import extras, metadata +from . import conf, extras from ._cli import cli, generate_parser from ._fields import MISSING_PUBLIC as MISSING from ._instantiators import UnsupportedTypeAnnotationError __all__ = [ + "conf", "extras", - "metadata", "cli", "generate_parser", "MISSING", diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index a1aca8f8..4df17a39 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -24,7 +24,7 @@ import termcolor from . import _fields, _instantiators, _resolver, _strings -from .metadata import _markers +from .conf import _markers try: # Python >=3.8. @@ -139,7 +139,7 @@ def _rule_handle_boolean_flags( if ( arg.field.default in _fields.MISSING_SINGLETONS or arg.field.positional - or _markers.FLAGS_OFF in arg.field.markers + or _markers.FLAG_CONVERSION_OFF in arg.field.markers ): # Treat bools as a normal parameter. return lowered diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 6c5169fa..87319bf2 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -17,8 +17,9 @@ import typing_extensions from typing_extensions import get_args, get_type_hints, is_typeddict +from . import conf # Avoid circular import. from . import _docstrings, _instantiators, _resolver, _singleton, _strings -from .metadata import _markers +from .conf import _markers @dataclasses.dataclass(frozen=True) @@ -141,10 +142,8 @@ def _try_field_list_from_callable( f: Union[Callable, Type], default_instance: _DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - from . import metadata as _metadata - f, found_subcommand_configs = _resolver.unwrap_annotated( - f, _metadata._subcommands._SubcommandConfiguration + f, conf._subcommands._SubcommandConfiguration ) if len(found_subcommand_configs) > 0: default_instance = found_subcommand_configs[0].default diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 9cca1d71..730cd228 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -17,8 +17,8 @@ _instantiators, _resolver, _strings, + conf, ) -from . import metadata as _metadata T = TypeVar("T") @@ -47,7 +47,7 @@ def from_callable( """Create a parser definition from a callable.""" # Resolve generic types. - markers = _resolver.unwrap_annotated(f, _metadata._markers.Marker)[1] + markers = _resolver.unwrap_annotated(f, conf._markers.Marker)[1] f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.narrow_type(f, default_instance) if parent_type_from_typevar is not None: @@ -94,7 +94,7 @@ def from_callable( ) # (1) Handle fields marked as fixed. - if _metadata._markers.FIXED in field.markers: + if conf._markers.FIXED in field.markers: args.append( _arguments.ArgumentDefinition( prefix=prefix, @@ -114,7 +114,7 @@ def from_callable( if subparsers_attempt is not None: if ( not subparsers_attempt.required - and _metadata._markers.SUBCOMMANDS_OFF in field.markers + and conf._markers.AVOID_SUBCOMMANDS in field.markers ): # Don't make a subparser. field = dataclasses.replace(field, typ=type(field.default)) @@ -285,12 +285,12 @@ def from_field( for option in options_no_none: name = _strings.subparser_name_from_type(prefix, option) option, found_subcommand_configs = _resolver.unwrap_annotated( - option, _metadata._subcommands._SubcommandConfiguration + option, conf._subcommands._SubcommandConfiguration ) if len(found_subcommand_configs) == 0: # Make a dummy subcommand config. found_subcommand_configs = ( - _metadata._subcommands._SubcommandConfiguration( + conf._subcommands._SubcommandConfiguration( "unused", description=None, default=( diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 7b2ddf14..4c30c1a2 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -54,7 +54,7 @@ def hyphen_separated_from_camel_case(name: str) -> str: def _subparser_name_from_type(cls: Type) -> str: - from .metadata import _subcommands # Prevent circular imports + from .conf import _subcommands # Prevent circular imports cls, type_from_typevar = _resolver.resolve_generic_types(cls) cls, found_subcommand_configs = _resolver.unwrap_annotated( diff --git a/dcargs/conf/__init__.py b/dcargs/conf/__init__.py new file mode 100644 index 00000000..7415c82e --- /dev/null +++ b/dcargs/conf/__init__.py @@ -0,0 +1,15 @@ +"""The :mod:`dcargs.conf` submodule contains helpers for attaching parsing-specific +configuration metadata to types via PEP 593 runtime annotations. + +Features here are supported, but are generally unnecessary and should be used sparingly. +""" + +from ._markers import AvoidSubcommands, Fixed, FlagConversionOff +from ._subcommands import subcommand + +__all__ = [ + "AvoidSubcommands", + "Fixed", + "FlagConversionOff", + "subcommand", +] diff --git a/dcargs/conf/_markers.py b/dcargs/conf/_markers.py new file mode 100644 index 00000000..91fa24d4 --- /dev/null +++ b/dcargs/conf/_markers.py @@ -0,0 +1,44 @@ +from typing import Type, TypeVar + +from typing_extensions import Annotated + +from .. import _singleton + + +class Marker(_singleton.Singleton): + pass + + +def _make_marker(description: str) -> Marker: + class _InnerMarker(Marker): + def __repr__(self): + return description + + return _InnerMarker() + + +# Current design issue: markers are applied recursively to nested structures, but can't +# be unapplied. + +T = TypeVar("T", bound=Type) + +FIXED = _make_marker("Fixed") +Fixed = Annotated[T, FIXED] +"""A type `T` can be annotated as `Fixed[T]` if we don't want dcargs to parse it. A +default value should be set instead.""" + +FLAG_CONVERSION_OFF = _make_marker("FlagConversionOff") +FlagConversionOff = Annotated[T, FLAG_CONVERSION_OFF] +"""Turn off flag conversion for booleans with default values. Instead, types annotated +with `bool` will expect an explicit True or False. + +Can be used directly on boolean annotations, `FlagConversionOff[bool]`, or recursively +applied to nested types.""" + +AVOID_SUBCOMMANDS = _make_marker("AvoidSubcommands") +AvoidSubcommands = Annotated[T, AVOID_SUBCOMMANDS] +"""Avoid creating subcommands when a default is provided for unions over nested types. +This simplifies CLI interfaces, but makes them less expressive. + +Can be used directly on union types, `AvoidSubcommands[Union[...]]`, or recursively +applied to nested types.""" diff --git a/dcargs/metadata/_subcommands.py b/dcargs/conf/_subcommands.py similarity index 100% rename from dcargs/metadata/_subcommands.py rename to dcargs/conf/_subcommands.py diff --git a/dcargs/extras/_base_configs.py b/dcargs/extras/_base_configs.py index 3a452888..e8576090 100644 --- a/dcargs/extras/_base_configs.py +++ b/dcargs/extras/_base_configs.py @@ -2,7 +2,7 @@ from typing_extensions import Annotated -from ..metadata import subcommand +from ..conf import subcommand T = TypeVar("T") @@ -27,11 +27,11 @@ def subcommand_union_from_mapping( Union[ Annotated[ Config, - dcargs.metadata.subcommand("small", default=Config(...)) + dcargs.conf.subcommand("small", default=Config(...)) ], Annotated[ Config, - dcargs.metadata.subcommand("big", default=Config(...)) + dcargs.conf.subcommand("big", default=Config(...)) ] ] ``` diff --git a/dcargs/metadata/__init__.py b/dcargs/metadata/__init__.py deleted file mode 100644 index 9324a808..00000000 --- a/dcargs/metadata/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -"""The :mod:`dcargs.metadata` submodule contains helpers for attaching parsing-specific -metadata to types. - -Features here are supported, but generally should be unnecessary. -""" - -from ._markers import Fixed, FlagsOff, SubcommandsOff -from ._subcommands import subcommand - -__all__ = [ - "Fixed", - "FlagsOff", - "SubcommandsOff", - "subcommand", -] diff --git a/dcargs/metadata/_markers.py b/dcargs/metadata/_markers.py deleted file mode 100644 index 0835bad3..00000000 --- a/dcargs/metadata/_markers.py +++ /dev/null @@ -1,33 +0,0 @@ -from typing import Type, TypeVar - -from typing_extensions import Annotated - -from .. import _singleton - - -class Marker(_singleton.Singleton): - pass - - -def _make_marker(description: str) -> Marker: - class _InnerMarker(Marker): - def __repr__(self): - return description - - return _InnerMarker() - - -T = TypeVar("T", bound=Type) - -FIXED = _make_marker("Fixed") -Fixed = Annotated[T, FIXED] -"""A type T can be annotated as Fixed[T] if we don't want dcargs to parse it. A default -value should be set instead.""" - -FLAGS_OFF = _make_marker("FlagsOff") -FlagsOff = Annotated[T, FLAGS_OFF] -"""Turn off flag conversion, which.""" - -SUBCOMMANDS_OFF = _make_marker("SubcommandsOff") -SubcommandsOff = Annotated[T, SUBCOMMANDS_OFF] -"""A boolean type can be annotated as NoFlag[bool] to turn off automatic flag generation.""" diff --git a/docs/source/goals_and_alternatives.md b/docs/source/goals_and_alternatives.md index 445903b9..e689b771 100644 --- a/docs/source/goals_and_alternatives.md +++ b/docs/source/goals_and_alternatives.md @@ -12,7 +12,7 @@ Usage distinctions are the result of two API goals: using the standard `typing.Literal` type, subparsers with `typing.Union` of nested types, and positional arguments with `/`. - In contrast, similar libraries have more expansive APIs (sometimes spanning - dozens of specialized class and functions), and often require + dozens of specialized class and functions), and require more library-specific structures, decorators, or metadata formats for configuring parsing behavior. - **Strict typing.** Any type that can be annotated and unambiguously parsed diff --git a/examples/10_base_configs.py b/examples/10_base_configs.py index c85b72e8..e87bf945 100644 --- a/examples/10_base_configs.py +++ b/examples/10_base_configs.py @@ -105,7 +105,7 @@ class ExperimentConfig: # Union[ # Annotated[ # ExperimentConfig, - # dcargs.metadata.subcommand( + # dcargs.conf.subcommand( # "small", # default=base_configs["small"], # description=descriptions["small"], @@ -113,7 +113,7 @@ class ExperimentConfig: # ], # Annotated[ # ExperimentConfig, - # dcargs.metadata.subcommand( + # dcargs.conf.subcommand( # "big", # default=base_configs["big"], # description=descriptions["big"], diff --git a/tests/test_metadata.py b/tests/test_metadata.py index ca653cb7..c00e56ee 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -19,7 +19,7 @@ class DefaultInstanceSMTPServer: @dataclasses.dataclass class DefaultInstanceSubparser: x: int - bc: dcargs.metadata.SubcommandsOff[ + bc: dcargs.conf.AvoidSubcommands[ Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] ] @@ -73,7 +73,7 @@ class DefaultInstanceSubparser: args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "5"], ) == dcargs.cli( - dcargs.metadata.SubcommandsOff[DefaultInstanceSubparser], + dcargs.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--x", "1", "--bc.y", "5"], default_instance=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=3) @@ -90,11 +90,11 @@ class DefaultInstanceSubparser: ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceSMTPServer(z=3)) assert ( dcargs.cli( - dcargs.metadata.SubcommandsOff[DefaultInstanceSubparser], + dcargs.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "8"], ) == dcargs.cli( - dcargs.metadata.SubcommandsOff[DefaultInstanceSubparser], + dcargs.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--bc.y", "8"], default_instance=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=7) @@ -117,8 +117,8 @@ class B: @dataclasses.dataclass class Nested2: subcommand: Union[ - Annotated[A, dcargs.metadata.subcommand("command-a", default=A(7))], - Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], + Annotated[A, dcargs.conf.subcommand("command-a", default=A(7))], + Annotated[B, dcargs.conf.subcommand("command-b", default=B(9))], ] @dataclasses.dataclass @@ -176,8 +176,8 @@ class Nested2(Generic[T]): class Nested1: nested2: Nested2[ Union[ - Annotated[A, dcargs.metadata.subcommand("command-a", default=A(7))], - Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], + Annotated[A, dcargs.conf.subcommand("command-a", default=A(7))], + Annotated[B, dcargs.conf.subcommand("command-b", default=B(9))], ] ] @@ -227,8 +227,8 @@ class B: @dataclasses.dataclass class Nested2(Generic[T]): subcommand: Union[ - Annotated[T, dcargs.metadata.subcommand("command-a", default=A(7))], - Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], + Annotated[T, dcargs.conf.subcommand("command-a", default=A(7))], + Annotated[B, dcargs.conf.subcommand("command-b", default=B(9))], ] @dataclasses.dataclass @@ -280,7 +280,7 @@ class A: ) == A(True) assert dcargs.cli( - dcargs.metadata.FlagsOff[A], + dcargs.conf.FlagConversionOff[A], args=["--x", "True"], default_instance=A(False), ) == A(True) @@ -291,7 +291,7 @@ def test_fixed(): @dataclasses.dataclass class A: - x: dcargs.metadata.Fixed[bool] + x: dcargs.conf.Fixed[bool] assert dcargs.cli( A, @@ -301,7 +301,7 @@ class A: with pytest.raises(SystemExit): assert dcargs.cli( - dcargs.metadata.FlagsOff[A], + dcargs.conf.FlagConversionOff[A], args=["--x", "True"], default_instance=A(False), ) == A(True) @@ -322,8 +322,8 @@ class A: with pytest.raises(SystemExit): assert dcargs.cli( - dcargs.metadata.Fixed[ - dcargs.metadata.FlagsOff[A], + dcargs.conf.Fixed[ + dcargs.conf.FlagConversionOff[A], ], args=["--x", "True"], default_instance=A(False), From e22247eabb4cfc7ff3bf3c3e8387668fca0ba5cc Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 1 Sep 2022 02:09:52 -0700 Subject: [PATCH 07/19] `default_instance` => `default`, `generate_parser` => `get_parser` --- dcargs/__init__.py | 4 +-- dcargs/_cli.py | 67 ++++++++++++++++++++++++----------- dcargs/_deprecated.py | 1 + dcargs/_parsers.py | 28 ++++++++++----- dcargs/_resolver.py | 4 ++- dcargs/conf/_subcommands.py | 10 +++--- tests/test_dcargs.py | 2 +- tests/test_dict_namedtuple.py | 10 +++--- tests/test_helptext.py | 2 +- tests/test_metadata.py | 26 +++++++------- tests/test_missing.py | 12 +++---- tests/test_nested.py | 42 +++++++++------------- 12 files changed, 119 insertions(+), 89 deletions(-) diff --git a/dcargs/__init__.py b/dcargs/__init__.py index 466b4945..3073b8b2 100644 --- a/dcargs/__init__.py +++ b/dcargs/__init__.py @@ -1,5 +1,5 @@ from . import conf, extras -from ._cli import cli, generate_parser +from ._cli import cli, get_parser from ._fields import MISSING_PUBLIC as MISSING from ._instantiators import UnsupportedTypeAnnotationError @@ -7,7 +7,7 @@ "conf", "extras", "cli", - "generate_parser", + "get_parser", "MISSING", "UnsupportedTypeAnnotationError", ] diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 1376bce8..71da4577 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -1,12 +1,12 @@ """Core public API.""" - import argparse import dataclasses +import warnings from typing import Callable, Optional, Sequence, Type, TypeVar, Union, cast, overload from typing_extensions import Literal, assert_never -from . import _argparse_formatter, _calling, _fields, _parsers, _strings +from . import _argparse_formatter, _calling, _fields, _parsers, _strings, conf OutT = TypeVar("OutT") @@ -26,7 +26,7 @@ def cli( prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[OutT] = None, + default: Optional[OutT] = None, ) -> OutT: ... @@ -38,7 +38,7 @@ def cli( prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[OutT] = None, + default: Optional[OutT] = None, ) -> OutT: ... @@ -49,7 +49,8 @@ def cli( prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[OutT] = None, + default: Optional[OutT] = None, + **deprecated_kwargs, ) -> OutT: """Call `f(...)`, with arguments populated from an automatically generated CLI interface. @@ -94,7 +95,7 @@ def cli( `argparse.ArgumentParser()`. args: If set, parse arguments from a sequence of strings instead of the commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`. - default_instance: An instance of `T` to use for default values; only supported + default: An instance of `T` to use for default values; only supported if `T` is a dataclass, TypedDict, or NamedTuple. Helpful for merging CLI arguments with values loaded from elsewhere. (for example, a config object loaded from a yaml file) @@ -108,42 +109,44 @@ def cli( prog=prog, description=description, args=args, - default_instance=default_instance, + default=default, + **deprecated_kwargs, ) return out @overload -def generate_parser( +def get_parser( f: Type[OutT], *, prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[OutT] = None, + default: Optional[OutT] = None, ) -> argparse.ArgumentParser: ... @overload -def generate_parser( +def get_parser( f: Callable[..., OutT], *, prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[OutT] = None, + default: Optional[OutT] = None, ) -> argparse.ArgumentParser: ... -def generate_parser( +def get_parser( f: Union[Type[OutT], Callable[..., OutT]], *, prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[OutT] = None, + default: Optional[OutT] = None, + **deprecated_kwargs, ) -> argparse.ArgumentParser: """Returns the argparse parser that would be used under-the-hood if `dcargs.cli()` was called with the same arguments. @@ -156,7 +159,8 @@ def generate_parser( prog=prog, description=description, args=args, - default_instance=default_instance, + default=default, + **deprecated_kwargs, ) assert isinstance(out, argparse.ArgumentParser) return out @@ -170,7 +174,8 @@ def _cli_impl( prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[OutT] = None, + default: Optional[OutT] = None, + **deprecated_kwargs, ) -> argparse.ArgumentParser: ... @@ -183,7 +188,8 @@ def _cli_impl( prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[OutT] = None, + default: Optional[OutT] = None, + **deprecated_kwargs, ) -> OutT: ... @@ -195,19 +201,37 @@ def _cli_impl( prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[OutT] = None, + default: Optional[OutT] = None, + **deprecated_kwargs, ) -> Union[OutT, argparse.ArgumentParser]: + + if "default_instance" in deprecated_kwargs: + warnings.warn( + "`default_instance=` is deprecated! use `default=` instead.", stacklevel=2 + ) + default = deprecated_kwargs["default_instance"] + if deprecated_kwargs.get("avoid_subparsers", False): + f = conf.AvoidSubcommands[f] + warnings.warn( + "`avoid_subparsers=` is deprecated! use `dcargs.conf.AvoidSubparsers[]` instead.", + stacklevel=2, + ) + + # Internally, we distinguish between two concepts: + # - "default", which is used for individual arguments. + # - "default_instance", which is used for _fields_ (which may be broken down into + # one or many arguments, depending on various factors). + # + # This could be revisited. default_instance_internal: Union[_fields.NonpropagatingMissingType, OutT] = ( - _fields.MISSING_NONPROP if default_instance is None else default_instance + _fields.MISSING_NONPROP if default is None else default ) if not _fields.is_nested_type(cast(Type, f), default_instance_internal): dummy_field = cast( dataclasses.Field, dataclasses.field( - default=default_instance - if default_instance is not None - else dataclasses.MISSING + default=default if default is not None else dataclasses.MISSING ), ) f = dataclasses.make_dataclass( @@ -224,6 +248,7 @@ def _cli_impl( description=description, parent_classes=set(), # Used for recursive calls. parent_type_from_typevar=None, # Used for recursive calls. + parent_markers=(), # Used for recursive calls. default_instance=default_instance_internal, # Overrides for default values. prefix="", # Used for recursive calls. ) diff --git a/dcargs/_deprecated.py b/dcargs/_deprecated.py index c879522a..05e32b46 100644 --- a/dcargs/_deprecated.py +++ b/dcargs/_deprecated.py @@ -1,2 +1,3 @@ from ._cli import cli as parse # noqa +from ._cli import get_parser as generate_parser # noqa from .extras._serialization import from_yaml, to_yaml # noqa diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 730cd228..128c5343 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -4,7 +4,19 @@ import argparse import dataclasses import itertools -from typing import Any, Callable, Dict, List, Optional, Set, Type, TypeVar, Union, cast +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + Set, + Tuple, + Type, + TypeVar, + Union, + cast, +) import termcolor from typing_extensions import get_args, get_origin @@ -39,6 +51,7 @@ def from_callable( description: Optional[str], parent_classes: Set[Type], parent_type_from_typevar: Optional[Dict[TypeVar, Type]], + parent_markers: Tuple[conf._markers.Marker, ...], default_instance: Union[ T, _fields.PropagatingMissingType, _fields.NonpropagatingMissingType ], @@ -47,7 +60,9 @@ def from_callable( """Create a parser definition from a callable.""" # Resolve generic types. - markers = _resolver.unwrap_annotated(f, conf._markers.Marker)[1] + markers = ( + parent_markers + _resolver.unwrap_annotated(f, conf._markers.Marker)[1] + ) f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.narrow_type(f, default_instance) if parent_type_from_typevar is not None: @@ -122,10 +137,6 @@ def from_callable( subparsers_from_name[ _strings.make_field_name([prefix, subparsers_attempt.name]) ] = subparsers_attempt - - for subparser_def in subparsers_attempt.parser_from_name.values(): - for arg in subparser_def.args: - arg.field.markers.extend(field.markers) continue # (3) Handle nested callables. @@ -142,14 +153,12 @@ def from_callable( description=None, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, + parent_markers=markers, default_instance=field.default, prefix=_strings.make_field_name([prefix, field.name]), ) args.extend(nested_parser.args) - for arg in nested_parser.args: - arg.field.markers.extend(field.markers) - # Include nested subparsers. subparsers_from_name.update(nested_parser.subparsers_from_name) @@ -307,6 +316,7 @@ def from_field( description=found_subcommand_configs[0].description, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, + parent_markers=tuple(field.markers), default_instance=found_subcommand_configs[0].default, prefix=prefix, ) diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index 46489672..8848eade 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -113,7 +113,9 @@ def type_from_typevar_constraints(typ: Union[Type, TypeVar]) -> Union[Type, Type def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: """Type narrowing: if we annotate as Animal but specify a default instance of Cat, we - should parse as Cat.""" + should parse as Cat. + + Note that Union types are intentionally excluded here.""" try: potential_subclass = type(default_instance) superclass = unwrap_annotated(typ)[0] diff --git a/dcargs/conf/_subcommands.py b/dcargs/conf/_subcommands.py index 104e5524..05f8ebc5 100644 --- a/dcargs/conf/_subcommands.py +++ b/dcargs/conf/_subcommands.py @@ -7,8 +7,8 @@ @dataclasses.dataclass(frozen=True) class _SubcommandConfiguration: name: str - description: Optional[str] default: Any + description: Optional[str] def __hash__(self) -> int: return object.__hash__(self) @@ -17,8 +17,8 @@ def __hash__(self) -> int: def subcommand( name: str, *, - description: Optional[str] = None, default: Any = MISSING_NONPROP, + description: Optional[str] = None, ) -> Any: """Returns a metadata object for configuring subcommands with `typing.Annotated`. This is useful but can make code harder to read, so usage is discouraged. @@ -34,16 +34,16 @@ def subcommand( This will create two subcommands: `nested-type-a` and `nested-type-b`. Annotating each type with `dcargs.metadata.subcommand()` allows us to override for - each subcommand the (a) name and (b) defaults. + each subcommand the (a) name, (b) defaults, and (c) helptext. ```python dcargs.cli( Union[ Annotated[ - NestedTypeA, subcommand("a", default=NestedTypeA(...)) + NestedTypeA, subcommand("a", default=NestedTypeA(...), description="...") ], Annotated[ - NestedTypeA, subcommand("b", default=NestedTypeA(...)) + NestedTypeA, subcommand("b", default=NestedTypeA(...), description="...") ], ] ) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 6c474064..656f1aa8 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -40,7 +40,7 @@ class ManyTypes: f: float p: pathlib.Path - assert isinstance(dcargs.generate_parser(ManyTypes), argparse.ArgumentParser) + assert isinstance(dcargs.get_parser(ManyTypes), argparse.ArgumentParser) # We can directly pass a dataclass to `dcargs.cli()`: assert dcargs.cli( diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index d1898e18..85889a26 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -165,7 +165,7 @@ class NestedTypedDict(TypedDict): dcargs.cli(NestedTypedDict, args=["--x", "1"]) -def test_helptext_and_default_instance_typeddict(): +def test_helptext_and_default_typeddict(): class HelptextTypedDict(TypedDict): """This docstring should be printed as a description.""" @@ -180,7 +180,7 @@ class HelptextTypedDict(TypedDict): f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.cli(HelptextTypedDict, default_instance={"z": 3}, args=["--help"]) + dcargs.cli(HelptextTypedDict, default={"z": 3}, args=["--help"]) helptext = dcargs._strings.strip_ansi_sequences(f.getvalue()) assert cast(str, HelptextTypedDict.__doc__) in helptext assert "--x INT" in helptext @@ -254,7 +254,7 @@ class HelptextNamedTupleDefault(NamedTuple): assert "Documentation 3 (default: 3)\n" in helptext -def test_helptext_and_default_instance_namedtuple(): +def test_helptext_and_default_namedtuple(): class HelptextNamedTuple(NamedTuple): """This docstring should be printed as a description.""" @@ -269,7 +269,7 @@ class HelptextNamedTuple(NamedTuple): with pytest.raises(SystemExit): dcargs.cli( HelptextNamedTuple, - default_instance=dcargs.MISSING, + default=dcargs.MISSING, args=[], ) @@ -278,7 +278,7 @@ class HelptextNamedTuple(NamedTuple): with contextlib.redirect_stdout(f): dcargs.cli( HelptextNamedTuple, - default_instance=HelptextNamedTuple( + default=HelptextNamedTuple( x=dcargs.MISSING, y=dcargs.MISSING, z=3, diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 485b70a1..1eae5ea1 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -25,7 +25,7 @@ def _get_helptext(f: Callable, args: List[str] = ["--help"]) -> str: target2 = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target2): with dcargs._argparse_formatter.ansi_context(): - dcargs.generate_parser(f).parse_args(args) + dcargs.get_parser(f).parse_args(args) assert target.getvalue() == target2.getvalue() return dcargs._strings.strip_ansi_sequences(target.getvalue()) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index c00e56ee..0b5dfcff 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -7,7 +7,7 @@ import dcargs -def test_avoid_subparser_with_default_instance(): +def test_avoid_subparser_with_default(): @dataclasses.dataclass class DefaultInstanceHTTPServer: y: int = 0 @@ -31,7 +31,7 @@ class DefaultInstanceSubparser: == dcargs.cli( DefaultInstanceSubparser, args=["--x", "1", "--bc.y", "5"], - default_instance=DefaultInstanceSubparser( + default=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=3) ), ) @@ -45,7 +45,7 @@ class DefaultInstanceSubparser: == dcargs.cli( DefaultInstanceSubparser, args=["--bc.y", "8"], - default_instance=DefaultInstanceSubparser( + default=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=7) ), ) @@ -53,7 +53,7 @@ class DefaultInstanceSubparser: ) -def test_avoid_subparser_with_default_instance_recursive(): +def test_avoid_subparser_with_default_recursive(): @dataclasses.dataclass class DefaultInstanceHTTPServer: y: int = 0 @@ -75,7 +75,7 @@ class DefaultInstanceSubparser: == dcargs.cli( dcargs.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--x", "1", "--bc.y", "5"], - default_instance=DefaultInstanceSubparser( + default=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=3) ), ) @@ -84,7 +84,7 @@ class DefaultInstanceSubparser: assert dcargs.cli( DefaultInstanceSubparser, args=["bc:default-instance-smtp-server", "--bc.z", "3"], - default_instance=DefaultInstanceSubparser( + default=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=5) ), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceSMTPServer(z=3)) @@ -96,7 +96,7 @@ class DefaultInstanceSubparser: == dcargs.cli( dcargs.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--bc.y", "8"], - default_instance=DefaultInstanceSubparser( + default=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=7) ), ) @@ -276,13 +276,13 @@ class A: assert dcargs.cli( A, args=["--x"], - default_instance=A(False), + default=A(False), ) == A(True) assert dcargs.cli( dcargs.conf.FlagConversionOff[A], args=["--x", "True"], - default_instance=A(False), + default=A(False), ) == A(True) @@ -296,14 +296,14 @@ class A: assert dcargs.cli( A, args=[], - default_instance=A(True), + default=A(True), ) == A(True) with pytest.raises(SystemExit): assert dcargs.cli( dcargs.conf.FlagConversionOff[A], args=["--x", "True"], - default_instance=A(False), + default=A(False), ) == A(True) @@ -317,7 +317,7 @@ class A: assert dcargs.cli( A, args=["--x"], - default_instance=A(False), + default=A(False), ) == A(True) with pytest.raises(SystemExit): @@ -326,5 +326,5 @@ class A: dcargs.conf.FlagConversionOff[A], ], args=["--x", "True"], - default_instance=A(False), + default=A(False), ) == A(True) diff --git a/tests/test_missing.py b/tests/test_missing.py index 74b0eb27..16cee168 100644 --- a/tests/test_missing.py +++ b/tests/test_missing.py @@ -29,7 +29,7 @@ class Args2: assert dcargs.cli(Args2, args=["--b", "7"]) == Args2(5, 7, 3) -def test_missing_default_instance(): +def test_missing_default(): @dataclasses.dataclass class Args2: a: int @@ -40,16 +40,16 @@ class Args2: dcargs.cli( Args2, args=[], - default_instance=Args2(5, dcargs.MISSING, 3), + default=Args2(5, dcargs.MISSING, 3), ) assert dcargs.cli( Args2, args=["--b", "7"], - default_instance=Args2(5, dcargs.MISSING, 3), + default=Args2(5, dcargs.MISSING, 3), ) == Args2(5, 7, 3) -def test_missing_nested_default_instance(): +def test_missing_nested_default(): @dataclasses.dataclass class Child: a: int = 5 @@ -64,10 +64,10 @@ class Parent: dcargs.cli( Parent, args=[], - default_instance=Parent(child=dcargs.MISSING), + default=Parent(child=dcargs.MISSING), ) assert dcargs.cli( Parent, args=["--child.a", "5", "--child.b", "7", "--child.c", "3"], - default_instance=Parent(child=dcargs.MISSING), + default=Parent(child=dcargs.MISSING), ) == Parent(Child(5, 7, 3)) diff --git a/tests/test_nested.py b/tests/test_nested.py index fc41f76a..e69332bc 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -38,7 +38,7 @@ class Nested: dcargs.cli(Nested, args=["--x", "1"]) -def test_nested_default_instance(): +def test_nested_default(): @dataclasses.dataclass class B: y: int = 1 @@ -48,9 +48,9 @@ class Nested: x: int = 2 b: B = B() - assert dcargs.cli( - Nested, args=[], default_instance=Nested(x=1, b=B(y=2)) - ) == Nested(x=1, b=B(y=2)) + assert dcargs.cli(Nested, args=[], default=Nested(x=1, b=B(y=2))) == Nested( + x=1, b=B(y=2) + ) def test_nested_default(): @@ -66,7 +66,7 @@ class Nested: assert ( Nested(x=1, b=B(y=3)) == dcargs.cli(Nested, args=["--x", "1", "--b.y", "3"]) - == dcargs.cli(Nested, args=[], default_instance=Nested(x=1, b=B(y=3))) + == dcargs.cli(Nested, args=[], default=Nested(x=1, b=B(y=3))) ) assert dcargs.cli(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=3)) @@ -232,7 +232,7 @@ class DefaultSubparser: == dcargs.cli( DefaultSubparser, args=[], - default_instance=DefaultSubparser(x=1, bc=DefaultHTTPServer(y=8)), + default=DefaultSubparser(x=1, bc=DefaultHTTPServer(y=8)), ) == DefaultSubparser(x=1, bc=DefaultHTTPServer(y=8)) ) @@ -243,7 +243,7 @@ class DefaultSubparser: dcargs.cli(DefaultSubparser, args=["--x", "1", "c", "--bc.y", "3"]) -def test_subparser_with_default_instance(): +def test_subparser_with_default(): @dataclasses.dataclass class DefaultInstanceHTTPServer: y: int = 0 @@ -265,25 +265,19 @@ class DefaultInstanceSubparser: == dcargs.cli( DefaultInstanceSubparser, args=[], - default_instance=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=5) - ), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)), ) == dcargs.cli( DefaultInstanceSubparser, args=["bc:default-instance-http-server"], - default_instance=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=5) - ), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) ) assert dcargs.cli( DefaultInstanceSubparser, args=["bc:default-instance-smtp-server", "--bc.z", "3"], - default_instance=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=5) - ), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceSMTPServer(z=3)) assert ( dcargs.cli( @@ -293,9 +287,7 @@ class DefaultInstanceSubparser: == dcargs.cli( DefaultInstanceSubparser, args=[], - default_instance=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=8) - ), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) ) @@ -459,7 +451,7 @@ class MultipleSubparsers: dcargs.cli( MultipleSubparsers, args=[], - default_instance=MultipleSubparsers( + default=MultipleSubparsers( Subcommand1(), Subcommand2(), Subcommand3(dcargs.MISSING), @@ -471,7 +463,7 @@ class MultipleSubparsers: args=[ "a:subcommand1", ], - default_instance=MultipleSubparsers( + default=MultipleSubparsers( Subcommand1(), Subcommand2(), Subcommand3(dcargs.MISSING), @@ -481,7 +473,7 @@ class MultipleSubparsers: dcargs.cli( MultipleSubparsers, args=["a:subcommand1", "b:subcommand2"], - default_instance=MultipleSubparsers( + default=MultipleSubparsers( Subcommand1(), Subcommand2(), Subcommand3(dcargs.MISSING), @@ -491,7 +483,7 @@ class MultipleSubparsers: dcargs.cli( MultipleSubparsers, args=["a:subcommand1", "b:subcommand2", "c:subcommand3"], - default_instance=MultipleSubparsers( + default=MultipleSubparsers( Subcommand1(), Subcommand2(), Subcommand3(dcargs.MISSING), @@ -500,7 +492,7 @@ class MultipleSubparsers: assert dcargs.cli( MultipleSubparsers, args=["a:subcommand1", "b:subcommand2", "c:subcommand3", "--c.z", "3"], - default_instance=MultipleSubparsers( + default=MultipleSubparsers( Subcommand1(), Subcommand2(), Subcommand3(dcargs.MISSING), @@ -509,7 +501,7 @@ class MultipleSubparsers: assert dcargs.cli( MultipleSubparsers, args=["a:subcommand1", "b:subcommand2", "c:subcommand2"], - default_instance=MultipleSubparsers( + default=MultipleSubparsers( Subcommand1(), Subcommand2(), Subcommand3(dcargs.MISSING), From 880e90d9d0258c3e3fdbf6d3ceeddbe6384c2dd1 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 1 Sep 2022 02:23:59 -0700 Subject: [PATCH 08/19] Cleanup --- dcargs/_cli.py | 6 +- dcargs/_parsers.py | 123 +++++++++--------- dcargs/conf/_subcommands.py | 2 +- dcargs/extras/_base_configs.py | 12 +- .../{08_subparsers.rst => 08_subcommands.rst} | 24 ++-- ...arsers.rst => 09_multiple_subcommands.rst} | 20 +-- docs/source/examples/10_base_configs.rst | 96 ++++++++------ docs/source/goals_and_alternatives.md | 19 ++- .../{08_subparsers.py => 08_subcommands.py} | 12 +- ...bparsers.py => 09_multiple_subcommands.py} | 10 +- tests/test_dict_namedtuple.py | 2 +- tests/test_metadata.py | 20 +-- tests/test_nested.py | 4 +- 13 files changed, 178 insertions(+), 172 deletions(-) rename docs/source/examples/{08_subparsers.rst => 08_subcommands.rst} (61%) rename docs/source/examples/{09_multiple_subparsers.rst => 09_multiple_subcommands.rst} (72%) rename examples/{08_subparsers.py => 08_subcommands.py} (70%) rename examples/{09_multiple_subparsers.py => 09_multiple_subcommands.py} (80%) diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 71da4577..02ce6465 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -204,16 +204,16 @@ def _cli_impl( default: Optional[OutT] = None, **deprecated_kwargs, ) -> Union[OutT, argparse.ArgumentParser]: - if "default_instance" in deprecated_kwargs: warnings.warn( "`default_instance=` is deprecated! use `default=` instead.", stacklevel=2 ) default = deprecated_kwargs["default_instance"] if deprecated_kwargs.get("avoid_subparsers", False): - f = conf.AvoidSubcommands[f] + f = conf.AvoidSubcommands[f] # type: ignore warnings.warn( - "`avoid_subparsers=` is deprecated! use `dcargs.conf.AvoidSubparsers[]` instead.", + "`avoid_subparsers=` is deprecated! use `dcargs.conf.AvoidSubparsers[]`" + " instead.", stacklevel=2, ) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 128c5343..197dc4fb 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -108,75 +108,70 @@ def from_callable( f"Field {field.name} has an unbound TypeVar: {field.typ}." ) - # (1) Handle fields marked as fixed. - if conf._markers.FIXED in field.markers: - args.append( - _arguments.ArgumentDefinition( - prefix=prefix, - field=field, - type_from_typevar=type_from_typevar, - ) - ) - continue - - # (2) Handle Unions over callables; these result in subparsers. - subparsers_attempt = SubparsersSpecification.from_field( - field, - type_from_typevar=type_from_typevar, - parent_classes=parent_classes, - prefix=_strings.make_field_name([prefix, field.name]), - ) - if subparsers_attempt is not None: - if ( - not subparsers_attempt.required - and conf._markers.AVOID_SUBCOMMANDS in field.markers - ): - # Don't make a subparser. - field = dataclasses.replace(field, typ=type(field.default)) - else: - subparsers_from_name[ - _strings.make_field_name([prefix, subparsers_attempt.name]) - ] = subparsers_attempt - continue - - # (3) Handle nested callables. - if _fields.is_nested_type(field.typ, field.default): - field = dataclasses.replace( + if conf._markers.FIXED not in field.markers: + # (1) Handle Unions over callables; these result in subparsers. + subparsers_attempt = SubparsersSpecification.from_field( field, - typ=_resolver.narrow_type( - field.typ, - field.default, - ), - ) - nested_parser = ParserSpecification.from_callable( - field.typ, - description=None, + type_from_typevar=type_from_typevar, parent_classes=parent_classes, - parent_type_from_typevar=type_from_typevar, - parent_markers=markers, - default_instance=field.default, prefix=_strings.make_field_name([prefix, field.name]), ) - args.extend(nested_parser.args) - - # Include nested subparsers. - subparsers_from_name.update(nested_parser.subparsers_from_name) - - # Include nested strings. - for k, v in nested_parser.helptext_from_nested_class_field_name.items(): - helptext_from_nested_class_field_name[ - _strings.make_field_name([field.name, k]) - ] = v - - if field.helptext is not None: - helptext_from_nested_class_field_name[field.name] = field.helptext - else: - helptext_from_nested_class_field_name[ - field.name - ] = _docstrings.get_callable_description(field.typ) - continue + if subparsers_attempt is not None: + if ( + not subparsers_attempt.required + and conf._markers.AVOID_SUBCOMMANDS in field.markers + ): + # Don't make a subparser. + field = dataclasses.replace(field, typ=type(field.default)) + else: + subparsers_from_name[ + _strings.make_field_name([prefix, subparsers_attempt.name]) + ] = subparsers_attempt + continue + + # (2) Handle nested callables. + if _fields.is_nested_type(field.typ, field.default): + field = dataclasses.replace( + field, + typ=_resolver.narrow_type( + field.typ, + field.default, + ), + ) + nested_parser = ParserSpecification.from_callable( + field.typ, + description=None, + parent_classes=parent_classes, + parent_type_from_typevar=type_from_typevar, + parent_markers=markers, + default_instance=field.default, + prefix=_strings.make_field_name([prefix, field.name]), + ) + args.extend(nested_parser.args) + + # Include nested subparsers. + subparsers_from_name.update(nested_parser.subparsers_from_name) + + # Include nested strings. + for ( + k, + v, + ) in nested_parser.helptext_from_nested_class_field_name.items(): + helptext_from_nested_class_field_name[ + _strings.make_field_name([field.name, k]) + ] = v + + if field.helptext is not None: + helptext_from_nested_class_field_name[ + field.name + ] = field.helptext + else: + helptext_from_nested_class_field_name[ + field.name + ] = _docstrings.get_callable_description(field.typ) + continue - # (4) Handle primitive types. These produce a single argument! + # (3) Handle primitive or fixed types. These produce a single argument! args.append( _arguments.ArgumentDefinition( prefix=prefix, diff --git a/dcargs/conf/_subcommands.py b/dcargs/conf/_subcommands.py index 05f8ebc5..e5a31957 100644 --- a/dcargs/conf/_subcommands.py +++ b/dcargs/conf/_subcommands.py @@ -49,4 +49,4 @@ def subcommand( ) ``` """ - return _SubcommandConfiguration(name, description, default) + return _SubcommandConfiguration(name, default, description) diff --git a/dcargs/extras/_base_configs.py b/dcargs/extras/_base_configs.py index e8576090..1159df5d 100644 --- a/dcargs/extras/_base_configs.py +++ b/dcargs/extras/_base_configs.py @@ -8,11 +8,11 @@ def subcommand_union_from_mapping( - defaults: Mapping[str, T], descriptions: Mapping[str, str] = {} + default_from_name: Mapping[str, T], descriptions: Mapping[str, str] = {} ) -> Type[T]: """Returns a Union type for defining subcommands that choose between nested types. - For example, when `base_mapping` is set to: + For example, when `default` is set to: ```python { @@ -39,14 +39,14 @@ def subcommand_union_from_mapping( This can be used directly in dcargs.cli: ```python - config = dcargs.cli(union_from_base_mapping(base_mapping)) + config = dcargs.cli(subcommand_union_from_mapping(default_from_name)) reveal_type(config) # Should be correct! ``` Or to generate annotations for classes and functions: ```python - SelectableConfig = union_from_base_mapping(base_mapping) + SelectableConfig = subcommand_union_from_mapping(default_from_name) def train( config: SelectableConfig, @@ -64,7 +64,7 @@ def train( if TYPE_CHECKING: SelectableConfig = ExperimentConfig else: - SelectableConfig = union_from_base_mapping(base_mapping) + SelectableConfig = subcommand_union_from_mapping(base_mapping) ``` """ return Union.__getitem__( # type: ignore @@ -72,6 +72,6 @@ def train( Annotated.__class_getitem__( # type: ignore (type(v), subcommand(k, default=v, description=descriptions.get(k))) ) - for k, v in defaults.items() + for k, v in default_from_name.items() ) ) diff --git a/docs/source/examples/08_subparsers.rst b/docs/source/examples/08_subcommands.rst similarity index 61% rename from docs/source/examples/08_subparsers.rst rename to docs/source/examples/08_subcommands.rst index bc1c20f6..841b408f 100644 --- a/docs/source/examples/08_subparsers.rst +++ b/docs/source/examples/08_subcommands.rst @@ -1,11 +1,11 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -8. Subparsers +8. Subcommands ========================================== -Unions over nested types (classes or dataclasses) are populated using subparsers. +Unions over nested types (classes or dataclasses) are populated using subcommands. @@ -49,38 +49,38 @@ Unions over nested types (classes or dataclasses) are populated using subparsers .. raw:: html - python 08_subparsers.py --help + python 08_subcommands.py --help -.. program-output:: python ../../examples/08_subparsers.py --help +.. program-output:: python ../../examples/08_subcommands.py --help ------------ .. raw:: html - python 08_subparsers.py cmd:commit --help + python 08_subcommands.py cmd:commit --help -.. program-output:: python ../../examples/08_subparsers.py cmd:commit --help +.. program-output:: python ../../examples/08_subcommands.py cmd:commit --help ------------ .. raw:: html - python 08_subparsers.py cmd:commit --cmd.message hello --cmd.all + python 08_subcommands.py cmd:commit --cmd.message hello --cmd.all -.. program-output:: python ../../examples/08_subparsers.py cmd:commit --cmd.message hello --cmd.all +.. program-output:: python ../../examples/08_subcommands.py cmd:commit --cmd.message hello --cmd.all ------------ .. raw:: html - python 08_subparsers.py cmd:checkout --help + python 08_subcommands.py cmd:checkout --help -.. program-output:: python ../../examples/08_subparsers.py cmd:checkout --help +.. program-output:: python ../../examples/08_subcommands.py cmd:checkout --help ------------ .. raw:: html - python 08_subparsers.py cmd:checkout --cmd.branch main + python 08_subcommands.py cmd:checkout --cmd.branch main -.. program-output:: python ../../examples/08_subparsers.py cmd:checkout --cmd.branch main +.. program-output:: python ../../examples/08_subcommands.py cmd:checkout --cmd.branch main diff --git a/docs/source/examples/09_multiple_subparsers.rst b/docs/source/examples/09_multiple_subcommands.rst similarity index 72% rename from docs/source/examples/09_multiple_subparsers.rst rename to docs/source/examples/09_multiple_subcommands.rst index 0af826b7..bf44d935 100644 --- a/docs/source/examples/09_multiple_subparsers.rst +++ b/docs/source/examples/09_multiple_subcommands.rst @@ -1,11 +1,11 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -9. Multiple Subparsers +9. Multiple Subcommands ========================================== -Multiple unions over nested types are populated using a series of subparsers. +Multiple unions over nested types are populated using a series of subcommands. @@ -75,30 +75,30 @@ Multiple unions over nested types are populated using a series of subparsers. .. raw:: html - python 09_multiple_subparsers.py + python 09_multiple_subcommands.py -.. program-output:: python ../../examples/09_multiple_subparsers.py +.. program-output:: python ../../examples/09_multiple_subcommands.py ------------ .. raw:: html - python 09_multiple_subparsers.py --help + python 09_multiple_subcommands.py --help -.. program-output:: python ../../examples/09_multiple_subparsers.py --help +.. program-output:: python ../../examples/09_multiple_subcommands.py --help ------------ .. raw:: html - python 09_multiple_subparsers.py dataset:mnist --help + python 09_multiple_subcommands.py dataset:mnist --help -.. program-output:: python ../../examples/09_multiple_subparsers.py dataset:mnist --help +.. program-output:: python ../../examples/09_multiple_subcommands.py dataset:mnist --help ------------ .. raw:: html - python 09_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 + python 09_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 -.. program-output:: python ../../examples/09_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 +.. program-output:: python ../../examples/09_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 diff --git a/docs/source/examples/10_base_configs.rst b/docs/source/examples/10_base_configs.rst index a78b030c..5e1a3b70 100644 --- a/docs/source/examples/10_base_configs.rst +++ b/docs/source/examples/10_base_configs.rst @@ -6,11 +6,11 @@ We can integrate ``dcargs.cli()`` into common configuration patterns: here, we select -one of multiple possible base configurations, and then use the CLI to either override -(existing) or fill in (missing) values. +one of multiple possible base configurations, create a subcommand for each one, and then +use the CLI to either override (existing) or fill in (missing) values. -Note that our interfaces don't prescribe any of the mechanics used for storing or -choosing between base configurations. A Hydra-style YAML approach could just as easily +Note that our interfaces don't prescribe any of the mechanics used for storing +base configurations. A Hydra-style YAML approach could just as easily be used for the config libary (although we generally prefer to avoid YAMLs; staying in Python is convenient for autocompletion and type checking). For selection, we could also avoid fussing with ``sys.argv`` by using a ``BASE_CONFIG`` environment variable. @@ -21,10 +21,9 @@ avoid fussing with ``sys.argv`` by using a ``BASE_CONFIG`` environment variable. :linenos: from dataclasses import dataclass - from typing import Callable, Literal, Mapping, Tuple, Type, TypeVar, Union + from typing import Callable, Literal, Tuple, Union from torch import nn - from typing_extensions import Annotated, reveal_type import dcargs @@ -69,49 +68,72 @@ avoid fussing with ``sys.argv`` by using a ``BASE_CONFIG`` environment variable. # Note that we could also define this library using separate YAML files (similar to # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type # checking + IDE support. - base_configs = { - "small": ExperimentConfig( - dataset="mnist", - optimizer=SgdOptimizer(), - batch_size=2048, - num_layers=4, - units=64, - train_steps=30_000, - # The dcargs.MISSING sentinel allows us to specify that the seed should have no - # default, and needs to be populated from the CLI. - seed=dcargs.MISSING, - activation=nn.ReLU, - ), - "big": ExperimentConfig( - dataset="imagenet-50", - optimizer=AdamOptimizer(), - batch_size=32, - num_layers=8, - units=256, - train_steps=100_000, - seed=dcargs.MISSING, - activation=nn.GELU, - ), - } + descriptions = {} + base_configs = {} + + descriptions["small"] = "Train a smaller model." + base_configs["small"] = ExperimentConfig( + dataset="mnist", + optimizer=SgdOptimizer(), + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + # The dcargs.MISSING sentinel allows us to specify that the seed should have no + # default, and needs to be populated from the CLI. + seed=dcargs.MISSING, + activation=nn.ReLU, + ) + + + descriptions["big"] = "Train a bigger model." + base_configs["big"] = ExperimentConfig( + dataset="imagenet-50", + optimizer=AdamOptimizer(), + batch_size=32, + num_layers=8, + units=256, + train_steps=100_000, + seed=dcargs.MISSING, + activation=nn.GELU, + ) if __name__ == "__main__": config = dcargs.cli( - dcargs.extras.union_type_from_mapping(base_configs), - # `avoid_subparsers` will avoid making a subparser for unions when a default is - # provided; it simplifies our CLI but makes it less expressive. - avoid_subparsers=True, + dcargs.extras.subcommand_union_from_mapping(base_configs, descriptions), ) - reveal_type(config) # Should ExperimentConfig, both staticaly and dynamically. + # Note that this is equivalent to: + # + # config = dcargs.cli( + # Union[ + # Annotated[ + # ExperimentConfig, + # dcargs.conf.subcommand( + # "small", + # default=base_configs["small"], + # description=descriptions["small"], + # ), + # ], + # Annotated[ + # ExperimentConfig, + # dcargs.conf.subcommand( + # "big", + # default=base_configs["big"], + # description=descriptions["big"], + # ), + # ], + # ] + # ) print(config) ------------ .. raw:: html - python 10_base_configs.py + python 10_base_configs.py --help -.. program-output:: python ../../examples/10_base_configs.py +.. program-output:: python ../../examples/10_base_configs.py --help ------------ diff --git a/docs/source/goals_and_alternatives.md b/docs/source/goals_and_alternatives.md index e689b771..e1667ac8 100644 --- a/docs/source/goals_and_alternatives.md +++ b/docs/source/goals_and_alternatives.md @@ -6,18 +6,17 @@ The core functionality of `dcargs` — generating argument parsers from type annotations — overlaps significantly with features offered by other libraries. Usage distinctions are the result of two API goals: -- **One uninvasive function.** Whenever possible, learning to use `dcargs` - should reduce to learning to write (type-annotated) Python. For example, types - are specified using standard annotations, helptext using docstrings, choices - using the standard `typing.Literal` type, subparsers with `typing.Union` of - nested types, and positional arguments with `/`. - - In contrast, similar libraries have more expansive APIs (sometimes spanning - dozens of specialized class and functions), and require more +- **One uninvasive function.** For all core functionality, learning to use + `dcargs` should reduce to learning to write (type-annotated) Python. For + example, types are specified using standard annotations, helptext using + docstrings, choices using the standard `typing.Literal` type, subcommands with + `typing.Union` of nested types, and positional arguments with `/`. + - In contrast, similar libraries have more expansive APIs , and require more library-specific structures, decorators, or metadata formats for configuring parsing behavior. - **Strict typing.** Any type that can be annotated and unambiguously parsed with an `argparse`-style CLI interface should work out-of-the-box; any public - API that isn't statically analyzable shouldn't be implemented. + API that isn't statically analyzable should be avoided. - In contrast, many similar libraries implement features that depend on dynamic argparse-style namespaces, or string-based accessors that can't be statically checked. @@ -56,10 +55,10 @@ More concretely, we can also compare specific features. A noncomprehensive set: [omegaconf]: https://omegaconf.readthedocs.io/en/2.1_branch/structured_config.html [^datargs_unions_nested]: One allowed per class. -[^tap_unions_nested]: Not supported, but API exists for creating subparsers that accomplish a similar goal. +[^tap_unions_nested]: Not supported, but API exists for creating subcommands that accomplish a similar goal. [^simp_unions_nested]: One allowed per class. [^yahp_unions_nested]: Not supported, but similar functionality available via ["registries"](https://docs.mosaicml.com/projects/yahp/en/stable/examples/registry.html). -[^typer_unions_nested]: Not supported, but API exists for creating subparsers that accomplish a similar goal. +[^typer_unions_nested]: Not supported, but API exists for creating subcommands that accomplish a similar goal. [^simp_literals]: Not supported for mixed (eg `Literal[5, "five"]`) or in container (eg `List[Literal[1, 2]]`) types. [^datargs_literals]: Not supported for mixed types (eg `Literal[5, "five"]`). [^typer_containers]: `typer` uses positional arguments for all required fields, which means that only one variable-length argument (such as `List[int]`) without a default is supported per argument parser. diff --git a/examples/08_subparsers.py b/examples/08_subcommands.py similarity index 70% rename from examples/08_subparsers.py rename to examples/08_subcommands.py index d0156da2..dba66b89 100644 --- a/examples/08_subparsers.py +++ b/examples/08_subcommands.py @@ -1,11 +1,11 @@ -"""Unions over nested types (classes or dataclasses) are populated using subparsers. +"""Unions over nested types (classes or dataclasses) are populated using subcommands. Usage: -`python ./08_subparsers.py --help` -`python ./08_subparsers.py cmd:commit --help` -`python ./08_subparsers.py cmd:commit --cmd.message hello --cmd.all` -`python ./08_subparsers.py cmd:checkout --help` -`python ./08_subparsers.py cmd:checkout --cmd.branch main` +`python ./08_subcommands.py --help` +`python ./08_subcommands.py cmd:commit --help` +`python ./08_subcommands.py cmd:commit --cmd.message hello --cmd.all` +`python ./08_subcommands.py cmd:checkout --help` +`python ./08_subcommands.py cmd:checkout --cmd.branch main` """ from __future__ import annotations diff --git a/examples/09_multiple_subparsers.py b/examples/09_multiple_subcommands.py similarity index 80% rename from examples/09_multiple_subparsers.py rename to examples/09_multiple_subcommands.py index 512d61df..cede20a1 100644 --- a/examples/09_multiple_subparsers.py +++ b/examples/09_multiple_subcommands.py @@ -1,10 +1,10 @@ -"""Multiple unions over nested types are populated using a series of subparsers. +"""Multiple unions over nested types are populated using a series of subcommands. Usage: -`python ./09_multiple_subparsers.py` -`python ./09_multiple_subparsers.py --help` -`python ./09_multiple_subparsers.py dataset:mnist --help` -`python ./09_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4` +`python ./09_multiple_subcommands.py` +`python ./09_multiple_subcommands.py --help` +`python ./09_multiple_subcommands.py dataset:mnist --help` +`python ./09_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4` """ from __future__ import annotations diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index 85889a26..c129a424 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -254,7 +254,7 @@ class HelptextNamedTupleDefault(NamedTuple): assert "Documentation 3 (default: 3)\n" in helptext -def test_helptext_and_default_namedtuple(): +def test_helptext_and_default_namedtuple_alternate(): class HelptextNamedTuple(NamedTuple): """This docstring should be printed as a description.""" diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 0b5dfcff..b7c8b394 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -31,9 +31,7 @@ class DefaultInstanceSubparser: == dcargs.cli( DefaultInstanceSubparser, args=["--x", "1", "--bc.y", "5"], - default=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=3) - ), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=3)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) ) @@ -45,9 +43,7 @@ class DefaultInstanceSubparser: == dcargs.cli( DefaultInstanceSubparser, args=["--bc.y", "8"], - default=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=7) - ), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) ) @@ -75,18 +71,14 @@ class DefaultInstanceSubparser: == dcargs.cli( dcargs.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--x", "1", "--bc.y", "5"], - default=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=3) - ), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=3)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) ) assert dcargs.cli( DefaultInstanceSubparser, args=["bc:default-instance-smtp-server", "--bc.z", "3"], - default=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=5) - ), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceSMTPServer(z=3)) assert ( dcargs.cli( @@ -96,9 +88,7 @@ class DefaultInstanceSubparser: == dcargs.cli( dcargs.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--bc.y", "8"], - default=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=7) - ), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) ) diff --git a/tests/test_nested.py b/tests/test_nested.py index e69332bc..0bdbf443 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -53,7 +53,7 @@ class Nested: ) -def test_nested_default(): +def test_nested_default_alternate(): @dataclasses.dataclass class B: y: int = 3 @@ -243,7 +243,7 @@ class DefaultSubparser: dcargs.cli(DefaultSubparser, args=["--x", "1", "c", "--bc.y", "3"]) -def test_subparser_with_default(): +def test_subparser_with_default_alternate(): @dataclasses.dataclass class DefaultInstanceHTTPServer: y: int = 0 From af4cf7afbf3ace0f4e6ed922c98c2019e5a2dcf2 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 1 Sep 2022 04:18:15 -0700 Subject: [PATCH 09/19] Strip colors from get_parser(), wrapping tweaks --- dcargs/_argparse_formatter.py | 44 +++++++++++++++++++++++++++++++++++ dcargs/_cli.py | 23 ++++++++++-------- dcargs/_parsers.py | 4 ++-- tests/test_helptext.py | 4 ++-- 4 files changed, 61 insertions(+), 14 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index bfc30292..0322cc92 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -5,6 +5,8 @@ import shutil from typing import Any, ContextManager, Generator +import termcolor + from . import _strings @@ -15,6 +17,22 @@ def monkeypatch_len(obj: Any) -> int: return len(obj) +def dummy_termcolor_context() -> ContextManager[None]: + """Context for turning termcolor off.""" + + def dummy_colored(*args, **kwargs) -> str: + return args[0] + + @contextlib.contextmanager + def inner() -> Generator[None, None, None]: + orig_colored = termcolor.colored + termcolor.colored = dummy_colored + yield + termcolor.colored = orig_colored + + return inner() + + def ansi_context() -> ContextManager[None]: """Context for working with ANSI codes + argparse: - Applies a temporary monkey patch for making argparse ignore ANSI codes when @@ -25,6 +43,7 @@ def ansi_context() -> ContextManager[None]: @contextlib.contextmanager def inner() -> Generator[None, None, None]: if not hasattr(argparse, "len"): + # Sketchy, but seems to work. argparse.len = monkeypatch_len # type: ignore try: # Use Colorama to support coloring in Windows shells. @@ -136,6 +155,7 @@ def _format_action(self, action): ) ) # + parts.append("%*s%s\n" % (indent_first, "", help_lines[0])) # type: ignore for line in help_lines[1:]: parts.append("%*s%s\n" % (help_position, "", line)) @@ -150,3 +170,27 @@ def _format_action(self, action): # return a single string return self._join_parts(parts) + + def _split_lines(self, text, width): + text = self._whitespace_matcher.sub(" ", text).strip() + # The textwrap module is used only for formatting help. + # Delay its import for speeding up the common usage of argparse. + import textwrap as textwrap + + # Sketchy, but seems to work. + textwrap.len = monkeypatch_len # type: ignore + out = textwrap.wrap(text, width) + del textwrap.len # type: ignore + return out + + def _fill_text(self, text, width, indent): + text = self._whitespace_matcher.sub(" ", text).strip() + import textwrap as textwrap + + # Sketchy, but seems to work. + textwrap.len = monkeypatch_len # type: ignore + out = textwrap.fill( + text, width, initial_indent=indent, subsequent_indent=indent + ) + del textwrap.len # type: ignore + return out diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 02ce6465..8849d413 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -152,16 +152,19 @@ def get_parser( was called with the same arguments. This can be useful for libraries like argcomplete, pyzshcomplete, or shtab, which - enable autocompletion for argparse parsers.""" - out = _cli_impl( - "parser", - f, - prog=prog, - description=description, - args=args, - default=default, - **deprecated_kwargs, - ) + enable autocompletion for argparse parsers. + + To reduce compatibility issues, strips out color formatting.""" + with _argparse_formatter.dummy_termcolor_context(): + out = _cli_impl( + "parser", + f, + prog=prog, + description=description, + args=args, + default=default, + **deprecated_kwargs, + ) assert isinstance(out, argparse.ArgumentParser) return out diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 197dc4fb..b09a7b11 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -346,8 +346,8 @@ def from_field( prefix, type(field.default) ) assert default_name in parser_from_name, ( - "Default with type {type(field.default)} was passed in, but no matching" - " subparser." + f"Default with type {type(field.default)} was passed in, but no" + " matching subparser." ) default_parser = parser_from_name[default_name] if any(map(lambda arg: arg.lowered.required, default_parser.args)): diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 1eae5ea1..56ad2963 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -26,9 +26,9 @@ def _get_helptext(f: Callable, args: List[str] = ["--help"]) -> str: with pytest.raises(SystemExit), contextlib.redirect_stdout(target2): with dcargs._argparse_formatter.ansi_context(): dcargs.get_parser(f).parse_args(args) - assert target.getvalue() == target2.getvalue() + assert dcargs._strings.strip_ansi_sequences(target.getvalue()) == target2.getvalue() - return dcargs._strings.strip_ansi_sequences(target.getvalue()) + return target2.getvalue() def test_helptext(): From 79eb7a515a74b7bfc529b516b122f68fa80237ba Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 3 Sep 2022 06:17:29 -0700 Subject: [PATCH 10/19] Test fix, prefix name toggle --- dcargs/_parsers.py | 27 +++++++++++++++++++-------- dcargs/_strings.py | 29 +++++++++++++++++------------ dcargs/conf/_subcommands.py | 11 +++++++---- dcargs/extras/_serialization.py | 2 +- 4 files changed, 44 insertions(+), 25 deletions(-) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 1ce645e5..cf5462ec 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -39,6 +39,7 @@ class ParserSpecification: """Each parser contains a list of arguments and optionally some subparsers.""" + f: Callable description: str args: List[_arguments.ArgumentDefinition] helptext_from_nested_class_field_name: Dict[str, Optional[str]] @@ -190,6 +191,7 @@ def from_callable( ) return ParserSpecification( + f=f, description=description if description is not None else _docstrings.get_callable_description(f), @@ -303,6 +305,7 @@ def from_field( is _resolver.unwrap_origin_strip_extras(option) else _fields.MISSING_NONPROP ), + prefix_name=True, ), ) @@ -333,6 +336,7 @@ def from_field( # If there are any required arguments in the default subparser, we should mark # the subparser group as a whole as required. + default_name = None if ( field.default is not None and field.default not in _fields.MISSING_SINGLETONS @@ -347,10 +351,20 @@ def from_field( default_name = _strings.subparser_name_from_type( prefix, type(field.default) ) - assert default_name in parser_from_name, ( - f"Default with type {type(field.default)} was passed in, but no" - " matching subparser." - ) + if default_name not in parser_from_name: + # If we can't find the subparser by name, search by type. This is needed + # when the user renames their subcommands. (eg via dcargs.conf.subcommand) + default_name = None + for name, parser in parser_from_name.items(): + if type(field.default) is _resolver.unwrap_origin_strip_extras( + parser.f + ): + default_name = name + break + assert default_name is not None, ( + f"Default with type {type(field.default)} was passed in, but no" + " matching subparser." + ) default_parser = parser_from_name[default_name] if any(map(lambda arg: arg.lowered.required, default_parser.args)): required = True @@ -367,10 +381,7 @@ def from_field( if field.helptext is not None: description_parts.append(field.helptext) if not required and field.default not in _fields.MISSING_SINGLETONS: - default = field.default - if default is not None: - default = _strings.subparser_name_from_type(prefix, type(default)) - description_parts.append(f" (default: {default})") + description_parts.append(f" (default: {default_name})") description = ( # We use `None` instead of an empty string to prevent a line break from # being created where the description would be. diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 4c30c1a2..15e5bfcd 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -3,7 +3,7 @@ import functools import re import textwrap -from typing import Iterable, List, Sequence, Type, Union +from typing import Iterable, List, Sequence, Tuple, Type, Union import termcolor @@ -53,7 +53,7 @@ def hyphen_separated_from_camel_case(name: str) -> str: return _camel_separator_pattern().sub(r"-\1", name).lower() -def _subparser_name_from_type(cls: Type) -> str: +def _subparser_name_from_type(cls: Type) -> Tuple[str, bool]: from .conf import _subcommands # Prevent circular imports cls, type_from_typevar = _resolver.resolve_generic_types(cls) @@ -63,24 +63,29 @@ def _subparser_name_from_type(cls: Type) -> str: # Subparser name from `dcargs.metadata.subcommand()`. if len(found_subcommand_configs) > 0: - return found_subcommand_configs[0].name + return found_subcommand_configs[0].name, found_subcommand_configs[0].prefix_name # Subparser name from class name. if len(type_from_typevar) == 0: assert hasattr(cls, "__name__") - return hyphen_separated_from_camel_case(cls.__name__) # type: ignore - - return "-".join( - map( - _subparser_name_from_type, - [cls] + list(type_from_typevar.values()), - ) + return hyphen_separated_from_camel_case(cls.__name__), True # type: ignore + + return ( + "-".join( + map( + lambda x: _subparser_name_from_type(x)[0], + [cls] + list(type_from_typevar.values()), + ) + ), + True, ) def subparser_name_from_type(prefix: str, cls: Union[Type, None]) -> str: - suffix = _subparser_name_from_type(cls) if cls is not None else "None" - if len(prefix) == 0: + suffix, use_prefix = ( + _subparser_name_from_type(cls) if cls is not None else ("None", True) + ) + if len(prefix) == 0 or not use_prefix: return suffix return f"{prefix}:{suffix}".replace("_", "-") diff --git a/dcargs/conf/_subcommands.py b/dcargs/conf/_subcommands.py index e5a31957..03d99661 100644 --- a/dcargs/conf/_subcommands.py +++ b/dcargs/conf/_subcommands.py @@ -9,6 +9,7 @@ class _SubcommandConfiguration: name: str default: Any description: Optional[str] + prefix_name: bool def __hash__(self) -> int: return object.__hash__(self) @@ -19,6 +20,7 @@ def subcommand( *, default: Any = MISSING_NONPROP, description: Optional[str] = None, + prefix_name: bool = True, ) -> Any: """Returns a metadata object for configuring subcommands with `typing.Annotated`. This is useful but can make code harder to read, so usage is discouraged. @@ -34,19 +36,20 @@ def subcommand( This will create two subcommands: `nested-type-a` and `nested-type-b`. Annotating each type with `dcargs.metadata.subcommand()` allows us to override for - each subcommand the (a) name, (b) defaults, and (c) helptext. + each subcommand the (a) name, (b) defaults, (c) helptext, and (d) whether to prefix + the name or not. ```python dcargs.cli( Union[ Annotated[ - NestedTypeA, subcommand("a", default=NestedTypeA(...), description="...") + NestedTypeA, subcommand("a", ...) ], Annotated[ - NestedTypeA, subcommand("b", default=NestedTypeA(...), description="...") + NestedTypeA, subcommand("b", ...) ], ] ) ``` """ - return _SubcommandConfiguration(name, default, description) + return _SubcommandConfiguration(name, default, description, prefix_name) diff --git a/dcargs/extras/_serialization.py b/dcargs/extras/_serialization.py index c5c433a4..6dc9931e 100644 --- a/dcargs/extras/_serialization.py +++ b/dcargs/extras/_serialization.py @@ -44,12 +44,12 @@ def _get_contained_special_types_from_type( else _parent_contained_dataclasses ) + cls, _ = _resolver.unwrap_annotated(cls) cls, type_from_typevar = _resolver.resolve_generic_types(cls) contained_dataclasses = {cls} def handle_type(typ) -> Set[Type]: - print(typ) # Handle dataclasses. if _resolver.is_dataclass(typ) and typ not in parent_contained_dataclasses: return _get_contained_special_types_from_type( From cfbf61869297bb1611e26fbb140f658d1a335175 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 4 Sep 2022 10:40:07 -0700 Subject: [PATCH 11/19] Generalize positional logic --- dcargs/_arguments.py | 10 ++-- dcargs/_calling.py | 2 +- dcargs/_cli.py | 1 - dcargs/_fields.py | 103 +++++++++++++++++++++++++++------------- dcargs/_parsers.py | 36 ++++---------- dcargs/conf/__init__.py | 8 ++-- dcargs/conf/_markers.py | 8 +++- 7 files changed, 97 insertions(+), 71 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 4df17a39..dfb8d08d 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -138,7 +138,7 @@ def _rule_handle_boolean_flags( if ( arg.field.default in _fields.MISSING_SINGLETONS - or arg.field.positional + or arg.field.is_positional() or _markers.FLAG_CONVERSION_OFF in arg.field.markers ): # Treat bools as a normal parameter. @@ -260,9 +260,9 @@ def _rule_generate_helptext( # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string docstring_help = docstring_help.replace("%", "%%") help_parts.append(docstring_help) - elif arg.field.positional and arg.field.name != _strings.dummy_field_name: + elif arg.field.is_positional() and arg.field.name != _strings.dummy_field_name: # Place the type in the helptext. Note that we skip this for dummy fields, which - # will sitll have the type in the metavar. + # will still have the type in the metavar. help_parts.append(str(lowered.metavar)) default = lowered.default @@ -309,7 +309,7 @@ def _rule_set_name_or_flag( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, ) -> LoweredArgumentDefinition: - if arg.field.positional: + if arg.field.is_positional(): name_or_flag = _strings.make_field_name([arg.prefix, arg.field.name]) elif lowered.action == "store_false": name_or_flag = "--" + _strings.make_field_name( @@ -331,7 +331,7 @@ def _rule_positional_special_handling( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, ) -> LoweredArgumentDefinition: - if not arg.field.positional: + if not arg.field.is_positional(): return lowered metavar = _strings.make_field_name([arg.prefix, arg.field.name]).upper() diff --git a/dcargs/_calling.py b/dcargs/_calling.py index d20656ca..1c979eca 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -163,7 +163,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: consumed_keywords |= consumed_keywords_child if value is not _fields.EXCLUDE_FROM_CALL: - if field.positional: + if field.is_positional(): args.append(value) else: kwargs[ diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 8849d413..64dc45e2 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -251,7 +251,6 @@ def _cli_impl( description=description, parent_classes=set(), # Used for recursive calls. parent_type_from_typevar=None, # Used for recursive calls. - parent_markers=(), # Used for recursive calls. default_instance=default_instance_internal, # Overrides for default values. prefix="", # Used for recursive calls. ) diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 87319bf2..95048efc 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -11,11 +11,23 @@ import itertools import typing import warnings -from typing import Any, Callable, Hashable, Iterable, List, Optional, Type, Union, cast +from typing import ( + Any, + Callable, + FrozenSet, + Hashable, + Iterable, + List, + Optional, + Tuple, + Type, + Union, + cast, +) import docstring_parser import typing_extensions -from typing_extensions import get_args, get_type_hints, is_typeddict +from typing_extensions import Annotated, get_args, get_type_hints, is_typeddict from . import conf # Avoid circular import. from . import _docstrings, _instantiators, _resolver, _singleton, _strings @@ -28,22 +40,49 @@ class FieldDefinition: typ: Type default: Any helptext: Optional[str] - positional: bool - markers: List[_markers.Marker] = dataclasses.field(default_factory=list) + markers: FrozenSet[_markers.Marker] # Override the name in our kwargs. Currently only used for dictionary types when # the key values aren't strings, but in the future could be used whenever the # user-facing argument name doesn't match the keyword expected by our callable. - name_override: Optional[Any] = None - - def __post_init__(self): # - # Auto-populate markers if unset; this is meant to not run when we do - # dataclasses.replace, etc. TODO: the whole markers design, handling of - # Annotated[], etc, should be revisited... - if len(self.markers) == 0: - self.markers.extend( - _resolver.unwrap_annotated(self.typ, _markers.Marker)[1] - ) + name_override: Optional[Any] + + @staticmethod + def make( + name: str, + typ: Type, + default: Any, + helptext: Optional[str], + *, + markers: Tuple[_markers.Marker, ...] = (), + name_override: Optional[Any] = None, + ): + _, inferred_markers = _resolver.unwrap_annotated(typ, _markers.Marker) + return FieldDefinition( + name, + typ, + default, + helptext, + frozenset(inferred_markers).union(markers), + name_override, + ) + + def add_markers(self, markers: Tuple[_markers.Marker, ...]) -> FieldDefinition: + if len(markers) == 0: + return self + return dataclasses.replace( + self, + typ=Annotated.__class_getitem__((self.typ,) + markers), # type: ignore + markers=self.markers.union(markers), + ) + + def is_positional(self) -> bool: + return ( + # Explicit positionals. + _markers.POSITIONAL in self.markers + # Dummy dataclasses should have a single positional field. + or self.name == _strings.dummy_field_name + ) class PropagatingMissingType(_singleton.Singleton): @@ -115,6 +154,10 @@ def field_list_from_callable( if isinstance(out, UnsupportedNestedTypeMessage): raise _instantiators.UnsupportedTypeAnnotationError(out.message) + + # Recursively apply markers. + _, parent_markers = _resolver.unwrap_annotated(f, _markers.Marker) + out = list(map(lambda field: field.add_markers(parent_markers), out)) return out @@ -249,12 +292,11 @@ def _try_field_list_from_typeddict( default = MISSING_PROP field_list.append( - FieldDefinition( + FieldDefinition.make( name=name, typ=typ, default=default, helptext=_docstrings.get_field_docstring(cls, name), - positional=False, ) ) return field_list @@ -281,12 +323,11 @@ def _try_field_list_from_namedtuple( default = MISSING_PROP field_list.append( - FieldDefinition( + FieldDefinition.make( name=name, typ=typ, default=default, helptext=_docstrings.get_field_docstring(cls, name), - positional=False, ) ) return field_list @@ -300,14 +341,11 @@ def _try_field_list_from_dataclass( for dc_field in filter(lambda field: field.init, _resolver.resolved_fields(cls)): default = _get_dataclass_field_default(dc_field, default_instance) field_list.append( - FieldDefinition( + FieldDefinition.make( name=dc_field.name, typ=dc_field.type, default=default, helptext=_docstrings.get_field_docstring(cls, dc_field.name), - # Only mark positional if using a dummy field, for taking single types - # directly as input. - positional=dc_field.name == _strings.dummy_field_name, ) ) return field_list @@ -346,7 +384,7 @@ def _field_list_from_tuple( for i, child in enumerate(children): default_i = default_instance[i] # type: ignore field_list.append( - FieldDefinition( + FieldDefinition.make( # Ideally we'd have --tuple[0] instead of --tuple.0 as the command-line # argument, but in practice the brackets are annoying because they # require escaping. @@ -354,10 +392,9 @@ def _field_list_from_tuple( typ=child, default=default_i, helptext="", - # This should really be positional=True, but the CLI is more - # intuitive for mixed nested/non-nested types in tuples when we - # stick with kwargs. Tuples are special-cased in _calling.py. - positional=False, + # This should really set the positional marker, but the CLI is more + # intuitive for mixed nested/non-nested types in tuples when we stick + # with kwargs. Tuples are special-cased in _calling.py. ) ) @@ -408,12 +445,11 @@ def _try_field_list_from_sequence( field_list = [] for i, default_i in enumerate(default_instance): # type: ignore field_list.append( - FieldDefinition( + FieldDefinition.make( name=str(i), typ=contained_type, default=default_i, helptext="", - positional=False, ) ) return field_list @@ -430,12 +466,11 @@ def _try_field_list_from_dict( field_list = [] for k, v in cast(dict, default_instance).items(): field_list.append( - FieldDefinition( + FieldDefinition.make( name=str(k) if not isinstance(k, enum.Enum) else k.name, typ=type(v), default=v, helptext=None, - positional=False, # Dictionary specific key: name_override=k, ) @@ -514,13 +549,15 @@ def _field_list_from_params( return out field_list.append( - FieldDefinition( + FieldDefinition.make( name=param.name, # Note that param.annotation does not resolve forward references. typ=hints[param.name], default=default, helptext=helptext, - positional=param.kind is inspect.Parameter.POSITIONAL_ONLY, + markers=(_markers.POSITIONAL,) + if param.kind is inspect.Parameter.POSITIONAL_ONLY + else (), ) ) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index cf5462ec..d29f03d8 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -4,19 +4,7 @@ import argparse import dataclasses import itertools -from typing import ( - Any, - Callable, - Dict, - List, - Optional, - Set, - Tuple, - Type, - TypeVar, - Union, - cast, -) +from typing import Any, Callable, Dict, List, Optional, Set, Type, TypeVar, Union, cast import termcolor from typing_extensions import get_args, get_origin @@ -29,8 +17,8 @@ _instantiators, _resolver, _strings, - conf, ) +from .conf import _markers, _subcommands T = TypeVar("T") @@ -52,7 +40,6 @@ def from_callable( description: Optional[str], parent_classes: Set[Type], parent_type_from_typevar: Optional[Dict[TypeVar, Type]], - parent_markers: Tuple[conf._markers.Marker, ...], default_instance: Union[ T, _fields.PropagatingMissingType, _fields.NonpropagatingMissingType ], @@ -61,9 +48,6 @@ def from_callable( """Create a parser definition from a callable.""" # Resolve generic types. - markers = ( - parent_markers + _resolver.unwrap_annotated(f, conf._markers.Marker)[1] - ) f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.narrow_type(f, default_instance) if parent_type_from_typevar is not None: @@ -95,6 +79,7 @@ def from_callable( for field in field_list: field = dataclasses.replace( field, + # Resolve generic types. typ=_resolver.type_from_typevar_constraints( _resolver.apply_type_from_typevar( field.typ, @@ -102,14 +87,13 @@ def from_callable( ) ), ) - field.markers.extend(markers) # TODO: would be nice to avoid this mutation! if isinstance(field.typ, TypeVar): raise _instantiators.UnsupportedTypeAnnotationError( f"Field {field.name} has an unbound TypeVar: {field.typ}." ) - if conf._markers.FIXED not in field.markers: + if _markers.FIXED not in field.markers: # (1) Handle Unions over callables; these result in subparsers. subparsers_attempt = SubparsersSpecification.from_field( field, @@ -120,7 +104,7 @@ def from_callable( if subparsers_attempt is not None: if ( not subparsers_attempt.required - and conf._markers.AVOID_SUBCOMMANDS in field.markers + and _markers.AVOID_SUBCOMMANDS in field.markers ): # Don't make a subparser. field = dataclasses.replace(field, typ=type(field.default)) @@ -144,7 +128,6 @@ def from_callable( description=None, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, - parent_markers=markers, default_instance=field.default, prefix=_strings.make_field_name([prefix, field.name]), ) @@ -226,7 +209,7 @@ def format_group_name(nested_field_name: str) -> str: # Add each argument. for arg in self.args: - if arg.field.positional: + if arg.field.is_positional(): arg.add_argument(positional_group) continue @@ -291,12 +274,12 @@ def from_field( for option in options_no_none: name = _strings.subparser_name_from_type(prefix, option) option, found_subcommand_configs = _resolver.unwrap_annotated( - option, conf._subcommands._SubcommandConfiguration + option, _subcommands._SubcommandConfiguration ) if len(found_subcommand_configs) == 0: # Make a dummy subcommand config. found_subcommand_configs = ( - conf._subcommands._SubcommandConfiguration( + _subcommands._SubcommandConfiguration( "unused", description=None, default=( @@ -314,7 +297,6 @@ def from_field( description=found_subcommand_configs[0].description, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, - parent_markers=tuple(field.markers), default_instance=found_subcommand_configs[0].default, prefix=prefix, ) @@ -353,7 +335,7 @@ def from_field( ) if default_name not in parser_from_name: # If we can't find the subparser by name, search by type. This is needed - # when the user renames their subcommands. (eg via dcargs.conf.subcommand) + # when the user renames their subcommands. (eg via dcargs.subcommand) default_name = None for name, parser in parser_from_name.items(): if type(field.default) is _resolver.unwrap_origin_strip_extras( diff --git a/dcargs/conf/__init__.py b/dcargs/conf/__init__.py index 7415c82e..e1a1743e 100644 --- a/dcargs/conf/__init__.py +++ b/dcargs/conf/__init__.py @@ -1,15 +1,17 @@ """The :mod:`dcargs.conf` submodule contains helpers for attaching parsing-specific -configuration metadata to types via PEP 593 runtime annotations. +configuration metadata to types via [PEP 593](https://peps.python.org/pep-0593/) runtime +annotations. -Features here are supported, but are generally unnecessary and should be used sparingly. +Features here are supported, but generally unnecessary and should be used sparingly. """ -from ._markers import AvoidSubcommands, Fixed, FlagConversionOff +from ._markers import AvoidSubcommands, Fixed, FlagConversionOff, Positional from ._subcommands import subcommand __all__ = [ "AvoidSubcommands", "Fixed", "FlagConversionOff", + "Positional", "subcommand", ] diff --git a/dcargs/conf/_markers.py b/dcargs/conf/_markers.py index 91fa24d4..3e610065 100644 --- a/dcargs/conf/_markers.py +++ b/dcargs/conf/_markers.py @@ -22,9 +22,15 @@ def __repr__(self): T = TypeVar("T", bound=Type) +POSITIONAL = _make_marker("Positional") +Positional = Annotated[T, POSITIONAL] +"""A type `T` can be annotated as `Positional[T]` if we want to parse it as a positional +argument.""" + + FIXED = _make_marker("Fixed") Fixed = Annotated[T, FIXED] -"""A type `T` can be annotated as `Fixed[T]` if we don't want dcargs to parse it. A +"""A type `T` can be annotated as `Fixed[T]` to prevent `dcargs.cli` from parsing it. A default value should be set instead.""" FLAG_CONVERSION_OFF = _make_marker("FlagConversionOff") From 87d4ad49b0deb62922d5a3d205e68e5286477674 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 5 Sep 2022 21:53:27 +0100 Subject: [PATCH 12/19] Replace `get_parser()` with `--dcargs-print-completion` --- dcargs/__init__.py | 3 +- dcargs/_argparse_formatter.py | 11 +-- dcargs/_cli.py | 144 ++++++++-------------------------- dcargs/_deprecated.py | 1 - dcargs/_instantiators.py | 12 ++- examples/01_functions.py | 0 poetry.lock | 58 +++++++++++--- pyproject.toml | 1 + 8 files changed, 85 insertions(+), 145 deletions(-) mode change 100644 => 100755 examples/01_functions.py diff --git a/dcargs/__init__.py b/dcargs/__init__.py index 3073b8b2..6f225244 100644 --- a/dcargs/__init__.py +++ b/dcargs/__init__.py @@ -1,5 +1,5 @@ from . import conf, extras -from ._cli import cli, get_parser +from ._cli import cli from ._fields import MISSING_PUBLIC as MISSING from ._instantiators import UnsupportedTypeAnnotationError @@ -7,7 +7,6 @@ "conf", "extras", "cli", - "get_parser", "MISSING", "UnsupportedTypeAnnotationError", ] diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index 0322cc92..a6dac359 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -184,13 +184,4 @@ def _split_lines(self, text, width): return out def _fill_text(self, text, width, indent): - text = self._whitespace_matcher.sub(" ", text).strip() - import textwrap as textwrap - - # Sketchy, but seems to work. - textwrap.len = monkeypatch_len # type: ignore - out = textwrap.fill( - text, width, initial_indent=indent, subsequent_indent=indent - ) - del textwrap.len # type: ignore - return out + return "".join(indent + line for line in text.splitlines(keepends=True)) diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 64dc45e2..913a0ed2 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -1,10 +1,11 @@ """Core public API.""" import argparse import dataclasses +import sys import warnings from typing import Callable, Optional, Sequence, Type, TypeVar, Union, cast, overload -from typing_extensions import Literal, assert_never +import shtab from . import _argparse_formatter, _calling, _fields, _parsers, _strings, conf @@ -103,110 +104,6 @@ def cli( Returns: The output of `f(...)`. """ - out = _cli_impl( - "f_out", - f, - prog=prog, - description=description, - args=args, - default=default, - **deprecated_kwargs, - ) - return out - - -@overload -def get_parser( - f: Type[OutT], - *, - prog: Optional[str] = None, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, - default: Optional[OutT] = None, -) -> argparse.ArgumentParser: - ... - - -@overload -def get_parser( - f: Callable[..., OutT], - *, - prog: Optional[str] = None, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, - default: Optional[OutT] = None, -) -> argparse.ArgumentParser: - ... - - -def get_parser( - f: Union[Type[OutT], Callable[..., OutT]], - *, - prog: Optional[str] = None, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, - default: Optional[OutT] = None, - **deprecated_kwargs, -) -> argparse.ArgumentParser: - """Returns the argparse parser that would be used under-the-hood if `dcargs.cli()` - was called with the same arguments. - - This can be useful for libraries like argcomplete, pyzshcomplete, or shtab, which - enable autocompletion for argparse parsers. - - To reduce compatibility issues, strips out color formatting.""" - with _argparse_formatter.dummy_termcolor_context(): - out = _cli_impl( - "parser", - f, - prog=prog, - description=description, - args=args, - default=default, - **deprecated_kwargs, - ) - assert isinstance(out, argparse.ArgumentParser) - return out - - -@overload -def _cli_impl( - _return_stage: Literal["parser"], - f: Callable[..., OutT], - *, - prog: Optional[str] = None, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, - default: Optional[OutT] = None, - **deprecated_kwargs, -) -> argparse.ArgumentParser: - ... - - -@overload -def _cli_impl( - _return_stage: Literal["f_out"], - f: Callable[..., OutT], - *, - prog: Optional[str] = None, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, - default: Optional[OutT] = None, - **deprecated_kwargs, -) -> OutT: - ... - - -def _cli_impl( - _return_stage: Literal["parser", "f_out"], - f: Callable[..., OutT], - *, - prog: Optional[str] = None, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, - default: Optional[OutT] = None, - **deprecated_kwargs, -) -> Union[OutT, argparse.ArgumentParser]: if "default_instance" in deprecated_kwargs: warnings.warn( "`default_instance=` is deprecated! use `default=` instead.", stacklevel=2 @@ -255,8 +152,24 @@ def _cli_impl( prefix="", # Used for recursive calls. ) + # If we pass in the --dcargs-print-completion flag: turn termcolor off, and ge the + # shell we want to generate a completion script for (bash/zsh/tcsh). + args = sys.argv[1:] if args is None else args + print_completion = len(args) >= 2 and args[0] == "--dcargs-print-completion" + + formatting_context = _argparse_formatter.ansi_context() + completion_shell = None + if print_completion: + formatting_context = _argparse_formatter.dummy_termcolor_context() + completion_shell = args[1] + assert completion_shell in ( + "bash", + "zsh", + "tcsh", + ), f"Shell should be one `bash`, `zsh`, or `tcsh`, but got {completion_shell}" + # Generate parser! - with _argparse_formatter.ansi_context(): + with formatting_context: parser = argparse.ArgumentParser( prog=prog, formatter_class=_argparse_formatter.make_formatter_class( @@ -264,9 +177,19 @@ def _cli_impl( ), ) parser_definition.apply(parser) - if _return_stage == "parser": - return parser + + if print_completion: + print( + shtab.complete( + parser=parser, + shell=completion_shell, + root_prefix=f"dcargs_{parser.prog}", + ) + ) + raise SystemExit() + value_from_prefixed_field_name = vars(parser.parse_args(args=args)) + value_from_prefixed_field_name.pop("dcargs_print_completion") if dummy_wrapped: value_from_prefixed_field_name = { @@ -297,7 +220,4 @@ def _cli_impl( if dummy_wrapped: out = getattr(out, _strings.dummy_field_name) - if _return_stage == "f_out": - return out - - assert_never(_return_stage) + return out diff --git a/dcargs/_deprecated.py b/dcargs/_deprecated.py index 05e32b46..c879522a 100644 --- a/dcargs/_deprecated.py +++ b/dcargs/_deprecated.py @@ -1,3 +1,2 @@ from ._cli import cli as parse # noqa -from ._cli import get_parser as generate_parser # noqa from .extras._serialization import from_yaml, to_yaml # noqa diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py index 4a0fd20b..d7ff78c4 100644 --- a/dcargs/_instantiators.py +++ b/dcargs/_instantiators.py @@ -22,13 +22,11 @@ Tuple[int, float] lambda strings: tuple( - [ - typ(x) - for typ, x in zip( - (int, float), - strings, - ) - ] + typ(x) + for typ, x in zip( + (int, float), + strings, + ) ) ``` """ diff --git a/examples/01_functions.py b/examples/01_functions.py old mode 100644 new mode 100755 diff --git a/poetry.lock b/poetry.lock index 2ee84552..14a89181 100644 --- a/poetry.lock +++ b/poetry.lock @@ -23,10 +23,10 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] +dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "sphinx", "sphinx-notfound-page", "zope.interface"] +docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] +tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "zope.interface"] +tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six"] [[package]] name = "backports.cached-property" @@ -87,9 +87,9 @@ typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] +docs = ["jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] [[package]] name = "iniconfig" @@ -134,6 +134,9 @@ category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +[package.dependencies] +setuptools = "*" + [[package]] name = "numpy" version = "1.21.1" @@ -177,8 +180,8 @@ python-versions = ">=3.6" importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} [package.extras] -testing = ["pytest-benchmark", "pytest"] -dev = ["tox", "pre-commit"] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] [[package]] name = "py" @@ -197,7 +200,7 @@ optional = false python-versions = ">=3.6.8" [package.extras] -diagrams = ["railroad-diagrams", "jinja2"] +diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pyright" @@ -250,7 +253,7 @@ coverage = {version = ">=5.2.1", extras = ["toml"]} pytest = ">=4.6" [package.extras] -testing = ["virtualenv", "pytest-xdist", "six", "process-tests", "hunter", "fields"] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] [[package]] name = "pyyaml" @@ -260,6 +263,27 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "setuptools" +version = "65.3.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "shtab" +version = "1.5.5" +description = "Automagic shell tab completion for Python CLI applications" +category = "main" +optional = false +python-versions = ">=3.2" + [[package]] name = "termcolor" version = "1.1.0" @@ -312,13 +336,13 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx"] +testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "897e78d65e9a00cd292034a23eb35922e934052dbba5b927795e39cf21b3f449" +content-hash = "880ee7ca4570c69fe1fadfac1b13d81c6719eaf7b45420bbb587985f9e9ef5ed" [metadata.files] antlr4-python3-runtime = [ @@ -552,6 +576,14 @@ pyyaml = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +setuptools = [ + {file = "setuptools-65.3.0-py3-none-any.whl", hash = "sha256:2e24e0bec025f035a2e72cdd1961119f557d78ad331bb00ff82efb2ab8da8e82"}, + {file = "setuptools-65.3.0.tar.gz", hash = "sha256:7732871f4f7fa58fb6bdcaeadb0161b2bd046c85905dbaa066bdcbcc81953b57"}, +] +shtab = [ + {file = "shtab-1.5.5-py2.py3-none-any.whl", hash = "sha256:f4bf7cc122fb434cb65b96db7e3adcf3b5258846e906e60c48b3725d2965c6b5"}, + {file = "shtab-1.5.5.tar.gz", hash = "sha256:f90a6ce64b821002d5881b6212992a27ab40c3bab36aabca8de118b0b78f61f6"}, +] termcolor = [ {file = "termcolor-1.1.0.tar.gz", hash = "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"}, ] diff --git a/pyproject.toml b/pyproject.toml index bd329a31..9e382d90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ termcolor = "^1.1.0" "backports.cached-property" = { version = "^1.0.2", python = "~3.7" } colorama = {version = "^0.4.0", platform = "win32"} frozendict = "^2.3.4" +shtab = "^1.5.5" [tool.poetry.dev-dependencies] pytest = "^7.1.2" From 519abfeeae57d1a4e065e7287fdcf162830d5200 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 6 Sep 2022 21:05:02 -0700 Subject: [PATCH 13/19] Update tests with new completion logic --- tests/test_dcargs.py | 2 -- tests/test_helptext.py | 10 ++++---- tests/test_print_completion.py | 43 ++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 7 deletions(-) create mode 100644 tests/test_print_completion.py diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 656f1aa8..ede75c57 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -40,8 +40,6 @@ class ManyTypes: f: float p: pathlib.Path - assert isinstance(dcargs.get_parser(ManyTypes), argparse.ArgumentParser) - # We can directly pass a dataclass to `dcargs.cli()`: assert dcargs.cli( ManyTypes, diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 56ad2963..bbfbde39 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -20,14 +20,14 @@ def _get_helptext(f: Callable, args: List[str] = ["--help"]) -> str: with pytest.raises(SystemExit), contextlib.redirect_stdout(target): dcargs.cli(f, args=args) - # Check against dcargs.generate_parser(); this should return the same underlying - # parser. + # Check helptext with vs without termcolor. This can help catch text wrapping bugs + # caused by ANSI sequences. target2 = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target2): - with dcargs._argparse_formatter.ansi_context(): - dcargs.get_parser(f).parse_args(args) - assert dcargs._strings.strip_ansi_sequences(target.getvalue()) == target2.getvalue() + with dcargs._argparse_formatter.dummy_termcolor_context(): + dcargs.cli(f, args=args) + assert target2.getvalue() == dcargs._strings.strip_ansi_sequences(target.getvalue()) return target2.getvalue() diff --git a/tests/test_print_completion.py b/tests/test_print_completion.py new file mode 100644 index 00000000..9f4c34d9 --- /dev/null +++ b/tests/test_print_completion.py @@ -0,0 +1,43 @@ +import contextlib +import dataclasses +import io +from typing import Union + +import pytest + +import dcargs + + +# https://github.com/brentyi/dcargs/issues/9 +@dataclasses.dataclass(frozen=True) +class Subtype: + data: int = 1 + + +@dataclasses.dataclass(frozen=True) +class TypeA: + subtype: Subtype = Subtype(1) + + +@dataclasses.dataclass(frozen=True) +class TypeB: + subtype: Subtype = Subtype(2) + + +@dataclasses.dataclass(frozen=True) +class Wrapper: + supertype: Union[TypeA, TypeB] = TypeA() + + +def test_bash(): + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + dcargs.cli(Wrapper, args=["--dcargs-print-completion", "bash"]) + assert "# AUTOMATCALLY GENERATED by `shtab`" in target.getvalue() + + +def test_zsh(): + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + dcargs.cli(Wrapper, args=["--dcargs-print-completion", "zsh"]) + assert "# AUTOMATCALLY GENERATED by `shtab`" in target.getvalue() From c81f6d93b9b6264163c223dcad3410ac71d4bc3e Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 6 Sep 2022 21:13:20 -0700 Subject: [PATCH 14/19] Fix errors from old completion logic --- dcargs/_cli.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 913a0ed2..41e58268 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -162,11 +162,6 @@ def cli( if print_completion: formatting_context = _argparse_formatter.dummy_termcolor_context() completion_shell = args[1] - assert completion_shell in ( - "bash", - "zsh", - "tcsh", - ), f"Shell should be one `bash`, `zsh`, or `tcsh`, but got {completion_shell}" # Generate parser! with formatting_context: @@ -179,6 +174,11 @@ def cli( parser_definition.apply(parser) if print_completion: + assert completion_shell in ( + "bash", + "zsh", + "tcsh", + ), f"Shell should be one `bash`, `zsh`, or `tcsh`, but got {completion_shell}" print( shtab.complete( parser=parser, @@ -189,7 +189,6 @@ def cli( raise SystemExit() value_from_prefixed_field_name = vars(parser.parse_args(args=args)) - value_from_prefixed_field_name.pop("dcargs_print_completion") if dummy_wrapped: value_from_prefixed_field_name = { From a0e19520418dbb484bcece944b3e4a24dc38aae7 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 6 Sep 2022 21:46:53 -0700 Subject: [PATCH 15/19] Improve + deprecate serialization helpers :floppy_disk: --- dcargs/_cli.py | 12 +++-- dcargs/extras/_serialization.py | 60 ++++++++++++++--------- docs/source/serialization.rst | 22 --------- tests/test_generics_and_serialization.py | 61 ++++++++++++++++++------ 4 files changed, 90 insertions(+), 65 deletions(-) delete mode 100644 docs/source/serialization.rst diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 41e58268..670c4bd9 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -127,6 +127,9 @@ def cli( _fields.MISSING_NONPROP if default is None else default ) + # We wrap our type with a dummy dataclass if it can't be treated as a nested type. + # For example: passing in f=int will result in a dataclass with a single field + # typed as int. if not _fields.is_nested_type(cast(Type, f), default_instance_internal): dummy_field = cast( dataclasses.Field, @@ -174,11 +177,10 @@ def cli( parser_definition.apply(parser) if print_completion: - assert completion_shell in ( - "bash", - "zsh", - "tcsh", - ), f"Shell should be one `bash`, `zsh`, or `tcsh`, but got {completion_shell}" + assert completion_shell in ("bash", "zsh", "tcsh",), ( + "Shell should be one `bash`, `zsh`, or `tcsh`, but got" + f" {completion_shell}" + ) print( shtab.complete( parser=parser, diff --git a/dcargs/extras/_serialization.py b/dcargs/extras/_serialization.py index 6dc9931e..333f2b32 100644 --- a/dcargs/extras/_serialization.py +++ b/dcargs/extras/_serialization.py @@ -17,20 +17,6 @@ DataclassType = TypeVar("DataclassType") -def _get_contained_special_types_from_instance(instance: Any) -> Set[Type]: - """Takes an object and recursively searches its cihldren for dataclass or enum - types.""" - if issubclass(type(instance), enum.Enum): - return {type(instance)} - elif not dataclasses.is_dataclass(instance): - return set() - - out = {type(instance)} - for v in vars(instance).values(): - out |= _get_contained_special_types_from_instance(v) - return out - - def _get_contained_special_types_from_type( cls: Type, _parent_contained_dataclasses: Optional[Set[Type]] = None, @@ -47,14 +33,14 @@ def _get_contained_special_types_from_type( cls, _ = _resolver.unwrap_annotated(cls) cls, type_from_typevar = _resolver.resolve_generic_types(cls) - contained_dataclasses = {cls} + contained_special_types = {cls} - def handle_type(typ) -> Set[Type]: + def handle_type(typ: Type) -> Set[Type]: # Handle dataclasses. if _resolver.is_dataclass(typ) and typ not in parent_contained_dataclasses: return _get_contained_special_types_from_type( typ, - _parent_contained_dataclasses=contained_dataclasses + _parent_contained_dataclasses=contained_special_types | parent_contained_dataclasses, ) @@ -67,16 +53,20 @@ def handle_type(typ) -> Set[Type]: # Handle generics. for typ in type_from_typevar.values(): - contained_dataclasses |= handle_type(typ) + contained_special_types |= handle_type(typ) if cls in parent_contained_dataclasses: - return contained_dataclasses + return contained_special_types # Handle fields. for field in _resolver.resolved_fields(cls): # type: ignore - contained_dataclasses |= handle_type(field.type) + contained_special_types |= handle_type(field.type) - return contained_dataclasses + # Handle subclasses. + for subclass in cls.__subclasses__(): + contained_special_types |= handle_type(subclass) + + return contained_special_types def _make_loader(cls: Type) -> Type[yaml.Loader]: @@ -134,7 +124,7 @@ class DataclassDumper(yaml.Dumper): def ignore_aliases(self, data): return super().ignore_aliases(data) or data is _fields.MISSING_PROP - contained_types = list(_get_contained_special_types_from_instance(instance)) + contained_types = list(_get_contained_special_types_from_type(type(instance))) contained_type_names = list(map(lambda cls: cls.__name__, contained_types)) # Note: this is currently a stricter than necessary assert. @@ -183,7 +173,19 @@ def from_yaml( stream: Union[str, IO[str], bytes, IO[bytes]], ) -> DataclassType: """Re-construct a dataclass instance from a yaml-compatible string, which should be - generated from `dcargs.extra.to_yaml()`. + generated from `dcargs.extras.to_yaml()`. + + As a secondary feature aimed at enabling the use of :func:`dcargs.cli` for general + configuration use cases, we also introduce functions for human-readable dataclass + serialization: :func:`dcargs.conf.from_yaml` and :func:`dcargs.conf.to_yaml` attempt + to strike a balance between flexibility and robustness — in contrast to naively + dumping or loading dataclass instances (via pickle, PyYAML, etc), explicit type + references enable custom tags that are robust against code reorganization and + refactor, while a PyYAML backend enables serialization of arbitrary Python objects. + + .. warning:: + Serialization functionality is deprecated. It may be removed in a future version + of :code:`dcargs`. Args: cls: Type to reconstruct. @@ -202,6 +204,18 @@ def to_yaml(instance: Any) -> str: """Serialize a dataclass; returns a yaml-compatible string that can be deserialized via `dcargs.extras.from_yaml()`. + As a secondary feature aimed at enabling the use of :func:`dcargs.cli` for general + configuration use cases, we also introduce functions for human-readable dataclass + serialization: :func:`dcargs.conf.from_yaml` and :func:`dcargs.conf.to_yaml` attempt + to strike a balance between flexibility and robustness — in contrast to naively + dumping or loading dataclass instances (via pickle, PyYAML, etc), explicit type + references enable custom tags that are robust against code reorganization and + refactor, while a PyYAML backend enables serialization of arbitrary Python objects. + + .. warning:: + Serialization functionality is deprecated. It may be removed in a future version + of :code:`dcargs`. + Args: instance: Dataclass instance to serialize. diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst deleted file mode 100644 index 6c47af1e..00000000 --- a/docs/source/serialization.rst +++ /dev/null @@ -1,22 +0,0 @@ -Serialization -==================================== - -As a secondary feature aimed at enabling the use of :func:`dcargs.cli` for general -configuration use cases, we also introduce functions for human-readable -dataclass serialization: - -- :func:`dcargs.extras.to_yaml` and :func:`dcargs.extras.from_yaml` convert - between YAML-style strings and dataclass instances. - -The functions attempt to strike a balance between flexibility and robustness — -in contrast to naively dumping or loading dataclass instances (via pickle, -PyYAML, etc), explicit type references enable custom tags that are robust -against code reorganization and refactor, while a PyYAML backend enables -serialization of arbitrary Python objects. - -Note that we generally prefer to use YAML purely for serialization, as opposed -to a configuration interface that humans are expected to manually write or -modify. Specifying things like loadable base configurations can be done directly -in Python, which enables all of the usual autocompletion and type checking -features. - diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index 43a4d895..4fe59935 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -5,6 +5,7 @@ from typing import Generic, List, Tuple, Type, TypeVar, Union import pytest +import yaml from typing_extensions import Annotated import dcargs @@ -217,7 +218,10 @@ class DataclassGeneric(Generic[T]): DataclassGeneric[Child], args=["--child.a", "5", "--child.b", "7"] ) assert parsed_instance == DataclassGeneric(Child(5, 7)) - _check_serialization_identity(DataclassGeneric[Child], parsed_instance) + + # Local generics will break. + with pytest.raises(yaml.constructor.ConstructorError): + _check_serialization_identity(DataclassGeneric[Child], parsed_instance) def test_generic_nested_dataclass_helptext(): @@ -263,14 +267,22 @@ class Subparser(Generic[T1, T2]): args="command:command-one --command.a 5".split(" "), ) assert parsed_instance == Subparser(CommandOne(5)) - _check_serialization_identity(Subparser[CommandOne, CommandTwo], parsed_instance) + # Local generics will break. + with pytest.raises(yaml.constructor.ConstructorError): + _check_serialization_identity( + Subparser[CommandOne, CommandTwo], parsed_instance + ) parsed_instance = dcargs.cli( Subparser[CommandOne, CommandTwo], args="command:command-two --command.b 7".split(" "), ) assert parsed_instance == Subparser(CommandTwo(7)) - _check_serialization_identity(Subparser[CommandOne, CommandTwo], parsed_instance) + # Local generics will break. + with pytest.raises(yaml.constructor.ConstructorError): + _check_serialization_identity( + Subparser[CommandOne, CommandTwo], parsed_instance + ) def test_generic_subparsers_in_container(): @@ -294,9 +306,11 @@ class Subparser(Generic[T1, T2]): assert parsed_instance == Subparser(Command([5, 3])) and isinstance( parsed_instance.command.a[0], int ) - _check_serialization_identity( - Subparser[Command[int], Command[float]], parsed_instance - ) + # Local generics will break. + with pytest.raises(yaml.constructor.ConstructorError): + _check_serialization_identity( + Subparser[Command[int], Command[float]], parsed_instance + ) parsed_instance = dcargs.cli( Subparser[Command[int], Command[float]], @@ -305,9 +319,11 @@ class Subparser(Generic[T1, T2]): assert parsed_instance == Subparser(Command([7.0, 2.0])) and isinstance( parsed_instance.command.a[0], float ) - _check_serialization_identity( - Subparser[Command[int], Command[float]], parsed_instance - ) + # Local generics will break. + with pytest.raises(yaml.constructor.ConstructorError): + _check_serialization_identity( + Subparser[Command[int], Command[float]], parsed_instance + ) def test_serialize_missing(): @@ -366,9 +382,7 @@ class Wrapper: subclass: Union[TypeA, TypeB] = TypeA(1) wrapper1 = Wrapper() # Create Wrapper object. - dcargs.extras.from_yaml( - Wrapper, dcargs.extras.to_yaml(wrapper1) - ) # Errors, no constructor for TypeA + assert wrapper1 == dcargs.extras.from_yaml(Wrapper, dcargs.extras.to_yaml(wrapper1)) def test_annotated(): @@ -383,6 +397,23 @@ class Wrapper: subclass: Annotated[TypeA, int] = TypeA(1) wrapper1 = Wrapper() # Create Wrapper object. - dcargs.extras.from_yaml( - Wrapper, dcargs.extras.to_yaml(wrapper1) - ) # Errors, no constructor for TypeA + assert wrapper1 == dcargs.extras.from_yaml(Wrapper, dcargs.extras.to_yaml(wrapper1)) + + +def test_superclass(): + # https://github.com/brentyi/dcargs/issues/7 + + @dataclasses.dataclass + class TypeA: + data: int + + @dataclasses.dataclass + class TypeASubclass(TypeA): + pass + + @dataclasses.dataclass + class Wrapper: + subclass: TypeA + + wrapper1 = Wrapper(TypeASubclass(3)) # Create Wrapper object. + assert wrapper1 == dcargs.extras.from_yaml(Wrapper, dcargs.extras.to_yaml(wrapper1)) From 11bff7cec27e4a76df601bb82598ed9149abe947 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 6 Sep 2022 22:50:00 -0700 Subject: [PATCH 16/19] Helptext improvements, tests, documentation --- dcargs/_docstrings.py | 29 +++++++----- docs/source/helptext_generation.md | 76 ++++++++++++++++++++++++++++++ docs/source/index.rst | 2 +- examples/02_dataclasses.py | 7 ++- tests/test_helptext.py | 52 ++++++++++++++++++++ 5 files changed, 152 insertions(+), 14 deletions(-) create mode 100644 docs/source/helptext_generation.md diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index 492620a6..b236b583 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -150,6 +150,12 @@ def get_class_tokenization_with_field( def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: """Get docstring for a field in a class.""" + docstring = inspect.getdoc(cls) + if docstring is not None: + for param_doc in docstring_parser.parse(docstring).params: + if param_doc.arg_name == field_name: + return param_doc.description + tokenization = get_class_tokenization_with_field(cls, field_name) if tokenization is None: # Currently only happens for dynamic dataclasses. return None @@ -277,17 +283,16 @@ def get_callable_description(f: Callable) -> str: default_doc = f.__name__ + str(inspect.signature(f)).replace(" -> None", "") if docstring == default_doc: return "" - return docstring - else: - parsed_docstring = docstring_parser.parse(docstring) - return "\n".join( - list( - filter( - lambda x: x is not None, # type: ignore - [ - parsed_docstring.short_description, - parsed_docstring.long_description, - ], - ) + + parsed_docstring = docstring_parser.parse(docstring) + return "\n".join( + list( + filter( + lambda x: x is not None, # type: ignore + [ + parsed_docstring.short_description, + parsed_docstring.long_description, + ], ) ) + ) diff --git a/docs/source/helptext_generation.md b/docs/source/helptext_generation.md new file mode 100644 index 00000000..f341e59d --- /dev/null +++ b/docs/source/helptext_generation.md @@ -0,0 +1,76 @@ +# Helptext generation + +In addition to type annotations, :func:`dcargs.cli()` will also parse docstrings +and comments. These are used to automatically generate helptext; see examples +for how these end up being formatted. + +## General callables + +For general callables, field helptext is extracted from the corresponding field +docstring. Our examples use Google-style docstrings, but ReST, Numpydoc-style +and Epydoc docstrings are supported as well. Under the hood, all of these +options use [docstring_parser](https://github.com/rr-/docstring_parser). + +```python +def main( + field1: str, + field2: int = 3, +) -> None: + """Function, whose arguments will be populated from a CLI interface. + + Args: + field1: A string field. + field2: A numeric field, with a default value. + """ + print(field1, field2) +``` + +## Dataclasses, TypedDict, NamedTuple + +For types defined using class attributes, enumerating each argument list in the +class docstring can be cumbersome. + +If they are unavailable, :func:`dcargs.cli` will generate helptext from +docstrings and comments on attributes. These are parsed via source code +inspection. + +**(1) Attribute docstrings** + +As per [PEP 257](https://peps.python.org/pep-0257/#what-is-a-docstring). + +```python +@dataclasses.dataclass +class Args: + field1: str + """A string field.""" + field2: int = 3 + """A numeric field, with a default value.""" +``` + +**(2) Inline comments** + +Inline comments can be more succinct than true attribute docstrings. + +```python +@dataclasses.dataclass +class Args: + field1: str # A string field. + field2: int = 3 # A numeric field, with a default value. +``` + +**(3) Preceding comments** + +These can also be handy for semantically grouping fields, such as the two string +fields below. + +```python +@dataclasses.dataclass +class Args: + # String fields. + field1: str + field2: str + + # An integer field. + # Multi-line comments are supported. + field3: int +``` diff --git a/docs/source/index.rst b/docs/source/index.rst index afa9ad10..901187d2 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -106,7 +106,7 @@ The broader goal is also a replacement for tools like :code:`hydra`, :glob: goals_and_alternatives - serialization + helptext_generation .. toctree:: diff --git a/examples/02_dataclasses.py b/examples/02_dataclasses.py index 4993138b..6acf0612 100644 --- a/examples/02_dataclasses.py +++ b/examples/02_dataclasses.py @@ -15,7 +15,12 @@ @dataclasses.dataclass class Args: """Description. - This should show up in the helptext!""" + This should show up in the helptext! + + Args: + field1: A string field!!! + field2: An int field!!! + """ field1: str # A string field. field2: int = 3 # A numeric field, with a default value. diff --git a/tests/test_helptext.py b/tests/test_helptext.py index bbfbde39..dd008965 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -54,6 +54,58 @@ class Helptext: assert "Documentation 3 (default: 3)\n" in helptext +def test_helptext_from_class_docstring(): + @dataclasses.dataclass + class Helptext2: + """This docstring should be printed as a description. + + Attributes: + x: Documentation 1 + y: Documentation 2 + z: Documentation 3 + """ + + x: int + y: Annotated[int, "ignored"] + z: int = 3 + + helptext = _get_helptext(Helptext2) + assert "This docstring should be printed as a description" in helptext + assert "Attributes" not in helptext + assert "x INT" in helptext + assert "y INT" in helptext + assert "z INT" in helptext + assert "Documentation 1 (required)\n" in helptext + assert "Documentation 2 (required)\n" in helptext + assert "Documentation 3 (default: 3)\n" in helptext + + +def test_helptext_from_class_docstring_args(): + @dataclasses.dataclass + class Helptext3: + """This docstring should be printed as a description. + + Args: + x: Documentation 1 + y: Documentation 2 + z: Documentation 3 + """ + + x: int + y: Annotated[int, "ignored"] + z: int = 3 + + helptext = _get_helptext(Helptext3) + assert "This docstring should be printed as a description" in helptext + assert "Args" not in helptext + assert "x INT" in helptext + assert "y INT" in helptext + assert "z INT" in helptext + assert "Documentation 1 (required)\n" in helptext + assert "Documentation 2 (required)\n" in helptext + assert "Documentation 3 (default: 3)\n" in helptext + + def test_helptext_inherited(): class UnrelatedParentClass: pass From b7a3f092b5c68e557e3fee2e61f5912bc47b766f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 7 Sep 2022 01:57:20 -0700 Subject: [PATCH 17/19] Tab completion documentation --- dcargs/_cli.py | 10 ++++- dcargs/extras/_serialization.py | 8 ++-- docs/source/index.rst | 3 +- docs/source/tab_completion.md | 79 +++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 7 deletions(-) create mode 100644 docs/source/tab_completion.md diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 670c4bd9..84a1caee 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -60,7 +60,7 @@ def cli( `f` is a class, `dcargs.cli()` returns an instance. The parser is generated by populating helptext from docstrings and types from - annotations; a broad range of core type annotations are supported... + annotations; a broad range of core type annotations are supported. - Types natively accepted by `argparse`: str, int, float, pathlib.Path, etc. - Default values for optional parameters. - Booleans, which are automatically converted to flags when provided a default @@ -87,6 +87,9 @@ def cli( - Optional unions over nested structures (optional subparsers). - Generics (including nested generics). + Completion scripts for interactive shells is also provided. To print a script that + can be used for tab completion, pass in `--dcargs-print-completion {bash/zsh/tcsh}`. + Args: f: Callable. prog: The name of the program printed in helptext. Mirrors argument from @@ -155,8 +158,11 @@ def cli( prefix="", # Used for recursive calls. ) - # If we pass in the --dcargs-print-completion flag: turn termcolor off, and ge the + # If we pass in the --dcargs-print-completion flag: turn termcolor off, and get the # shell we want to generate a completion script for (bash/zsh/tcsh). + # + # Note that shtab also offers an add_argument_to() functions that fulfills a similar + # goal, but manual parsing of argv is convenient for turning off colors. args = sys.argv[1:] if args is None else args print_completion = len(args) >= 2 and args[0] == "--dcargs-print-completion" diff --git a/dcargs/extras/_serialization.py b/dcargs/extras/_serialization.py index 333f2b32..9d23f31c 100644 --- a/dcargs/extras/_serialization.py +++ b/dcargs/extras/_serialization.py @@ -184,8 +184,8 @@ def from_yaml( refactor, while a PyYAML backend enables serialization of arbitrary Python objects. .. warning:: - Serialization functionality is deprecated. It may be removed in a future version - of :code:`dcargs`. + Serialization functionality is stable but deprecated. It may be removed in a + future version of :code:`dcargs`. Args: cls: Type to reconstruct. @@ -213,8 +213,8 @@ def to_yaml(instance: Any) -> str: refactor, while a PyYAML backend enables serialization of arbitrary Python objects. .. warning:: - Serialization functionality is deprecated. It may be removed in a future version - of :code:`dcargs`. + Serialization functionality is stable but deprecated. It may be removed in a + future version of :code:`dcargs`. Args: instance: Dataclass instance to serialize. diff --git a/docs/source/index.rst b/docs/source/index.rst index 901187d2..0ab9f442 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -105,8 +105,9 @@ The broader goal is also a replacement for tools like :code:`hydra`, :hidden: :glob: - goals_and_alternatives helptext_generation + tab_completion + goals_and_alternatives .. toctree:: diff --git a/docs/source/tab_completion.md b/docs/source/tab_completion.md new file mode 100644 index 00000000..c9fad0a1 --- /dev/null +++ b/docs/source/tab_completion.md @@ -0,0 +1,79 @@ +# Tab completion + +Interfaces built with :func:`dcargs.cli()` can be tab completed in interactive +shells without any source code modification. + +Completion scripts can be generated by passing the +`--dcargs-print-completion {bash/zsh/tcsh}` flag to a dcargs CLI. This generates +a completion script via [shtab](https://docs.iterative.ai/shtab/) and prints it +to stdout. To set up tab completion, the printed script simply needs to be +written somewhere where your shell will find it. + +## Zsh + +For zsh, one option is to emulate the pattern used for completions in +[poetry](https://python-poetry.org/docs): + +``` +# Set up zsh autocompletion for 01_functions.py, which is located in +# dcargs/examples. + +# (1) Make directory for local completions. +mkdir -p ~/.zfunc + +# (2) Write completion script. The name here (_01_functions_py) doesn't matter, +# as long as it's prefixed with an underscore. +python 01_functions.py > ~/.zfunc/_01_functions_py +``` + +And if it's not in your `.zshrc` already: + +``` +# (3) Add .zfunc to our function search path, then initialize completions. +# Ideally this should go in your .zshrc! +fpath+=~/.zfunc +autoload -Uz compinit && compinit +``` + +## Bash + +Local completion scripts for bash can be written as described in the +[bash-complete documentation](https://github.com/scop/bash-completion/blob/master/README.md#FAQ). + +> **Q.** Where should I install my own local completions? +> +> **A.** Put them in the completions subdir of `$BASH_COMPLETION_USER_DIR` +> (defaults to `$XDG_DATA_HOME/bash-completion` or +> `~/.local/share/bash-completion` if `$XDG_DATA_HOME` is not set) to have them +> loaded automatically on demand when the respective command is being completed +> [...] + +Borrowing from the `bash-completion` source[^bash_completion], we can run: + + + +[^bash_completion]: `completion_dir` query is borrowed from [here](https://github.com/scop/bash-completion/blob/966a4e043822613040546e8c003509798c5fae1a/bash_completion#L2440). + + + +```bash +# Set up bash autocompletion for 01_functions.py, which is located in +# dcargs/examples. + +# (1) Find and make completion directory. +completion_dir=${BASH_COMPLETION_USER_DIR:-${XDG_DATA_HOME:-$HOME/.local/share}/bash-completion}/completions/ +mkdir -p $completion_dir + +# (2) Write completion scripts. Note that the name of the completion script must +# match the name of the file. +python 01_functions.py --dcargs-print-completion bash > ${completion_dir}/01_functions.py +``` + +In contrast to zsh, tab completion in bash requires that scripts are either set +up as an entry point or run as :code:`./01_functions.py `, as opposed to +with the `python` command, :code:`python 01_functions.py `. + +Making a script directly executable typically requires: + +1. A permissions update: `chmod +x ./01_functions.py`. +2. A shebang as the first line of your script: `#!/usr/bin/env python` From 016b87bf4b5780f35691563f50bc14962d68139f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 7 Sep 2022 02:36:46 -0700 Subject: [PATCH 18/19] `dcargs.conf` documentation, example --- dcargs/_parsers.py | 4 + dcargs/_strings.py | 12 ++- dcargs/conf/_markers.py | 5 +- dcargs/conf/_subcommands.py | 4 +- docs/source/examples/04_flags.rst | 2 + docs/source/examples/07_positional_args.rst | 2 + docs/source/examples/08_subcommands.rst | 3 + .../examples/16_advanced_configuration.rst | 76 +++++++++++++++++++ examples/02_dataclasses.py | 7 +- examples/04_flags.py | 2 + examples/07_positional_args.py | 2 + examples/08_subcommands.py | 3 + examples/16_advanced_configuration.py | 61 +++++++++++++++ 13 files changed, 170 insertions(+), 13 deletions(-) create mode 100644 docs/source/examples/16_advanced_configuration.rst create mode 100644 examples/16_advanced_configuration.py diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index d29f03d8..31e3ca0e 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -336,7 +336,11 @@ def from_field( if default_name not in parser_from_name: # If we can't find the subparser by name, search by type. This is needed # when the user renames their subcommands. (eg via dcargs.subcommand) + # + # TODO: this will display some weird behaviors if multiple subcommands + # have the same type. default_name = None + for name, parser in parser_from_name.items(): if type(field.default) is _resolver.unwrap_origin_strip_extras( parser.f diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 15e5bfcd..13e58fac 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -62,13 +62,19 @@ def _subparser_name_from_type(cls: Type) -> Tuple[str, bool]: ) # Subparser name from `dcargs.metadata.subcommand()`. + found_name = None + prefix_name = True if len(found_subcommand_configs) > 0: - return found_subcommand_configs[0].name, found_subcommand_configs[0].prefix_name + found_name = found_subcommand_configs[0].name + prefix_name = found_subcommand_configs[0].prefix_name + + if found_name is not None: + return found_name, prefix_name # Subparser name from class name. if len(type_from_typevar) == 0: assert hasattr(cls, "__name__") - return hyphen_separated_from_camel_case(cls.__name__), True # type: ignore + return hyphen_separated_from_camel_case(cls.__name__), prefix_name # type: ignore return ( "-".join( @@ -77,7 +83,7 @@ def _subparser_name_from_type(cls: Type) -> Tuple[str, bool]: [cls] + list(type_from_typevar.values()), ) ), - True, + prefix_name, ) diff --git a/dcargs/conf/_markers.py b/dcargs/conf/_markers.py index 3e610065..6aa83f9c 100644 --- a/dcargs/conf/_markers.py +++ b/dcargs/conf/_markers.py @@ -30,8 +30,9 @@ def __repr__(self): FIXED = _make_marker("Fixed") Fixed = Annotated[T, FIXED] -"""A type `T` can be annotated as `Fixed[T]` to prevent `dcargs.cli` from parsing it. A -default value should be set instead.""" +"""A type `T` can be annotated as `Fixed[T]` to prevent `dcargs.cli` from parsing it; a +default value should be set instead. Note that fields with defaults that can't be parsed +will also be marked as fixed automatically.""" FLAG_CONVERSION_OFF = _make_marker("FlagConversionOff") FlagConversionOff = Annotated[T, FLAG_CONVERSION_OFF] diff --git a/dcargs/conf/_subcommands.py b/dcargs/conf/_subcommands.py index 03d99661..92f84b03 100644 --- a/dcargs/conf/_subcommands.py +++ b/dcargs/conf/_subcommands.py @@ -6,7 +6,7 @@ @dataclasses.dataclass(frozen=True) class _SubcommandConfiguration: - name: str + name: Optional[str] default: Any description: Optional[str] prefix_name: bool @@ -16,7 +16,7 @@ def __hash__(self) -> int: def subcommand( - name: str, + name: Optional[str] = None, *, default: Any = MISSING_NONPROP, description: Optional[str] = None, diff --git a/docs/source/examples/04_flags.rst b/docs/source/examples/04_flags.rst index ceece83d..52de2c13 100644 --- a/docs/source/examples/04_flags.rst +++ b/docs/source/examples/04_flags.rst @@ -8,6 +8,8 @@ Booleans can either be expected to be explicitly passed in, or, if given a default value, automatically converted to flags. +To turn off conversion, see :func:`dcargs.conf.FlagConversionOff`. + .. code-block:: python diff --git a/docs/source/examples/07_positional_args.rst b/docs/source/examples/07_positional_args.rst index 270c9b9d..0e508d82 100644 --- a/docs/source/examples/07_positional_args.rst +++ b/docs/source/examples/07_positional_args.rst @@ -7,6 +7,8 @@ Positional-only arguments in functions are converted to positional CLI arguments. +For more general positional arguments, see :func:`dcargs.conf.Positional`. + .. code-block:: python diff --git a/docs/source/examples/08_subcommands.rst b/docs/source/examples/08_subcommands.rst index 841b408f..0ce749ef 100644 --- a/docs/source/examples/08_subcommands.rst +++ b/docs/source/examples/08_subcommands.rst @@ -7,6 +7,9 @@ Unions over nested types (classes or dataclasses) are populated using subcommands. +For configuring subcommands beyond what can be expressed with type annotations, see +:func:`dcargs.conf.subcommand()`. + .. code-block:: python diff --git a/docs/source/examples/16_advanced_configuration.rst b/docs/source/examples/16_advanced_configuration.rst new file mode 100644 index 00000000..fdef0bc8 --- /dev/null +++ b/docs/source/examples/16_advanced_configuration.rst @@ -0,0 +1,76 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +16. Advanced Configuration +========================================== + + +The :mod:`dcargs.conf` module contains utilities that can be used to configure +command-line interfaces beyond what is expressible via static type annotations. + +Features here are supported, but generally unnecessary and should be used sparingly. + + + +.. code-block:: python + :linenos: + + import dataclasses + from typing import Union + + from typing_extensions import Annotated + + import dcargs + + + @dataclasses.dataclass(frozen=True) + class CheckoutArgs: + """Checkout a branch.""" + + branch: str + + + @dataclasses.dataclass(frozen=True) + class CommitArgs: + """Commit changes.""" + + message: str + all: bool = False + + + @dataclasses.dataclass + class Args: + # A boolean field with flag conversion turned off. + boolean: dcargs.conf.FlagConversionOff[bool] = False + + # A numeric field parsed as a positional argument. + positional: dcargs.conf.Positional[int] = 3 + + # A numeric field that can't be changed via the CLI. + fixed: dcargs.conf.Fixed[int] = 5 + + # A union over nested structures, but without subcommand generation. When a default + # is provided, the type is simply fixed to that default. + union_without_subcommand: dcargs.conf.AvoidSubcommands[ + Union[CheckoutArgs, CommitArgs] + ] = CheckoutArgs("main") + + # `dcargs.conf.subcommand()` can be used to configure subcommands in a Union. + renamed_subcommand: Union[ + Annotated[ + CheckoutArgs, dcargs.conf.subcommand(name="checkout", prefix_name=False) + ], + Annotated[CommitArgs, dcargs.conf.subcommand(name="commit", prefix_name=False)], + ] = CheckoutArgs("main") + + + if __name__ == "__main__": + print(dcargs.cli(Args)) + +------------ + +.. raw:: html + + python 16_advanced_configuration.py --help + +.. program-output:: python ../../examples/16_advanced_configuration.py --help diff --git a/examples/02_dataclasses.py b/examples/02_dataclasses.py index 6acf0612..4993138b 100644 --- a/examples/02_dataclasses.py +++ b/examples/02_dataclasses.py @@ -15,12 +15,7 @@ @dataclasses.dataclass class Args: """Description. - This should show up in the helptext! - - Args: - field1: A string field!!! - field2: An int field!!! - """ + This should show up in the helptext!""" field1: str # A string field. field2: int = 3 # A numeric field, with a default value. diff --git a/examples/04_flags.py b/examples/04_flags.py index 233e5fb4..635acea2 100644 --- a/examples/04_flags.py +++ b/examples/04_flags.py @@ -1,6 +1,8 @@ """Booleans can either be expected to be explicitly passed in, or, if given a default value, automatically converted to flags. +To turn off conversion, see :func:`dcargs.conf.FlagConversionOff`. + Usage: `python ./04_flags.py --help` `python ./04_flags.py --boolean True` diff --git a/examples/07_positional_args.py b/examples/07_positional_args.py index 317eccaa..fc81f712 100644 --- a/examples/07_positional_args.py +++ b/examples/07_positional_args.py @@ -1,5 +1,7 @@ """Positional-only arguments in functions are converted to positional CLI arguments. +For more general positional arguments, see :func:`dcargs.conf.Positional`. + Usage: `python ./07_positional_args.py --help` `python ./07_positional_args.py ./a ./b --optimizer.learning-rate 1e-5` diff --git a/examples/08_subcommands.py b/examples/08_subcommands.py index dba66b89..7ae6b5db 100644 --- a/examples/08_subcommands.py +++ b/examples/08_subcommands.py @@ -1,5 +1,8 @@ """Unions over nested types (classes or dataclasses) are populated using subcommands. +For configuring subcommands beyond what can be expressed with type annotations, see +:func:`dcargs.conf.subcommand()`. + Usage: `python ./08_subcommands.py --help` `python ./08_subcommands.py cmd:commit --help` diff --git a/examples/16_advanced_configuration.py b/examples/16_advanced_configuration.py new file mode 100644 index 00000000..8f0c5815 --- /dev/null +++ b/examples/16_advanced_configuration.py @@ -0,0 +1,61 @@ +"""The :mod:`dcargs.conf` module contains utilities that can be used to configure +command-line interfaces beyond what is expressible via static type annotations. + +Features here are supported, but generally unnecessary and should be used sparingly. + +Usage: +`python ./16_advanced_configuration.py --help` +""" + +import dataclasses +from typing import Union + +from typing_extensions import Annotated + +import dcargs + + +@dataclasses.dataclass(frozen=True) +class CheckoutArgs: + """Checkout a branch.""" + + branch: str + + +@dataclasses.dataclass(frozen=True) +class CommitArgs: + """Commit changes.""" + + message: str + all: bool = False + + +@dataclasses.dataclass +class Args: + # A boolean field with flag conversion turned off. + boolean: dcargs.conf.FlagConversionOff[bool] = False + + # A numeric field parsed as a positional argument. + positional: dcargs.conf.Positional[int] = 3 + + # A numeric field that can't be changed via the CLI. + fixed: dcargs.conf.Fixed[int] = 5 + + # A union over nested structures, but without subcommand generation. When a default + # is provided, the type is simply fixed to that default. + union_without_subcommand: dcargs.conf.AvoidSubcommands[ + Union[CheckoutArgs, CommitArgs] + ] = CheckoutArgs("main") + + # `dcargs.conf.subcommand()` can be used to configure subcommands in a Union. Here, + # we make the subcommand names more succinct. + renamed_subcommand: Union[ + Annotated[ + CheckoutArgs, dcargs.conf.subcommand(name="checkout", prefix_name=False) + ], + Annotated[CommitArgs, dcargs.conf.subcommand(name="commit", prefix_name=False)], + ] = CheckoutArgs("main") + + +if __name__ == "__main__": + print(dcargs.cli(Args)) From 5833870895f6ffacd305912facc38e54b4192518 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 7 Sep 2022 13:35:36 -0700 Subject: [PATCH 19/19] Docs polish --- docs/source/examples/16_advanced_configuration.rst | 3 ++- docs/source/tab_completion.md | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/source/examples/16_advanced_configuration.rst b/docs/source/examples/16_advanced_configuration.rst index fdef0bc8..12c27e8c 100644 --- a/docs/source/examples/16_advanced_configuration.rst +++ b/docs/source/examples/16_advanced_configuration.rst @@ -55,7 +55,8 @@ Features here are supported, but generally unnecessary and should be used sparin Union[CheckoutArgs, CommitArgs] ] = CheckoutArgs("main") - # `dcargs.conf.subcommand()` can be used to configure subcommands in a Union. + # `dcargs.conf.subcommand()` can be used to configure subcommands in a Union. Here, + # we make the subcommand names more succinct. renamed_subcommand: Union[ Annotated[ CheckoutArgs, dcargs.conf.subcommand(name="checkout", prefix_name=False) diff --git a/docs/source/tab_completion.md b/docs/source/tab_completion.md index c9fad0a1..559e693d 100644 --- a/docs/source/tab_completion.md +++ b/docs/source/tab_completion.md @@ -14,7 +14,7 @@ written somewhere where your shell will find it. For zsh, one option is to emulate the pattern used for completions in [poetry](https://python-poetry.org/docs): -``` +```bash # Set up zsh autocompletion for 01_functions.py, which is located in # dcargs/examples. @@ -23,12 +23,12 @@ mkdir -p ~/.zfunc # (2) Write completion script. The name here (_01_functions_py) doesn't matter, # as long as it's prefixed with an underscore. -python 01_functions.py > ~/.zfunc/_01_functions_py +python 01_functions.py --dcargs-print-completion zsh > ~/.zfunc/_01_functions_py ``` And if it's not in your `.zshrc` already: -``` +```bash # (3) Add .zfunc to our function search path, then initialize completions. # Ideally this should go in your .zshrc! fpath+=~/.zfunc