From 67f98de4e3f85283917fcbaebfdb71ea2c78ce4e Mon Sep 17 00:00:00 2001 From: yukinarit Date: Sun, 20 Oct 2024 15:12:09 +0900 Subject: [PATCH] Add skip_none parameter The `skip_none` attribute is an optional feature originally implemented to prevent null values from appearing in TOML serialized outputs. When set to True, any field in the class with a None value is excluded from the serialized output, ensuring that TOML files (or other formats) remain clean and free from null entries. Closes #421 --- examples/runner.py | 2 ++ serde/json.py | 4 ++++ serde/msgpack.py | 10 +++++--- serde/pickle.py | 1 + serde/se.py | 43 +++++++++++++++++++++++++++------- serde/toml.py | 13 +++++++++- serde/yaml.py | 4 ++++ tests/test_basics.py | 22 ----------------- tests/test_toml.py | 56 ++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 121 insertions(+), 34 deletions(-) create mode 100644 tests/test_toml.py diff --git a/examples/runner.py b/examples/runner.py index 26d01680..a8398078 100644 --- a/examples/runner.py +++ b/examples/runner.py @@ -132,5 +132,7 @@ def run(module: typing.Any) -> None: if __name__ == "__main__": try: run_all() + print("-----------------") + print("all tests passed successfully!") except Exception: sys.exit(1) diff --git a/serde/json.py b/serde/json.py index 47ec1d16..3d73be6b 100644 --- a/serde/json.py +++ b/serde/json.py @@ -56,6 +56,7 @@ def to_json( se: type[Serializer[str]] = JsonSerializer, reuse_instances: bool = False, convert_sets: bool = True, + skip_none: bool = False, **opts: Any, ) -> str: """ @@ -68,6 +69,9 @@ def to_json( argument with [orjson](https://github.com/ijl/orjson#numpy), or the `default` argument with Python standard json library. + * `skip_none`: When set to True, any field in the class with a None value is excluded from the + serialized output. Defaults to False. + If you want to use another json package, you can subclass `JsonSerializer` and implement your own logic. """ diff --git a/serde/msgpack.py b/serde/msgpack.py index 8e02d844..6d8bcb52 100644 --- a/serde/msgpack.py +++ b/serde/msgpack.py @@ -56,9 +56,12 @@ def to_msgpack( as a `msgpack.ExtType` If you supply other keyword arguments, they will be passed in `msgpack.packb` function. - If `named` is True, field names are preserved, namely the object is encoded as `dict` then - serialized into MsgPack. If `named` is False, the object is encoded as `tuple` then serialized - into MsgPack. `named=False` will produces compact binary. + * `named`: If `named` is True, field names are preserved, namely the object is encoded as `dict` + then serialized into MsgPack. If `named` is False, the object is encoded as `tuple` then + serialized into MsgPack. `named=False` will produces compact binary. + + * `skip_none`: When set to True, any field in the class with a None value is excluded from the + serialized output. Defaults to False. If you want to use the other msgpack package, you can subclass `MsgPackSerializer` and implement your own logic. @@ -107,6 +110,7 @@ def from_msgpack( de: type[Deserializer[bytes]] = MsgPackDeserializer, named: bool = True, ext_dict: Optional[dict[int, type[Any]]] = None, + skip_none: bool = False, **opts: Any, ) -> Any: """ diff --git a/serde/pickle.py b/serde/pickle.py index dda5445f..c20443a3 100644 --- a/serde/pickle.py +++ b/serde/pickle.py @@ -30,6 +30,7 @@ def to_pickle( se: type[Serializer[bytes]] = PickleSerializer, reuse_instances: bool = False, convert_sets: bool = True, + skip_none: bool = False, **opts: Any, ) -> bytes: return se.serialize( diff --git a/serde/se.py b/serde/se.py index d4abb375..8fba9c0b 100644 --- a/serde/se.py +++ b/serde/se.py @@ -334,13 +334,17 @@ def to_obj( named: bool, reuse_instances: Optional[bool] = None, convert_sets: Optional[bool] = None, + skip_none: bool = False, c: Optional[Any] = None, ) -> Any: def serializable_to_obj(object: Any) -> Any: serde_scope: Scope = getattr(object, SERDE_SCOPE) func_name = TO_DICT if named else TO_ITER return serde_scope.funcs[func_name]( - object, reuse_instances=reuse_instances, convert_sets=convert_sets + object, + reuse_instances=reuse_instances, + convert_sets=convert_sets, + skip_none=skip_none, ) try: @@ -349,6 +353,7 @@ def serializable_to_obj(object: Any) -> Any: named=named, reuse_instances=reuse_instances, convert_sets=convert_sets, + skip_none=skip_none, ) # If a class in the argument is a non-dataclass class e.g. Union[Foo, Bar], @@ -377,7 +382,11 @@ def serializable_to_obj(object: Any) -> Any: return {k: thisfunc(v) for k, v in o.items()} elif is_str_serializable_instance(o) or is_datetime_instance(o): return CACHE.serialize( - c or o.__class__, o, reuse_instances=reuse_instances, convert_sets=convert_sets + c or o.__class__, + o, + reuse_instances=reuse_instances, + convert_sets=convert_sets, + skip_none=skip_none, ) return o @@ -398,6 +407,7 @@ def to_tuple( c: Optional[type[Any]] = None, reuse_instances: Optional[bool] = None, convert_sets: Optional[bool] = None, + skip_none: bool = False, ) -> tuple[Any, ...]: """ Serialize object into tuple. @@ -419,7 +429,12 @@ def to_tuple( [(10, 'foo', 100.0, True), (20, 'foo', 100.0, True)] """ return to_obj( # type: ignore - o, named=False, c=c, reuse_instances=reuse_instances, convert_sets=convert_sets + o, + named=False, + c=c, + reuse_instances=reuse_instances, + convert_sets=convert_sets, + skip_none=skip_none, ) @@ -435,6 +450,7 @@ def to_dict( c: Optional[type[Any]] = None, reuse_instances: Optional[bool] = None, convert_sets: Optional[bool] = None, + skip_none: bool = False, ) -> dict[Any, Any]: """ Serialize object into python dictionary. This function ensures that the dataclass's fields are @@ -450,6 +466,8 @@ def to_dict( deserialization. When `convert_sets` is set to True, pyserde will convert sets to lists during serialization and back to sets during deserialization. This is useful for data formats that do not natively support sets. + * `skip_none`: When set to True, any field in the class with a None value is excluded from the + serialized output. Defaults to False. >>> from serde import serde >>> @serde @@ -470,7 +488,12 @@ def to_dict( [{'i': 10, 's': 'foo', 'f': 100.0, 'b': True}, {'i': 20, 's': 'foo', 'f': 100.0, 'b': True}] """ return to_obj( # type: ignore - o, named=True, c=c, reuse_instances=reuse_instances, convert_sets=convert_sets + o, + named=True, + c=c, + reuse_instances=reuse_instances, + convert_sets=convert_sets, + skip_none=skip_none, ) @@ -524,7 +547,7 @@ def sefields(cls: type[Any], serialize_class_var: bool = False) -> Iterator[SeFi loader=jinja2.DictLoader( { "dict": """ -def {{func}}(obj, reuse_instances = None, convert_sets = None): +def {{func}}(obj, reuse_instances = None, convert_sets = None, skip_none = False): if reuse_instances is None: reuse_instances = {{serde_scope.reuse_instances_default}} if convert_sets is None: @@ -534,13 +557,17 @@ def {{func}}(obj, reuse_instances = None, convert_sets = None): res = {} {% for f in fields -%} + subres = {{rvalue(f)}} {% if not f.skip -%} {% if f.skip_if -%} - subres = {{rvalue(f)}} if not {{f.skip_if.name}}(subres): {{lvalue(f)}} = subres {% else -%} - {{lvalue(f)}} = {{rvalue(f)}} + if skip_none: + if subres is not None: + {{lvalue(f)}} = subres + else: + {{lvalue(f)}} = subres {% endif -%} {% endif %} @@ -548,7 +575,7 @@ def {{func}}(obj, reuse_instances = None, convert_sets = None): return res """, "iter": """ -def {{func}}(obj, reuse_instances=None, convert_sets=None): +def {{func}}(obj, reuse_instances=None, convert_sets=None, skip_none=False): if reuse_instances is None: reuse_instances = {{serde_scope.reuse_instances_default}} if convert_sets is None: diff --git a/serde/toml.py b/serde/toml.py index 2bb85ab1..c46508f7 100644 --- a/serde/toml.py +++ b/serde/toml.py @@ -40,6 +40,7 @@ def to_toml( se: type[Serializer[str]] = TomlSerializer, reuse_instances: bool = False, convert_sets: bool = True, + skip_none: bool = True, **opts: Any, ) -> str: """ @@ -48,11 +49,21 @@ def to_toml( You can pass any serializable `obj`. If you supply keyword arguments other than `se`, they will be passed in `toml_w.dumps` function. + * `skip_none`: When set to True, any field in the class with a None value is excluded from the + serialized output. Defaults to True. + If you want to use the other toml package, you can subclass `TomlSerializer` and implement your own logic. """ return se.serialize( - to_dict(obj, c=cls, reuse_instances=reuse_instances, convert_sets=convert_sets), **opts + to_dict( + obj, + c=cls, + reuse_instances=reuse_instances, + convert_sets=convert_sets, + skip_none=skip_none, + ), + **opts, ) diff --git a/serde/yaml.py b/serde/yaml.py index 3e076a3f..8d15f00f 100644 --- a/serde/yaml.py +++ b/serde/yaml.py @@ -32,6 +32,7 @@ def to_yaml( se: type[Serializer[str]] = YamlSerializer, reuse_instances: bool = False, convert_sets: bool = True, + skip_none: bool = False, **opts: Any, ) -> str: """ @@ -40,6 +41,9 @@ def to_yaml( You can pass any serializable `obj`. If you supply keyword arguments other than `se`, they will be passed in `yaml.safe_dump` function. + * `skip_none`: When set to True, any field in the class with a None value is excluded from the + serialized output. Defaults to False. + If you want to use the other yaml package, you can subclass `YamlSerializer` and implement your own logic. """ diff --git a/tests/test_basics.py b/tests/test_basics.py index 9b062df5..ea7c3188 100644 --- a/tests/test_basics.py +++ b/tests/test_basics.py @@ -458,28 +458,6 @@ def test_msgpack_unnamed(): assert p == serde.msgpack.from_msgpack(data.Pri, d, named=False) -def test_toml(): - @serde.serde - class Foo: - v: Optional[int] - - f = Foo(10) - assert "v = 10\n" == serde.toml.to_toml(f) - assert f == serde.toml.from_toml(Foo, "v = 10\n") - - # TODO: Should raise SerdeError - with pytest.raises(TypeError): - f = Foo(None) - serde.toml.to_toml(f) - - @serde.serde - class Foo: - v: set[int] - - f = Foo({1, 2, 3}) - serde.toml.to_toml(f) - - @pytest.mark.parametrize("se,de", all_formats) def test_rename(se, de): @serde.serde diff --git a/tests/test_toml.py b/tests/test_toml.py new file mode 100644 index 00000000..414af187 --- /dev/null +++ b/tests/test_toml.py @@ -0,0 +1,56 @@ +from serde import serde +from serde.toml import to_toml, from_toml +from typing import Optional +import pytest + + +def toml_basics() -> None: + @serde + class Foo: + v: Optional[int] + + f = Foo(10) + assert "v = 10\n" == to_toml(f) + assert f == from_toml(Foo, "v = 10\n") + + @serde + class Bar: + v: set[int] + + b = Bar({1, 2, 3}) + to_toml(b) + + +def test_skip_none() -> None: + @serde + class Foo: + a: int + b: Optional[int] + + f = Foo(10, 100) + assert ( + to_toml(f) + == """\ +a = 10 +b = 100 +""" + ) + + f = Foo(10, None) + assert ( + to_toml(f) + == """\ +a = 10 +""" + ) + + +def test_skip_none_container_not_supported_yet() -> None: + @serde + class Foo: + a: int + b: list[Optional[int]] + + f = Foo(10, [100, None]) + with pytest.raises(TypeError): + to_toml(f)