From 43721b27eedc0ea9a2855c6d638ddf31551c7b58 Mon Sep 17 00:00:00 2001 From: yukinarit Date: Sun, 7 Jan 2024 14:25:29 +0900 Subject: [PATCH] Implement global (de)serializer You apply the custom (de)serialization for entire codebase by registering class (de)serializer by add_serializer and add_deserializer. Registered class (de)serializers are stacked in pyserde's global space and automatically used for all the pyserde classes. --- README.md | 3 +- docs/en/SUMMARY.md | 1 + docs/en/class-attributes.md | 26 ++++++- docs/en/extension.md | 89 ++++++++++++++++++++++ examples/custom_class_serializer.py | 6 +- examples/global_custom_class_serializer.py | 69 +++++++++++++++++ serde/__init__.py | 4 + serde/core.py | 19 +++++ serde/de.py | 43 ++++++++--- serde/se.py | 24 +++--- 10 files changed, 258 insertions(+), 26 deletions(-) create mode 100644 docs/en/extension.md create mode 100644 examples/global_custom_class_serializer.py diff --git a/README.md b/README.md index fe77fb31..08b8fd06 100644 --- a/README.md +++ b/README.md @@ -92,8 +92,9 @@ Foo(i=10, s='foo', f=100.0, b=True) - [Rename](https://github.com/yukinarit/pyserde/blob/main/docs/en/field-attributes.md#rename) - [Alias](https://github.com/yukinarit/pyserde/blob/main/docs/en/field-attributes.md#alias) - Skip (de)serialization ([skip](https://github.com/yukinarit/pyserde/blob/main/docs/en/field-attributes.md#skip), [skip_if](https://github.com/yukinarit/pyserde/blob/main/docs/en/field-attributes.md#skip_if), [skip_if_false](https://github.com/yukinarit/pyserde/blob/main/docs/en/field-attributes.md#skip_if_false), [skip_if_default](https://github.com/yukinarit/pyserde/blob/main/docs/en/field-attributes.md#skip_if_default)) -- [Custom class (de)serializer](https://github.com/yukinarit/pyserde/blob/main/docs/en/class-attributes.md#serializer--deserializer) - [Custom field (de)serializer](https://github.com/yukinarit/pyserde/blob/main/docs/en/field-attributes.md#serializerdeserializer) +- [Custom class (de)serializer](https://github.com/yukinarit/pyserde/blob/main/docs/en/class-attributes.md#class_serializer--class_deserializer) +- [Custom global (de)serializer](https://github.com/yukinarit/pyserde/blob/main/docs/en/extension.md#custom-global-deserializer) - [Flatten](https://github.com/yukinarit/pyserde/blob/main/docs/en/field-attributes.md#flatten) ## Contributors ✨ diff --git a/docs/en/SUMMARY.md b/docs/en/SUMMARY.md index b8c863e0..dd8f1329 100644 --- a/docs/en/SUMMARY.md +++ b/docs/en/SUMMARY.md @@ -9,4 +9,5 @@ - [Field Attributes](field-attributes.md) - [Union](union.md) - [Type Check](type-check.md) +- [Extension](extension.md) - [FAQ](faq.md) diff --git a/docs/en/class-attributes.md b/docs/en/class-attributes.md index 90db7d56..ea390381 100644 --- a/docs/en/class-attributes.md +++ b/docs/en/class-attributes.md @@ -75,12 +75,12 @@ New in v0.7.0. See [Union](union.md). If you want to use a custom (de)serializer at class level, you can pass your (de)serializer object in `class_serializer` and `class_deserializer` class attributes. Class custom (de)serializer depends on a python library [plum](https://github.com/beartype/plum) which allows multiple method overloading like C++. With plum, you can write robust custom (de)serializer in a quite neat way. ```python -class MySerializer(ClassSerializer): +class MySerializer: @dispatch def serialize(self, value: datetime) -> str: return value.strftime("%d/%m/%y") -class MyDeserializer(ClassDeserializer): +class MyDeserializer: @dispatch def deserialize(self, cls: Type[datetime], value: Any) -> datetime: return datetime.strptime(value, "%d/%m/%y") @@ -97,6 +97,28 @@ Also, * If both field and class serializer specified, field serializer is prioritized * If both legacy and new class serializer specified, new class serializer is prioritized +> 💡 Tip: If you implements multiple `serialize` methods, you will receive "Redefinition of unused `serialize`" warning from type checker. In such case, try using `plum.overload` and `plum.dispatch` to workaround it. See [plum's documentation](https://beartype.github.io/plum/integration.html) for more information. +> +> ```python +> from plum import dispatch, overload +> +> class Serializer: +> # use @overload +> @overload +> def serialize(self, value: int) -> Any: +> return str(value) +> +> # use @overload +> @overload +> def serialize(self, value: float) -> Any: +> return int(value) +> +> # Add method time and make sure to add @dispatch. Plum will do all the magic to erase warnings from type checker. +> @dispatch +> def serialize(self, value: Any) -> Any: +> ... +> ``` + See [examples/custom_class_serializer.py](https://github.com/yukinarit/pyserde/blob/main/examples/custom_class_serializer.py) for complete example. New in v0.13.0. diff --git a/docs/en/extension.md b/docs/en/extension.md new file mode 100644 index 00000000..13725804 --- /dev/null +++ b/docs/en/extension.md @@ -0,0 +1,89 @@ +# Extending pyserde + +pyserde offers three ways to extend pyserde to support non builtin types. + +## Custom field (de)serializer + +See [custom field serializer](./field-attributes.md#serializerdeserializer). + +> 💡 Tip: wrapping `serde.field` with your own field function makes +> +> ```python +> import serde +> +> def field(*args, **kwargs): +> serde.field(*args, **kwargs, serializer=str) +> +> @serde +> class Foo: +> a: int = field(default=0) # Configuring field serializer +> ``` + +## Custom class (de)serializer + +See [custom class serializer](./class-attributes.md#class_serializer--class_deserializer). + +## Custom global (de)serializer + +You apply the custom (de)serialization for entire codebase by registering class (de)serializer by `add_serializer` and `add_deserializer`. Registered class (de)serializers are stacked in pyserde's global space and automatically used for all the pyserde classes. + +e.g. Implementing custom (de)serialization for `datetime.timedelta` using [isodate](https://pypi.org/project/isodate/) package. + +Here is the code of registering class (de)serializer for `datetime.timedelta`. + +```python +from datetime import timedelta +from plum import dispatch +from typing import Type, Any +import isodate +import serde + +class Serializer: + @dispatch + def serialize(self, value: timedelta) -> Any: + return isodate.duration_isoformat(value) + +class Deserializer: + @dispatch + def deserialize(self, cls: Type[timedelta], value: Any) -> timedelta: + return isodate.parse_duration(value) + +def init() -> None: + serde.add_serializer(Serializer()) + serde.add_deserializer(Deserializer()) +``` + +Users of this package can reuse custom (de)serialization functionality for `datetime.timedelta` just by calling `serde_timedelta.init()`. + +```python +import serde_timedelta +from serde import serde +from serde.json import to_json, from_json +from datetime import timedelta + +serde_timedelta.init() + +@serde +class Foo: + a: timedelta + +f = Foo(timedelta(hours=10)) +json = to_json(f) +print(json) +print(from_json(Foo, json)) +``` +and you get `datetime.timedelta` to be serialized in ISO 8601 duration format! +```bash +{"a":"PT10H"} +Foo(a=datetime.timedelta(seconds=36000)) +``` + +> 💡 Tip: You can register as many class (de)serializer as you want. This means you can use as many pyserde extensions as you want. +> Registered (de)serializers are stacked in the memory. A (de)serializer can be overridden by another (de)serializer. +> +> e.g. If you register 3 custom serializers in this order, the first serializer will completely overridden by the 3rd one. 2nd one works because it is implemented for a different type. +> 1. Register Serializer for `int` +> 2. Register Serializer for `float` +> 3. Register Serializer for `int` + +New in v0.13.0. \ No newline at end of file diff --git a/examples/custom_class_serializer.py b/examples/custom_class_serializer.py index 3bdd1450..150e9238 100644 --- a/examples/custom_class_serializer.py +++ b/examples/custom_class_serializer.py @@ -3,21 +3,19 @@ from datetime import datetime from serde import ( serde, - ClassSerializer, - ClassDeserializer, field, ) from serde.json import from_json, to_json from typing import Type, Any, List -class MySerializer(ClassSerializer): +class MySerializer: @dispatch def serialize(self, value: datetime) -> str: return value.strftime("%d/%m/%y") -class MyDeserializer(ClassDeserializer): +class MyDeserializer: @dispatch def deserialize(self, cls: Type[datetime], value: Any) -> datetime: return datetime.strptime(value, "%d/%m/%y") diff --git a/examples/global_custom_class_serializer.py b/examples/global_custom_class_serializer.py new file mode 100644 index 00000000..f3027ab9 --- /dev/null +++ b/examples/global_custom_class_serializer.py @@ -0,0 +1,69 @@ +from plum import dispatch +from dataclasses import dataclass +from datetime import datetime +from serde import serde, add_serializer, add_deserializer +from serde.json import from_json, to_json +from typing import Type, Any + + +class MySerializer: + @dispatch + def serialize(self, value: datetime) -> str: + return value.strftime("%d/%m/%y") + + +class MyDeserializer: + @dispatch + def deserialize(self, cls: Type[datetime], value: Any) -> datetime: + return datetime.strptime(value, "%d/%m/%y") + + +class MySerializer2: + @dispatch + def serialize(self, value: int) -> str: + return str(value) + + +class MyDeserializer2: + @dispatch + def deserialize(self, cls: Type[int], value: Any) -> int: + return int(value) + + +class MySerializer3: + @dispatch + def serialize(self, value: float) -> str: + return str(value) + + +class MyDeserializer3: + @dispatch + def deserialize(self, cls: Type[float], value: Any) -> float: + return float(value) + + +add_serializer(MySerializer()) +add_serializer(MySerializer2()) +add_deserializer(MyDeserializer()) +add_deserializer(MyDeserializer2()) + + +@serde(class_serializer=MySerializer3(), class_deserializer=MyDeserializer3()) +@dataclass +class Foo: + a: datetime + b: int + c: float + + +def main() -> None: + dt = datetime(2021, 1, 1, 0, 0, 0) + f = Foo(dt, 10, 100.0) + print(f"Into Json: {to_json(f)}") + + s = '{"a": "01/01/21", "b": "10", "c": "100.0"}' + print(f"From Json: {from_json(Foo, s)}") + + +if __name__ == "__main__": + main() diff --git a/serde/__init__.py b/serde/__init__.py index c6c1138d..4311210c 100644 --- a/serde/__init__.py +++ b/serde/__init__.py @@ -44,6 +44,8 @@ init, logger, should_impl_dataclass, + add_serializer, + add_deserializer, ) from .de import ( DeserializeFunc, @@ -103,6 +105,8 @@ "logger", "ClassSerializer", "ClassDeserializer", + "add_serializer", + "add_deserializer", ] diff --git a/serde/core.py b/serde/core.py index b8eb1021..1688462d 100644 --- a/serde/core.py +++ b/serde/core.py @@ -1057,3 +1057,22 @@ class ClassDeserializer(Protocol): def deserialize(self, cls: Any, value: Any) -> Any: pass + + +GLOBAL_CLASS_SERIALIZER: List[ClassSerializer] = [] + +GLOBAL_CLASS_DESERIALIZER: List[ClassDeserializer] = [] + + +def add_serializer(serializer: ClassSerializer) -> None: + """ + Register custom global serializer. + """ + GLOBAL_CLASS_SERIALIZER.append(serializer) + + +def add_deserializer(deserializer: ClassDeserializer) -> None: + """ + Register custom global deserializer. + """ + GLOBAL_CLASS_DESERIALIZER.append(deserializer) diff --git a/serde/de.py b/serde/de.py index 26557717..5088a4c7 100644 --- a/serde/de.py +++ b/serde/de.py @@ -5,15 +5,27 @@ from __future__ import annotations import abc +import itertools import collections import dataclasses import functools import typing from dataclasses import dataclass, is_dataclass -from typing import Any, Callable, Dict, Generic, List, Optional, TypeVar, overload, Union, Sequence +from typing import ( + Any, + Callable, + Dict, + Generic, + List, + Optional, + TypeVar, + overload, + Union, + Sequence, + Iterable, +) import jinja2 -import plum from typing_extensions import Type, dataclass_transform from .compat import ( @@ -57,6 +69,7 @@ typename, ) from .core import ( + GLOBAL_CLASS_DESERIALIZER, ClassDeserializer, FROM_DICT, FROM_ITER, @@ -231,6 +244,12 @@ def wrap(cls: Type[T]) -> Type[T]: scope = Scope(cls, reuse_instances_default=reuse_instances_default) setattr(cls, SERDE_SCOPE, scope) + class_deserializers: List[ClassDeserializer] = list( + itertools.chain( + GLOBAL_CLASS_DESERIALIZER, [class_deserializer] if class_deserializer else [] + ) + ) + # Set some globals for all generated functions g["cls"] = cls g["serde_scope"] = scope @@ -250,7 +269,7 @@ def wrap(cls: Type[T]) -> Type[T]: g["coerce"] = coerce g["_exists_by_aliases"] = _exists_by_aliases g["_get_by_aliases"] = _get_by_aliases - g["class_deserializer"] = class_deserializer + g["class_deserializers"] = class_deserializers if deserializer: g["serde_legacy_custom_class_deserializer"] = functools.partial( serde_legacy_custom_class_deserializer, custom=deserializer @@ -685,16 +704,20 @@ def render(self, arg: DeField[Any]) -> str: """ Render rvalue """ - implemented_methods: Dict[Type[Any], plum.Signature] = {} - if self.class_deserializer: - implemented_methods = { - get_args(sig.types[1])[0]: sig - for sig in self.class_deserializer.__class__.deserialize.methods # type: ignore - } + implemented_methods: Dict[Type[Any], int] = {} + class_deserializers: Iterable[ClassDeserializer] = itertools.chain( + GLOBAL_CLASS_DESERIALIZER, [self.class_deserializer] if self.class_deserializer else [] + ) + for n, class_deserializer in enumerate(class_deserializers): + for sig in class_deserializer.__class__.deserialize.methods: # type: ignore + implemented_methods[get_args(sig.types[1])[0]] = n custom_deserializer_available = arg.type in implemented_methods if custom_deserializer_available and not arg.deserializer: - res = f"class_deserializer.deserialize({typename(arg.type)}, {arg.data})" + res = ( + f"class_deserializers[{implemented_methods[arg.type]}].deserialize(" + f"{typename(arg.type)}, {arg.data})" + ) elif arg.deserializer and arg.deserializer.inner is not default_deserializer: res = self.custom_field_deserializer(arg) elif is_generic(arg.type): diff --git a/serde/se.py b/serde/se.py index 29c666cb..884c9f90 100644 --- a/serde/se.py +++ b/serde/se.py @@ -5,11 +5,11 @@ from __future__ import annotations import abc -import plum import copy import dataclasses import functools import typing +import itertools from dataclasses import dataclass, is_dataclass from typing import ( Any, @@ -86,6 +86,7 @@ raise_unsupported_type, render_type_check, union_func_name, + GLOBAL_CLASS_SERIALIZER, ) from .numpy import ( is_numpy_array, @@ -223,6 +224,10 @@ def wrap(cls: Type[T]) -> Type[T]: ) setattr(cls, SERDE_SCOPE, scope) + class_serializers: List[ClassSerializer] = list( + itertools.chain(GLOBAL_CLASS_SERIALIZER, [class_serializer] if class_serializer else []) + ) + # Set some globals for all generated functions g["cls"] = cls g["copy"] = copy @@ -239,7 +244,7 @@ def wrap(cls: Type[T]) -> Type[T]: g["TypeCheck"] = TypeCheck g["NoCheck"] = NoCheck g["coerce"] = coerce - g["class_serializer"] = class_serializer + g["class_serializers"] = class_serializers if serializer: g["serde_legacy_custom_class_serializer"] = functools.partial( serde_legacy_custom_class_serializer, custom=serializer @@ -727,16 +732,17 @@ def render(self, arg: SeField[Any]) -> str: (coerce(str, foo[0]), foo[1].__serde__.funcs['to_iter'](foo[1], reuse_instances=reuse_instances, \ convert_sets=convert_sets), coerce(int, foo[2]),)" """ - implemented_methods: Dict[Type[Any], plum.Signature] = {} - if self.class_serializer: - implemented_methods = { - sig.types[1]: sig - for sig in self.class_serializer.__class__.serialize.methods # type: ignore - } + implemented_methods: Dict[Type[Any], int] = {} + class_serializers: Iterable[ClassSerializer] = itertools.chain( + GLOBAL_CLASS_SERIALIZER, [self.class_serializer] if self.class_serializer else [] + ) + for n, class_serializer in enumerate(class_serializers): + for sig in class_serializer.__class__.serialize.methods: # type: ignore + implemented_methods[sig.types[1]] = n custom_serializer_available = arg.type in implemented_methods if custom_serializer_available and not arg.serializer: - res = f"class_serializer.serialize({arg.varname})" + res = f"class_serializers[{implemented_methods[arg.type]}].serialize({arg.varname})" elif arg.serializer and arg.serializer.inner is not default_serializer: res = self.custom_field_serializer(arg) elif is_dataclass(arg.type):