From 61c08d4a4e8323be9260f49d461ede35fe68597e Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Mon, 1 Apr 2024 16:02:16 -0700 Subject: [PATCH 01/24] Post init option for class generator --- src/hdmf/build/classgenerator.py | 9 +++++++-- src/hdmf/build/manager.py | 13 +++++++++++-- src/hdmf/common/__init__.py | 8 ++++++-- src/hdmf/utils.py | 2 ++ 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index d2e7d4fc0..2eb88a0f3 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -1,5 +1,6 @@ from copy import deepcopy from datetime import datetime, date +import types import numpy as np @@ -35,6 +36,8 @@ def register_generator(self, **kwargs): {'name': 'spec', 'type': BaseStorageSpec, 'doc': ''}, {'name': 'parent_cls', 'type': type, 'doc': ''}, {'name': 'attr_names', 'type': dict, 'doc': ''}, + {'name': 'post_init_method', 'type': types.FunctionType, 'default': None, + 'doc': 'The function used as a post_init method to validate the class generation.'}, {'name': 'type_map', 'type': 'hdmf.build.manager.TypeMap', 'doc': ''}, returns='the class for the given namespace and data_type', rtype=type) def generate_class(self, **kwargs): @@ -42,8 +45,8 @@ def generate_class(self, **kwargs): If no class has been associated with the ``data_type`` from ``namespace``, a class will be dynamically created and returned. """ - data_type, spec, parent_cls, attr_names, type_map = getargs('data_type', 'spec', 'parent_cls', 'attr_names', - 'type_map', kwargs) + data_type, spec, parent_cls, attr_names, type_map, post_init_method = getargs('data_type', 'spec', 'parent_cls', 'attr_names', + 'type_map', 'post_init_method', kwargs) not_inherited_fields = dict() for k, field_spec in attr_names.items(): @@ -82,6 +85,8 @@ def generate_class(self, **kwargs): + str(e) + " Please define that type before defining '%s'." % name) cls = ExtenderMeta(data_type, tuple(bases), classdict) + if post_init_method is not None: + setattr(post_init_method, cls.__postinit, True) return cls diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index a26de3279..d1eb6f3b0 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -1,6 +1,7 @@ import logging from collections import OrderedDict, deque from copy import copy +import types from .builders import DatasetBuilder, GroupBuilder, LinkBuilder, Builder, BaseBuilder from .classgenerator import ClassGenerator, CustomClassGenerator, MCIClassGenerator @@ -498,11 +499,14 @@ def get_container_cls(self, **kwargs): created and returned. """ # NOTE: this internally used function get_container_cls will be removed in favor of get_dt_container_cls + # Deprecated: Will be removed by HDMF 4.0 namespace, data_type, autogen = getargs('namespace', 'data_type', 'autogen', kwargs) return self.get_dt_container_cls(data_type, namespace, autogen) @docval({"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"}, {"name": "namespace", "type": str, "doc": "the namespace containing the data_type", "default": None}, + {'name': 'post_init_method', 'type': types.FunctionType, 'default': None, + 'doc': 'The function used as a post_init method to validate the class generation.'}, {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, returns='the class for the given namespace and data_type', rtype=type) def get_dt_container_cls(self, **kwargs): @@ -513,7 +517,7 @@ def get_dt_container_cls(self, **kwargs): Replaces get_container_cls but namespace is optional. If namespace is unknown, it will be looked up from all namespaces. """ - namespace, data_type, autogen = getargs('namespace', 'data_type', 'autogen', kwargs) + namespace, data_type, post_init_method, autogen = getargs('namespace', 'data_type', 'post_init_method','autogen', kwargs) # namespace is unknown, so look it up if namespace is None: @@ -532,7 +536,12 @@ def get_dt_container_cls(self, **kwargs): self.__check_dependent_types(spec, namespace) parent_cls = self.__get_parent_cls(namespace, data_type, spec) attr_names = self.__default_mapper_cls.get_attr_names(spec) - cls = self.__class_generator.generate_class(data_type, spec, parent_cls, attr_names, self) + cls = self.__class_generator.generate_class(data_type=data_type, + spec=spec, + parent_cls=parent_cls, + attr_names=attr_names, + post_init_method=post_init_method, + type_map=self) self.register_container_type(namespace, data_type, cls) return cls diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 248ca1095..623c4ec6a 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -3,6 +3,7 @@ ''' import os.path from copy import deepcopy +import types CORE_NAMESPACE = 'hdmf-common' EXP_NAMESPACE = 'hdmf-experimental' @@ -136,12 +137,15 @@ def available_namespaces(): @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the Container class for'}, {'name': 'namespace', 'type': str, 'doc': 'the namespace the data_type is defined in'}, + {'name': 'post_init_method', 'type': types.FunctionType, 'default': None, + 'doc': 'The function used as a post_init method to validate the class generation.'}, + {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, is_method=False) def get_class(**kwargs): """Get the class object of the Container subclass corresponding to a given neurdata_type. """ - data_type, namespace = getargs('data_type', 'namespace', kwargs) - return __TYPE_MAP.get_dt_container_cls(data_type, namespace) + data_type, namespace, post_init_method = getargs('data_type', 'namespace', 'post_init_method', kwargs) + return __TYPE_MAP.get_dt_container_cls(data_type, namespace, post_init_method) @docval({'name': 'extensions', 'type': (str, TypeMap, list), diff --git a/src/hdmf/utils.py b/src/hdmf/utils.py index 5e0b61539..d4ccde647 100644 --- a/src/hdmf/utils.py +++ b/src/hdmf/utils.py @@ -513,6 +513,8 @@ def __resolve_type(t): return t elif isinstance(t, type): return t + # elif isinstance(t, types.FunctionType): + # return t elif isinstance(t, (list, tuple)): ret = list() for i in t: From a735a15267b7c6896b70d2a39f0870c22dad0d25 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Mon, 1 Apr 2024 16:09:09 -0700 Subject: [PATCH 02/24] Update utils.py --- src/hdmf/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/hdmf/utils.py b/src/hdmf/utils.py index d4ccde647..5e0b61539 100644 --- a/src/hdmf/utils.py +++ b/src/hdmf/utils.py @@ -513,8 +513,6 @@ def __resolve_type(t): return t elif isinstance(t, type): return t - # elif isinstance(t, types.FunctionType): - # return t elif isinstance(t, (list, tuple)): ret = list() for i in t: From 587cc4efd88349d958315ba39e570073d7f35c95 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 3 Apr 2024 11:20:54 -0700 Subject: [PATCH 03/24] checkpoint --- src/hdmf/build/classgenerator.py | 6 ++++++ src/hdmf/utils.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index 2eb88a0f3..2fc106e0b 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -85,10 +85,16 @@ def generate_class(self, **kwargs): + str(e) + " Please define that type before defining '%s'." % name) cls = ExtenderMeta(data_type, tuple(bases), classdict) + breakpoint() if post_init_method is not None: setattr(post_init_method, cls.__postinit, True) return cls + cls = ExtenderMeta(data_type, tuple(bases), classdict) + if post_init_method is not None: + setattr(A, 'post_init_method', post_init_method) + + class TypeDoesNotExistError(Exception): # pragma: no cover pass diff --git a/src/hdmf/utils.py b/src/hdmf/utils.py index 5e0b61539..5c0d89caf 100644 --- a/src/hdmf/utils.py +++ b/src/hdmf/utils.py @@ -876,6 +876,8 @@ def __init__(cls, name, bases, classdict): it = (a for a in it if hasattr(a, cls.__preinit)) for func in it: func(name, bases, classdict) + if name == 'Container': + breakpoint() super().__init__(name, bases, classdict) it = (getattr(cls, n) for n in dir(cls)) it = (a for a in it if hasattr(a, cls.__postinit)) From d2ab947ecc3337631d84846977b671b030f3b6bc Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 3 Apr 2024 14:08:58 -0700 Subject: [PATCH 04/24] clean up --- src/hdmf/build/classgenerator.py | 9 +++++++-- src/hdmf/utils.py | 2 -- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index 2fc106e0b..020cf45be 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -92,8 +92,9 @@ def generate_class(self, **kwargs): cls = ExtenderMeta(data_type, tuple(bases), classdict) if post_init_method is not None: - setattr(A, 'post_init_method', post_init_method) - + cls.post_init_method = MethodType(post_init_method, cls) # set as bounded method + else: + cls.post_init_method = post_init_method # set to None class TypeDoesNotExistError(Exception): # pragma: no cover @@ -354,6 +355,10 @@ def __init__(self, **kwargs): for f in fixed_value_attrs_to_set: self.fields[f] = getattr(not_inherited_fields[f], 'value') + if self.post_init_method is not None: + self.post_init_method(kwargs) + + classdict['__init__'] = __init__ diff --git a/src/hdmf/utils.py b/src/hdmf/utils.py index 5c0d89caf..5e0b61539 100644 --- a/src/hdmf/utils.py +++ b/src/hdmf/utils.py @@ -876,8 +876,6 @@ def __init__(cls, name, bases, classdict): it = (a for a in it if hasattr(a, cls.__preinit)) for func in it: func(name, bases, classdict) - if name == 'Container': - breakpoint() super().__init__(name, bases, classdict) it = (getattr(cls, n) for n in dir(cls)) it = (a for a in it if hasattr(a, cls.__postinit)) From 8b7e601c3ce1583760cac9d4495e076fb8053a94 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 3 Apr 2024 14:17:08 -0700 Subject: [PATCH 05/24] clean up --- src/hdmf/build/classgenerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index 020cf45be..3296766c1 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -92,7 +92,7 @@ def generate_class(self, **kwargs): cls = ExtenderMeta(data_type, tuple(bases), classdict) if post_init_method is not None: - cls.post_init_method = MethodType(post_init_method, cls) # set as bounded method + cls.post_init_method = types.MethodType(post_init_method, cls) # set as bounded method else: cls.post_init_method = post_init_method # set to None From ef0e5ed71e24cfd12a56cb637ff43c99f527d510 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 3 Apr 2024 14:18:21 -0700 Subject: [PATCH 06/24] callable --- src/hdmf/build/classgenerator.py | 2 +- src/hdmf/build/manager.py | 2 +- src/hdmf/common/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index 3296766c1..769c2b1c6 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -36,7 +36,7 @@ def register_generator(self, **kwargs): {'name': 'spec', 'type': BaseStorageSpec, 'doc': ''}, {'name': 'parent_cls', 'type': type, 'doc': ''}, {'name': 'attr_names', 'type': dict, 'doc': ''}, - {'name': 'post_init_method', 'type': types.FunctionType, 'default': None, + {'name': 'post_init_method', 'type': types.Callable, 'default': None, 'doc': 'The function used as a post_init method to validate the class generation.'}, {'name': 'type_map', 'type': 'hdmf.build.manager.TypeMap', 'doc': ''}, returns='the class for the given namespace and data_type', rtype=type) diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index d1eb6f3b0..03056ae2b 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -505,7 +505,7 @@ def get_container_cls(self, **kwargs): @docval({"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"}, {"name": "namespace", "type": str, "doc": "the namespace containing the data_type", "default": None}, - {'name': 'post_init_method', 'type': types.FunctionType, 'default': None, + {'name': 'post_init_method', 'type': types.Callable, 'default': None, 'doc': 'The function used as a post_init method to validate the class generation.'}, {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, returns='the class for the given namespace and data_type', rtype=type) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 623c4ec6a..2e0ea86ac 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -137,7 +137,7 @@ def available_namespaces(): @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the Container class for'}, {'name': 'namespace', 'type': str, 'doc': 'the namespace the data_type is defined in'}, - {'name': 'post_init_method', 'type': types.FunctionType, 'default': None, + {'name': 'post_init_method', 'type': types.Callable, 'default': None, 'doc': 'The function used as a post_init method to validate the class generation.'}, {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, is_method=False) From a6d330c5e91a8e5c99f95d54ec63f302efefc670 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 3 Apr 2024 15:07:46 -0700 Subject: [PATCH 07/24] typing --- src/hdmf/build/classgenerator.py | 3 ++- src/hdmf/build/manager.py | 4 ++-- src/hdmf/common/__init__.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index 769c2b1c6..dc648be42 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -1,6 +1,7 @@ from copy import deepcopy from datetime import datetime, date import types +import typing import numpy as np @@ -36,7 +37,7 @@ def register_generator(self, **kwargs): {'name': 'spec', 'type': BaseStorageSpec, 'doc': ''}, {'name': 'parent_cls', 'type': type, 'doc': ''}, {'name': 'attr_names', 'type': dict, 'doc': ''}, - {'name': 'post_init_method', 'type': types.Callable, 'default': None, + {'name': 'post_init_method', 'type': typing.Callable, 'default': None, 'doc': 'The function used as a post_init method to validate the class generation.'}, {'name': 'type_map', 'type': 'hdmf.build.manager.TypeMap', 'doc': ''}, returns='the class for the given namespace and data_type', rtype=type) diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index 03056ae2b..b845a08cb 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -1,7 +1,7 @@ import logging from collections import OrderedDict, deque from copy import copy -import types +import typing from .builders import DatasetBuilder, GroupBuilder, LinkBuilder, Builder, BaseBuilder from .classgenerator import ClassGenerator, CustomClassGenerator, MCIClassGenerator @@ -505,7 +505,7 @@ def get_container_cls(self, **kwargs): @docval({"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"}, {"name": "namespace", "type": str, "doc": "the namespace containing the data_type", "default": None}, - {'name': 'post_init_method', 'type': types.Callable, 'default': None, + {'name': 'post_init_method', 'type': typing.Callable, 'default': None, 'doc': 'The function used as a post_init method to validate the class generation.'}, {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, returns='the class for the given namespace and data_type', rtype=type) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 2e0ea86ac..dce5295b3 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -3,7 +3,7 @@ ''' import os.path from copy import deepcopy -import types +import typing CORE_NAMESPACE = 'hdmf-common' EXP_NAMESPACE = 'hdmf-experimental' @@ -137,7 +137,7 @@ def available_namespaces(): @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the Container class for'}, {'name': 'namespace', 'type': str, 'doc': 'the namespace the data_type is defined in'}, - {'name': 'post_init_method', 'type': types.Callable, 'default': None, + {'name': 'post_init_method', 'type': typing.Callable, 'default': None, 'doc': 'The function used as a post_init method to validate the class generation.'}, {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, is_method=False) From e983aa0c1ca3015de4de830ab7fc088a4bbce572 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 3 Apr 2024 15:10:21 -0700 Subject: [PATCH 08/24] typing --- src/hdmf/build/classgenerator.py | 3 +-- src/hdmf/build/manager.py | 4 ++-- src/hdmf/common/__init__.py | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index dc648be42..3296766c1 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -1,7 +1,6 @@ from copy import deepcopy from datetime import datetime, date import types -import typing import numpy as np @@ -37,7 +36,7 @@ def register_generator(self, **kwargs): {'name': 'spec', 'type': BaseStorageSpec, 'doc': ''}, {'name': 'parent_cls', 'type': type, 'doc': ''}, {'name': 'attr_names', 'type': dict, 'doc': ''}, - {'name': 'post_init_method', 'type': typing.Callable, 'default': None, + {'name': 'post_init_method', 'type': types.FunctionType, 'default': None, 'doc': 'The function used as a post_init method to validate the class generation.'}, {'name': 'type_map', 'type': 'hdmf.build.manager.TypeMap', 'doc': ''}, returns='the class for the given namespace and data_type', rtype=type) diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index b845a08cb..d1eb6f3b0 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -1,7 +1,7 @@ import logging from collections import OrderedDict, deque from copy import copy -import typing +import types from .builders import DatasetBuilder, GroupBuilder, LinkBuilder, Builder, BaseBuilder from .classgenerator import ClassGenerator, CustomClassGenerator, MCIClassGenerator @@ -505,7 +505,7 @@ def get_container_cls(self, **kwargs): @docval({"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"}, {"name": "namespace", "type": str, "doc": "the namespace containing the data_type", "default": None}, - {'name': 'post_init_method', 'type': typing.Callable, 'default': None, + {'name': 'post_init_method', 'type': types.FunctionType, 'default': None, 'doc': 'The function used as a post_init method to validate the class generation.'}, {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, returns='the class for the given namespace and data_type', rtype=type) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index dce5295b3..623c4ec6a 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -3,7 +3,7 @@ ''' import os.path from copy import deepcopy -import typing +import types CORE_NAMESPACE = 'hdmf-common' EXP_NAMESPACE = 'hdmf-experimental' @@ -137,7 +137,7 @@ def available_namespaces(): @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the Container class for'}, {'name': 'namespace', 'type': str, 'doc': 'the namespace the data_type is defined in'}, - {'name': 'post_init_method', 'type': typing.Callable, 'default': None, + {'name': 'post_init_method', 'type': types.FunctionType, 'default': None, 'doc': 'The function used as a post_init method to validate the class generation.'}, {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, is_method=False) From 245a1ea6491f5362f875d11f4b96d591661928d5 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 3 Apr 2024 15:14:47 -0700 Subject: [PATCH 09/24] ruff --- src/hdmf/build/classgenerator.py | 19 +++++++++---------- src/hdmf/build/manager.py | 7 ++++--- src/hdmf/common/__init__.py | 4 ++-- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index 3296766c1..ababacf44 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -1,6 +1,6 @@ from copy import deepcopy from datetime import datetime, date -import types +import types as tp import numpy as np @@ -36,7 +36,7 @@ def register_generator(self, **kwargs): {'name': 'spec', 'type': BaseStorageSpec, 'doc': ''}, {'name': 'parent_cls', 'type': type, 'doc': ''}, {'name': 'attr_names', 'type': dict, 'doc': ''}, - {'name': 'post_init_method', 'type': types.FunctionType, 'default': None, + {'name': 'post_init_method', 'type': tp.FunctionType, 'default': None, 'doc': 'The function used as a post_init method to validate the class generation.'}, {'name': 'type_map', 'type': 'hdmf.build.manager.TypeMap', 'doc': ''}, returns='the class for the given namespace and data_type', rtype=type) @@ -45,8 +45,10 @@ def generate_class(self, **kwargs): If no class has been associated with the ``data_type`` from ``namespace``, a class will be dynamically created and returned. """ - data_type, spec, parent_cls, attr_names, type_map, post_init_method = getargs('data_type', 'spec', 'parent_cls', 'attr_names', - 'type_map', 'post_init_method', kwargs) + data_type, spec, parent_cls, attr_names, type_map, post_init_method = getargs('data_type', 'spec', + 'parent_cls', 'attr_names', + 'type_map', + 'post_init_method', kwargs) not_inherited_fields = dict() for k, field_spec in attr_names.items(): @@ -85,17 +87,14 @@ def generate_class(self, **kwargs): + str(e) + " Please define that type before defining '%s'." % name) cls = ExtenderMeta(data_type, tuple(bases), classdict) - breakpoint() - if post_init_method is not None: - setattr(post_init_method, cls.__postinit, True) - return cls - cls = ExtenderMeta(data_type, tuple(bases), classdict) if post_init_method is not None: - cls.post_init_method = types.MethodType(post_init_method, cls) # set as bounded method + cls.post_init_method = tp.MethodType(post_init_method, cls) # set as bounded method else: cls.post_init_method = post_init_method # set to None + return cls + class TypeDoesNotExistError(Exception): # pragma: no cover pass diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index d1eb6f3b0..49ba7476e 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -1,7 +1,7 @@ import logging from collections import OrderedDict, deque from copy import copy -import types +import types as tp from .builders import DatasetBuilder, GroupBuilder, LinkBuilder, Builder, BaseBuilder from .classgenerator import ClassGenerator, CustomClassGenerator, MCIClassGenerator @@ -505,7 +505,7 @@ def get_container_cls(self, **kwargs): @docval({"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"}, {"name": "namespace", "type": str, "doc": "the namespace containing the data_type", "default": None}, - {'name': 'post_init_method', 'type': types.FunctionType, 'default': None, + {'name': 'post_init_method', 'type': tp.FunctionType, 'default': None, 'doc': 'The function used as a post_init method to validate the class generation.'}, {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, returns='the class for the given namespace and data_type', rtype=type) @@ -517,7 +517,8 @@ def get_dt_container_cls(self, **kwargs): Replaces get_container_cls but namespace is optional. If namespace is unknown, it will be looked up from all namespaces. """ - namespace, data_type, post_init_method, autogen = getargs('namespace', 'data_type', 'post_init_method','autogen', kwargs) + namespace, data_type, post_init_method, autogen = getargs('namespace', 'data_type', + 'post_init_method','autogen', kwargs) # namespace is unknown, so look it up if namespace is None: diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 623c4ec6a..76ccfca02 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -3,7 +3,7 @@ ''' import os.path from copy import deepcopy -import types +import types as tp CORE_NAMESPACE = 'hdmf-common' EXP_NAMESPACE = 'hdmf-experimental' @@ -137,7 +137,7 @@ def available_namespaces(): @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the Container class for'}, {'name': 'namespace', 'type': str, 'doc': 'the namespace the data_type is defined in'}, - {'name': 'post_init_method', 'type': types.FunctionType, 'default': None, + {'name': 'post_init_method', 'type': tp.FunctionType, 'default': None, 'doc': 'The function used as a post_init method to validate the class generation.'}, {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, is_method=False) From 5df6bc13a17473b4f8f0695aaa12c9db1a4fb2eb Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 3 Apr 2024 16:44:11 -0700 Subject: [PATCH 10/24] callable --- src/hdmf/build/classgenerator.py | 3 ++- src/hdmf/build/manager.py | 4 ++-- src/hdmf/common/__init__.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index ababacf44..5b3a5db01 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -1,5 +1,6 @@ from copy import deepcopy from datetime import datetime, date +from collections.abc import Callable import types as tp import numpy as np @@ -36,7 +37,7 @@ def register_generator(self, **kwargs): {'name': 'spec', 'type': BaseStorageSpec, 'doc': ''}, {'name': 'parent_cls', 'type': type, 'doc': ''}, {'name': 'attr_names', 'type': dict, 'doc': ''}, - {'name': 'post_init_method', 'type': tp.FunctionType, 'default': None, + {'name': 'post_init_method', 'type': Callable, 'default': None, 'doc': 'The function used as a post_init method to validate the class generation.'}, {'name': 'type_map', 'type': 'hdmf.build.manager.TypeMap', 'doc': ''}, returns='the class for the given namespace and data_type', rtype=type) diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index 49ba7476e..9bdc2fbdb 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -1,7 +1,7 @@ import logging from collections import OrderedDict, deque from copy import copy -import types as tp +from collections.abc import Callable from .builders import DatasetBuilder, GroupBuilder, LinkBuilder, Builder, BaseBuilder from .classgenerator import ClassGenerator, CustomClassGenerator, MCIClassGenerator @@ -505,7 +505,7 @@ def get_container_cls(self, **kwargs): @docval({"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"}, {"name": "namespace", "type": str, "doc": "the namespace containing the data_type", "default": None}, - {'name': 'post_init_method', 'type': tp.FunctionType, 'default': None, + {'name': 'post_init_method', 'type': Callable, 'default': None, 'doc': 'The function used as a post_init method to validate the class generation.'}, {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, returns='the class for the given namespace and data_type', rtype=type) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 76ccfca02..7416fcf3d 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -3,7 +3,7 @@ ''' import os.path from copy import deepcopy -import types as tp +from collections.abc import Callable CORE_NAMESPACE = 'hdmf-common' EXP_NAMESPACE = 'hdmf-experimental' @@ -137,7 +137,7 @@ def available_namespaces(): @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the Container class for'}, {'name': 'namespace', 'type': str, 'doc': 'the namespace the data_type is defined in'}, - {'name': 'post_init_method', 'type': tp.FunctionType, 'default': None, + {'name': 'post_init_method', 'type': Callable, 'default': None, 'doc': 'The function used as a post_init method to validate the class generation.'}, {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, is_method=False) From 4799bb90056efd009958cd0686f021b00bc9b331 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 3 Apr 2024 18:20:47 -0700 Subject: [PATCH 11/24] concept --- src/hdmf/build/classgenerator.py | 4 +- src/hdmf/build/manager.py | 1 + tests/unit/build_tests/test_classgenerator.py | 52 +++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index 5b3a5db01..c5de3b20d 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -93,7 +93,6 @@ def generate_class(self, **kwargs): cls.post_init_method = tp.MethodType(post_init_method, cls) # set as bounded method else: cls.post_init_method = post_init_method # set to None - return cls @@ -330,6 +329,7 @@ def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name): @docval(*docval_args, allow_positional=AllowPositional.WARNING) def __init__(self, **kwargs): + original_kwargs = dict(kwargs) if name is not None: # force container name to be the fixed name in the spec kwargs.update(name=name) @@ -356,7 +356,7 @@ def __init__(self, **kwargs): self.fields[f] = getattr(not_inherited_fields[f], 'value') if self.post_init_method is not None: - self.post_init_method(kwargs) + self.post_init_method(**original_kwargs) classdict['__init__'] = __init__ diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index 9bdc2fbdb..25b9b81bd 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -532,6 +532,7 @@ def get_dt_container_cls(self, **kwargs): raise ValueError("Namespace could not be resolved.") cls = self.__get_container_cls(namespace, data_type) + if cls is None and autogen: # dynamically generate a class spec = self.__ns_catalog.get_spec(namespace, data_type) self.__check_dependent_types(spec, namespace) diff --git a/tests/unit/build_tests/test_classgenerator.py b/tests/unit/build_tests/test_classgenerator.py index 0c117820b..29a3fe175 100644 --- a/tests/unit/build_tests/test_classgenerator.py +++ b/tests/unit/build_tests/test_classgenerator.py @@ -82,6 +82,58 @@ def test_no_generators(self): self.assertTrue(hasattr(cls, '__init__')) +class TestPostInitGetClass(TestCase): + def setUp(self): + # self.bar_spec = GroupSpec( + # doc='A test group specification with a data type', + # data_type_def='Bar', + # datasets=[ + # DatasetSpec( + # doc='a dataset', + # dtype='int', + # name='data', + # attributes=[AttributeSpec(name='attr1', doc='an integer attribute', dtype='int')] + # ) + # ]) + # specs = [self.bar_spec] + # containers = {'Bar': Bar} + # from hdmf.common import get_type_map + # self.type_map = get_type_map() + # self.spec_catalog = self.type_map.namespace_catalog.get_namespace(CORE_NAMESPACE).catalog + spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Baz', + attributes=[ + AttributeSpec(name='attr1', doc='a int attribute', dtype='int') + ] + ) + + spec_catalog = SpecCatalog() + spec_catalog.register_spec(spec, 'test.yaml') + namespace = SpecNamespace( + doc='a test namespace', + name=CORE_NAMESPACE, + schema=[{'source': 'test.yaml'}], + version='0.1.0', + catalog=spec_catalog + ) + namespace_catalog = NamespaceCatalog() + namespace_catalog.add_namespace(CORE_NAMESPACE, namespace) + self.type_map = TypeMap(namespace_catalog) + + + def test_post_init(self): + def post_init_method(self, **kwargs): + attr1 = kwargs['attr1'] + if attr1<10: + msg = "attr1 should be >=10" + raise ValueError(msg) + + cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE, post_init_method) + + with self.assertRaises(ValueError): + instance = cls(name='instance', attr1=9) + class TestDynamicContainer(TestCase): def setUp(self): From c14d37aed40225e02e4f290d9ab23d93d471de0b Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 10 Apr 2024 08:14:52 -0700 Subject: [PATCH 12/24] checkpoint --- src/hdmf/build/classgenerator.py | 11 +++++------ tests/unit/build_tests/test_classgenerator.py | 18 +----------------- 2 files changed, 6 insertions(+), 23 deletions(-) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index c5de3b20d..e0b0b3b61 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -1,7 +1,6 @@ from copy import deepcopy from datetime import datetime, date from collections.abc import Callable -import types as tp import numpy as np @@ -88,11 +87,8 @@ def generate_class(self, **kwargs): + str(e) + " Please define that type before defining '%s'." % name) cls = ExtenderMeta(data_type, tuple(bases), classdict) + cls.post_init_method = post_init_method - if post_init_method is not None: - cls.post_init_method = tp.MethodType(post_init_method, cls) # set as bounded method - else: - cls.post_init_method = post_init_method # set to None return cls @@ -358,7 +354,6 @@ def __init__(self, **kwargs): if self.post_init_method is not None: self.post_init_method(**original_kwargs) - classdict['__init__'] = __init__ @@ -433,6 +428,7 @@ def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name): def __init__(self, **kwargs): # store the values passed to init for each MCI attribute so that they can be added # after calling __init__ + original_kwargs = dict(kwargs) new_kwargs = list() for field_clsconf in classdict['__clsconf__']: attr_name = field_clsconf['attr'] @@ -460,5 +456,8 @@ def __init__(self, **kwargs): add_method = getattr(self, new_kwarg['add_method_name']) add_method(new_kwarg['value']) + if self.post_init_method is not None: + self.post_init_method(**original_kwargs) + # override __init__ classdict['__init__'] = __init__ diff --git a/tests/unit/build_tests/test_classgenerator.py b/tests/unit/build_tests/test_classgenerator.py index 29a3fe175..4838ee827 100644 --- a/tests/unit/build_tests/test_classgenerator.py +++ b/tests/unit/build_tests/test_classgenerator.py @@ -84,22 +84,6 @@ def test_no_generators(self): class TestPostInitGetClass(TestCase): def setUp(self): - # self.bar_spec = GroupSpec( - # doc='A test group specification with a data type', - # data_type_def='Bar', - # datasets=[ - # DatasetSpec( - # doc='a dataset', - # dtype='int', - # name='data', - # attributes=[AttributeSpec(name='attr1', doc='an integer attribute', dtype='int')] - # ) - # ]) - # specs = [self.bar_spec] - # containers = {'Bar': Bar} - # from hdmf.common import get_type_map - # self.type_map = get_type_map() - # self.spec_catalog = self.type_map.namespace_catalog.get_namespace(CORE_NAMESPACE).catalog spec = GroupSpec( doc='A test group specification with a data type', data_type_def='Baz', @@ -132,7 +116,7 @@ def post_init_method(self, **kwargs): cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE, post_init_method) with self.assertRaises(ValueError): - instance = cls(name='instance', attr1=9) + cls(name='instance', attr1=9) class TestDynamicContainer(TestCase): From 1417d3a695bb2675084fe0246f58e74bf9ba819e Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 10 Apr 2024 08:48:56 -0700 Subject: [PATCH 13/24] tests --- src/hdmf/build/classgenerator.py | 2 +- tests/unit/build_tests/test_classgenerator.py | 56 ++++++++++++++++--- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index e0b0b3b61..ac3bf1d42 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -322,7 +322,7 @@ def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name): fixed_value_attrs_to_set.append(attr_name) elif attr_name not in attrs_not_to_set: attrs_to_set.append(attr_name) - + @docval(*docval_args, allow_positional=AllowPositional.WARNING) def __init__(self, **kwargs): original_kwargs = dict(kwargs) diff --git a/tests/unit/build_tests/test_classgenerator.py b/tests/unit/build_tests/test_classgenerator.py index 4838ee827..28e47f1ad 100644 --- a/tests/unit/build_tests/test_classgenerator.py +++ b/tests/unit/build_tests/test_classgenerator.py @@ -84,6 +84,14 @@ def test_no_generators(self): class TestPostInitGetClass(TestCase): def setUp(self): + def post_init_method(self, **kwargs): + attr1 = kwargs['attr1'] + if attr1<10: + msg = "attr1 should be >=10" + raise ValueError(msg) + self.post_init=post_init_method + + def test_post_init(self): spec = GroupSpec( doc='A test group specification with a data type', data_type_def='Baz', @@ -103,21 +111,51 @@ def setUp(self): ) namespace_catalog = NamespaceCatalog() namespace_catalog.add_namespace(CORE_NAMESPACE, namespace) - self.type_map = TypeMap(namespace_catalog) - + type_map = TypeMap(namespace_catalog) - def test_post_init(self): - def post_init_method(self, **kwargs): - attr1 = kwargs['attr1'] - if attr1<10: - msg = "attr1 should be >=10" - raise ValueError(msg) - cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE, post_init_method) + cls = type_map.get_dt_container_cls('Baz', CORE_NAMESPACE, self.post_init) with self.assertRaises(ValueError): cls(name='instance', attr1=9) + def test_multi_container_post_init(self): + bar_spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Bar', + datasets=[ + DatasetSpec( + doc='a dataset', + dtype='int', + name='data', + attributes=[AttributeSpec(name='attr2', doc='an integer attribute', dtype='int')] + ) + ], + attributes=[AttributeSpec(name='attr1', doc='a string attribute', dtype='text')]) + + multi_spec = GroupSpec(doc='A test extension that contains a multi', + data_type_def='Multi', + groups=[GroupSpec(data_type_inc=bar_spec, doc='test multi', quantity='*')], + attributes=[AttributeSpec(name='attr1', doc='a float attribute', dtype='float')]) + + spec_catalog = SpecCatalog() + spec_catalog.register_spec(bar_spec, 'test.yaml') + spec_catalog.register_spec(multi_spec, 'test.yaml') + namespace = SpecNamespace( + doc='a test namespace', + name=CORE_NAMESPACE, + schema=[{'source': 'test.yaml'}], + version='0.1.0', + catalog=spec_catalog + ) + namespace_catalog = NamespaceCatalog() + namespace_catalog.add_namespace(CORE_NAMESPACE, namespace) + type_map = TypeMap(namespace_catalog) + Multi = type_map.get_dt_container_cls('Multi', CORE_NAMESPACE, self.post_init) + + with self.assertRaises(ValueError): + Multi(name='instance', attr1=9.1) + class TestDynamicContainer(TestCase): def setUp(self): From 2b6a693812f282ccab879d3c52eaf5896edcb84c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Apr 2024 15:49:17 +0000 Subject: [PATCH 14/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/hdmf/build/classgenerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index ac3bf1d42..e0b0b3b61 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -322,7 +322,7 @@ def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name): fixed_value_attrs_to_set.append(attr_name) elif attr_name not in attrs_not_to_set: attrs_to_set.append(attr_name) - + @docval(*docval_args, allow_positional=AllowPositional.WARNING) def __init__(self, **kwargs): original_kwargs = dict(kwargs) From 420c1d1041b1253d64e2365759b480ec6e23cba3 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 10 Apr 2024 10:15:34 -0700 Subject: [PATCH 15/24] space --- tests/unit/build_tests/test_classgenerator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/build_tests/test_classgenerator.py b/tests/unit/build_tests/test_classgenerator.py index 28e47f1ad..a36237946 100644 --- a/tests/unit/build_tests/test_classgenerator.py +++ b/tests/unit/build_tests/test_classgenerator.py @@ -113,7 +113,6 @@ def test_post_init(self): namespace_catalog.add_namespace(CORE_NAMESPACE, namespace) type_map = TypeMap(namespace_catalog) - cls = type_map.get_dt_container_cls('Baz', CORE_NAMESPACE, self.post_init) with self.assertRaises(ValueError): From 1b78eabd7d2052a5986144a29a135877a5c8fb97 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 10 Apr 2024 15:40:55 -0700 Subject: [PATCH 16/24] checkpoint: before editing existing tests --- src/hdmf/build/classgenerator.py | 15 +++++++++++++-- tests/unit/build_tests/test_classgenerator.py | 7 ++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index e0b0b3b61..f61d67c02 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -323,8 +323,16 @@ def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name): elif attr_name not in attrs_not_to_set: attrs_to_set.append(attr_name) - @docval(*docval_args, allow_positional=AllowPositional.WARNING) + @docval(*docval_args, + {'name': 'post_init_bool', 'type': bool, 'default': True, + 'doc': 'bool to skip post_init for MCIClassGenerator'}, + allow_positional=AllowPositional.WARNING) def __init__(self, **kwargs): + try: + post_init_bool = popargs('post_init_bool', kwargs) + except KeyError: + post_init_bool = True + original_kwargs = dict(kwargs) if name is not None: # force container name to be the fixed name in the spec kwargs.update(name=name) @@ -351,7 +359,8 @@ def __init__(self, **kwargs): for f in fixed_value_attrs_to_set: self.fields[f] = getattr(not_inherited_fields[f], 'value') - if self.post_init_method is not None: + if self.post_init_method is not None and post_init_bool: + # self.post_init_method(**original_kwargs) classdict['__init__'] = __init__ @@ -449,6 +458,7 @@ def __init__(self, **kwargs): kwargs[attr_name] = list() # call the parent class init without the MCI attribute + kwargs['post_init_bool'] = False previous_init(self, **kwargs) # call the add method for each MCI attribute @@ -457,6 +467,7 @@ def __init__(self, **kwargs): add_method(new_kwarg['value']) if self.post_init_method is not None: + self.post_init_method(**original_kwargs) # override __init__ diff --git a/tests/unit/build_tests/test_classgenerator.py b/tests/unit/build_tests/test_classgenerator.py index a36237946..2db1cf65e 100644 --- a/tests/unit/build_tests/test_classgenerator.py +++ b/tests/unit/build_tests/test_classgenerator.py @@ -2,6 +2,7 @@ import os import shutil import tempfile +from warnings import warn from hdmf.build import TypeMap, CustomClassGenerator from hdmf.build.classgenerator import ClassGenerator, MCIClassGenerator @@ -88,7 +89,7 @@ def post_init_method(self, **kwargs): attr1 = kwargs['attr1'] if attr1<10: msg = "attr1 should be >=10" - raise ValueError(msg) + warn(msg) self.post_init=post_init_method def test_post_init(self): @@ -115,7 +116,7 @@ def test_post_init(self): cls = type_map.get_dt_container_cls('Baz', CORE_NAMESPACE, self.post_init) - with self.assertRaises(ValueError): + with self.assertWarns(Warning): cls(name='instance', attr1=9) def test_multi_container_post_init(self): @@ -152,7 +153,7 @@ def test_multi_container_post_init(self): type_map = TypeMap(namespace_catalog) Multi = type_map.get_dt_container_cls('Multi', CORE_NAMESPACE, self.post_init) - with self.assertRaises(ValueError): + with self.assertWarns(Warning): Multi(name='instance', attr1=9.1) class TestDynamicContainer(TestCase): From 14541ceace85cd3cddda434ac0d35032d5173458 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 11 Apr 2024 17:28:50 -0700 Subject: [PATCH 17/24] remove try --- src/hdmf/build/classgenerator.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index f61d67c02..ba19b4e10 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -328,10 +328,7 @@ def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name): 'doc': 'bool to skip post_init for MCIClassGenerator'}, allow_positional=AllowPositional.WARNING) def __init__(self, **kwargs): - try: - post_init_bool = popargs('post_init_bool', kwargs) - except KeyError: - post_init_bool = True + post_init_bool = popargs('post_init_bool', kwargs) original_kwargs = dict(kwargs) if name is not None: # force container name to be the fixed name in the spec From ab136ff6163a3c5c2e1f4fb85d05e62a1cf75b7a Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 2 May 2024 07:49:07 -0700 Subject: [PATCH 18/24] tests pass --- src/hdmf/build/classgenerator.py | 21 +++++++++++++------ tests/unit/build_tests/test_classgenerator.py | 17 +++++++++------ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index ba19b4e10..1728de821 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -323,12 +323,17 @@ def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name): elif attr_name not in attrs_not_to_set: attrs_to_set.append(attr_name) + # We want to use the skip_post_init of the current class and not the parent class + for item in docval_args: + if item['name'] == 'skip_post_init': + docval_args.remove(item) + @docval(*docval_args, - {'name': 'post_init_bool', 'type': bool, 'default': True, - 'doc': 'bool to skip post_init for MCIClassGenerator'}, + {'name': 'skip_post_init', 'type': bool, 'default': False, + 'doc': 'bool to skip post_init'}, allow_positional=AllowPositional.WARNING) def __init__(self, **kwargs): - post_init_bool = popargs('post_init_bool', kwargs) + skip_post_init = popargs('skip_post_init', kwargs) original_kwargs = dict(kwargs) if name is not None: # force container name to be the fixed name in the spec @@ -356,8 +361,7 @@ def __init__(self, **kwargs): for f in fixed_value_attrs_to_set: self.fields[f] = getattr(not_inherited_fields[f], 'value') - if self.post_init_method is not None and post_init_bool: - # + if self.post_init_method is not None and not skip_post_init: self.post_init_method(**original_kwargs) classdict['__init__'] = __init__ @@ -430,6 +434,11 @@ def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name): if '__clsconf__' in classdict: previous_init = classdict['__init__'] + # We want to use the skip_post_init of the current class and not the parent class + for item in docval_args: + if item['name'] == 'skip_post_init': + docval_args.remove(item) + @docval(*docval_args, allow_positional=AllowPositional.WARNING) def __init__(self, **kwargs): # store the values passed to init for each MCI attribute so that they can be added @@ -455,7 +464,7 @@ def __init__(self, **kwargs): kwargs[attr_name] = list() # call the parent class init without the MCI attribute - kwargs['post_init_bool'] = False + kwargs['skip_post_init'] = True previous_init(self, **kwargs) # call the add method for each MCI attribute diff --git a/tests/unit/build_tests/test_classgenerator.py b/tests/unit/build_tests/test_classgenerator.py index 2db1cf65e..52fdc4839 100644 --- a/tests/unit/build_tests/test_classgenerator.py +++ b/tests/unit/build_tests/test_classgenerator.py @@ -183,13 +183,15 @@ def test_dynamic_container_creation(self): AttributeSpec('attr4', 'another float attribute', 'float')]) self.spec_catalog.register_spec(baz_spec, 'extension.yaml') cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE) - expected_args = {'name', 'data', 'attr1', 'attr2', 'attr3', 'attr4'} + expected_args = {'name', 'data', 'attr1', 'attr2', 'attr3', 'attr4', 'skip_post_init'} received_args = set() + for x in get_docval(cls.__init__): if x['name'] != 'foo': received_args.add(x['name']) with self.subTest(name=x['name']): - self.assertNotIn('default', x) + if x['name'] != 'skip_post_init': + self.assertNotIn('default', x) self.assertSetEqual(expected_args, received_args) self.assertEqual(cls.__name__, 'Baz') self.assertTrue(issubclass(cls, Bar)) @@ -209,7 +211,7 @@ def test_dynamic_container_creation_defaults(self): AttributeSpec('attr4', 'another float attribute', 'float')]) self.spec_catalog.register_spec(baz_spec, 'extension.yaml') cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE) - expected_args = {'name', 'data', 'attr1', 'attr2', 'attr3', 'attr4', 'foo'} + expected_args = {'name', 'data', 'attr1', 'attr2', 'attr3', 'attr4', 'foo', 'skip_post_init'} received_args = set(map(lambda x: x['name'], get_docval(cls.__init__))) self.assertSetEqual(expected_args, received_args) self.assertEqual(cls.__name__, 'Baz') @@ -359,13 +361,14 @@ def __init__(self, **kwargs): AttributeSpec('attr4', 'another float attribute', 'float')]) self.spec_catalog.register_spec(baz_spec, 'extension.yaml') cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE) - expected_args = {'name', 'data', 'attr2', 'attr3', 'attr4'} + expected_args = {'name', 'data', 'attr2', 'attr3', 'attr4', 'skip_post_init'} received_args = set() for x in get_docval(cls.__init__): if x['name'] != 'foo': received_args.add(x['name']) with self.subTest(name=x['name']): - self.assertNotIn('default', x) + if x['name'] != 'skip_post_init': + self.assertNotIn('default', x) self.assertSetEqual(expected_args, received_args) self.assertTrue(issubclass(cls, FixedAttrBar)) inst = cls(name="My Baz", data=[1, 2, 3, 4], attr2=1000, attr3=98.6, attr4=1.0) @@ -519,7 +522,7 @@ def setUp(self): def test_init_docval(self): cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE) # generate the class - expected_args = {'name'} # 'attr1' should not be included + expected_args = {'name', 'skip_post_init'} # 'attr1' should not be included received_args = set() for x in get_docval(cls.__init__): received_args.add(x['name']) @@ -592,6 +595,8 @@ def test_gen_parent_class(self): {'name': 'my_baz1', 'doc': 'A composition inside with a fixed name', 'type': baz1_cls}, {'name': 'my_baz2', 'doc': 'A composition inside with a fixed name', 'type': baz2_cls}, {'name': 'my_baz1_link', 'doc': 'A composition inside without a fixed name', 'type': baz1_cls}, + {'name': 'skip_post_init', 'type': bool, 'default': False, + 'doc': 'bool to skip post_init'} )) def test_init_fields(self): From 936ccb0915f752423ac80f9f08a6b5570b10a82f Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 2 May 2024 07:55:11 -0700 Subject: [PATCH 19/24] tests pass --- src/hdmf/build/classgenerator.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index 1728de821..325462452 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -434,11 +434,6 @@ def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name): if '__clsconf__' in classdict: previous_init = classdict['__init__'] - # We want to use the skip_post_init of the current class and not the parent class - for item in docval_args: - if item['name'] == 'skip_post_init': - docval_args.remove(item) - @docval(*docval_args, allow_positional=AllowPositional.WARNING) def __init__(self, **kwargs): # store the values passed to init for each MCI attribute so that they can be added From 15d57ea63b2f718fd99747fd22bf7421df3364c6 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Fri, 3 May 2024 16:42:50 -0700 Subject: [PATCH 20/24] Update __init__.py --- src/hdmf/common/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 7416fcf3d..4d724d1d1 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -139,7 +139,6 @@ def available_namespaces(): {'name': 'namespace', 'type': str, 'doc': 'the namespace the data_type is defined in'}, {'name': 'post_init_method', 'type': Callable, 'default': None, 'doc': 'The function used as a post_init method to validate the class generation.'}, - {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, is_method=False) def get_class(**kwargs): """Get the class object of the Container subclass corresponding to a given neurdata_type. From c40e36760fbeae10d09ac4ccc4448fe1cde9bcd8 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Fri, 3 May 2024 17:48:11 -0700 Subject: [PATCH 21/24] Update classgenerator.py --- src/hdmf/build/classgenerator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index 325462452..a3336b98e 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -468,7 +468,6 @@ def __init__(self, **kwargs): add_method(new_kwarg['value']) if self.post_init_method is not None: - self.post_init_method(**original_kwargs) # override __init__ From 383790b2ae0d1a0fa7f6aea1457f0ea72ad56b33 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Fri, 3 May 2024 17:50:10 -0700 Subject: [PATCH 22/24] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d5a2cc62..909ef5253 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Added `TypeConfigurator` to automatically wrap fields with `TermSetWrapper` according to a configuration file. @mavaylon1 [#1016](https://github.com/hdmf-dev/hdmf/pull/1016) - Updated `TermSetWrapper` to support validating a single field within a compound array. @mavaylon1 [#1061](https://github.com/hdmf-dev/hdmf/pull/1061) - Updated testing to not install in editable mode and not run `coverage` by default. @rly [#1107](https://github.com/hdmf-dev/hdmf/pull/1107) +- Add `post_init_method` parameter when generating classes to perform post-init functionality, i.e., validation. @mavaylon1 [#1089](https://github.com/hdmf-dev/hdmf/pull/1089) ## HDMF 3.13.0 (March 20, 2024) From 201b8c4f06a8c520cb16692d6789695a05a18d74 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Tue, 7 May 2024 04:44:20 -0700 Subject: [PATCH 23/24] VectorData Expand by Default via write_dataset (#1093) --- CHANGELOG.md | 1 + src/hdmf/backends/hdf5/h5tools.py | 34 +++++++++++++++++++++++------ tests/unit/test_io_hdf5.py | 2 +- tests/unit/test_io_hdf5_h5tools.py | 35 +++++++++++++++++++++++++----- 4 files changed, 59 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d5a2cc62..d44a1db06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Added `TypeConfigurator` to automatically wrap fields with `TermSetWrapper` according to a configuration file. @mavaylon1 [#1016](https://github.com/hdmf-dev/hdmf/pull/1016) - Updated `TermSetWrapper` to support validating a single field within a compound array. @mavaylon1 [#1061](https://github.com/hdmf-dev/hdmf/pull/1061) - Updated testing to not install in editable mode and not run `coverage` by default. @rly [#1107](https://github.com/hdmf-dev/hdmf/pull/1107) +- Updated the default behavior for writing HDF5 datasets to be expandandable datasets with chunking enabled by default. This does not override user set chunking parameters. @mavaylon1 [#1093](https://github.com/hdmf-dev/hdmf/pull/1093) ## HDMF 3.13.0 (March 20, 2024) diff --git a/src/hdmf/backends/hdf5/h5tools.py b/src/hdmf/backends/hdf5/h5tools.py index 05ce36e13..4444ec486 100644 --- a/src/hdmf/backends/hdf5/h5tools.py +++ b/src/hdmf/backends/hdf5/h5tools.py @@ -364,7 +364,10 @@ def copy_file(self, **kwargs): 'default': True}, {'name': 'herd', 'type': 'hdmf.common.resources.HERD', 'doc': 'A HERD object to populate with references.', - 'default': None}) + 'default': None}, + {'name': 'expandable', 'type': bool, 'default': True, + 'doc': ('Bool to set whether datasets are expandable by setting the max shape for all dimensions', + 'of a dataset to None and enabling auto-chunking by default.')}) def write(self, **kwargs): """Write the container to an HDF5 file.""" if self.__mode == 'r': @@ -804,10 +807,16 @@ def close_linked_files(self): 'doc': 'exhaust DataChunkIterators one at a time. If False, exhaust them concurrently', 'default': True}, {'name': 'export_source', 'type': str, - 'doc': 'The source of the builders when exporting', 'default': None}) + 'doc': 'The source of the builders when exporting', 'default': None}, + {'name': 'expandable', 'type': bool, 'default': True, + 'doc': ('Bool to set whether datasets are expandable by setting the max shape for all dimensions', + 'of a dataset to None and enabling auto-chunking by default.')}) def write_builder(self, **kwargs): f_builder = popargs('builder', kwargs) - link_data, exhaust_dci, export_source = getargs('link_data', 'exhaust_dci', 'export_source', kwargs) + link_data, exhaust_dci, export_source = getargs('link_data', + 'exhaust_dci', + 'export_source', + kwargs) self.logger.debug("Writing GroupBuilder '%s' to path '%s' with kwargs=%s" % (f_builder.name, self.source, kwargs)) for name, gbldr in f_builder.groups.items(): @@ -978,6 +987,9 @@ def _filler(): 'default': True}, {'name': 'export_source', 'type': str, 'doc': 'The source of the builders when exporting', 'default': None}, + {'name': 'expandable', 'type': bool, 'default': True, + 'doc': ('Bool to set whether datasets are expandable by setting the max shape for all dimensions', + 'of a dataset to None and enabling auto-chunking by default.')}, returns='the Group that was created', rtype=Group) def write_group(self, **kwargs): parent, builder = popargs('parent', 'builder', kwargs) @@ -1078,6 +1090,9 @@ def write_link(self, **kwargs): 'default': True}, {'name': 'export_source', 'type': str, 'doc': 'The source of the builders when exporting', 'default': None}, + {'name': 'expandable', 'type': bool, 'default': True, + 'doc': ('Bool to set whether datasets are expandable by setting the max shape for all dimensions', + 'of a dataset to None and enabling auto-chunking by default.')}, returns='the Dataset that was created', rtype=Dataset) def write_dataset(self, **kwargs): # noqa: C901 """ Write a dataset to HDF5 @@ -1085,7 +1100,7 @@ def write_dataset(self, **kwargs): # noqa: C901 The function uses other dataset-dependent write functions, e.g, ``__scalar_fill__``, ``__list_fill__``, and ``__setup_chunked_dset__`` to write the data. """ - parent, builder = popargs('parent', 'builder', kwargs) + parent, builder, expandable = popargs('parent', 'builder', 'expandable', kwargs) link_data, exhaust_dci, export_source = getargs('link_data', 'exhaust_dci', 'export_source', kwargs) self.logger.debug("Writing DatasetBuilder '%s' to parent group '%s'" % (builder.name, parent.name)) if self.get_written(builder): @@ -1102,6 +1117,7 @@ def write_dataset(self, **kwargs): # noqa: C901 data = data.data else: options['io_settings'] = {} + attributes = builder.attributes options['dtype'] = builder.dtype dset = None @@ -1206,7 +1222,7 @@ def _filler(): return # If the compound data type contains only regular data (i.e., no references) then we can write it as usual else: - dset = self.__list_fill__(parent, name, data, options) + dset = self.__list_fill__(parent, name, data, expandable, options) # Write a dataset containing references, i.e., a region or object reference. # NOTE: we can ignore options['io_settings'] for scalar data elif self.__is_ref(options['dtype']): @@ -1301,7 +1317,7 @@ def _filler(): self.__dci_queue.append(dataset=dset, data=data) # Write a regular in memory array (e.g., numpy array, list etc.) elif hasattr(data, '__len__'): - dset = self.__list_fill__(parent, name, data, options) + dset = self.__list_fill__(parent, name, data, expandable, options) # Write a regular scalar dataset else: dset = self.__scalar_fill__(parent, name, data, options) @@ -1429,7 +1445,7 @@ def __chunked_iter_fill__(cls, parent, name, data, options=None): return dset @classmethod - def __list_fill__(cls, parent, name, data, options=None): + def __list_fill__(cls, parent, name, data, expandable, options=None): # define the io settings and data type if necessary io_settings = {} dtype = None @@ -1451,6 +1467,10 @@ def __list_fill__(cls, parent, name, data, options=None): data_shape = (len(data),) else: data_shape = get_data_shape(data) + if expandable: + # Don't override existing settings + if 'maxshape' not in io_settings: + io_settings['maxshape'] = tuple([None]*len(data_shape)) # Create the dataset try: diff --git a/tests/unit/test_io_hdf5.py b/tests/unit/test_io_hdf5.py index 0dae1fbbe..b4bc72d8d 100644 --- a/tests/unit/test_io_hdf5.py +++ b/tests/unit/test_io_hdf5.py @@ -225,5 +225,5 @@ def test_dataset_shape(self): io.write_builder(self.builder) builder = io.read_builder() dset = builder['test_bucket']['foo_holder']['foo1']['my_data'].data - self.assertEqual(get_data_shape(dset), (10,)) + self.assertEqual(get_data_shape(dset), (None,)) io.close() diff --git a/tests/unit/test_io_hdf5_h5tools.py b/tests/unit/test_io_hdf5_h5tools.py index 5a4fd5a32..2efea40d6 100644 --- a/tests/unit/test_io_hdf5_h5tools.py +++ b/tests/unit/test_io_hdf5_h5tools.py @@ -28,6 +28,7 @@ from hdmf.testing import TestCase, remove_test_file from hdmf.common.resources import HERD from hdmf.term_set import TermSet, TermSetWrapper +from hdmf.utils import get_data_shape from tests.unit.helpers.utils import (Foo, FooBucket, FooFile, get_foo_buildmanager, @@ -163,6 +164,7 @@ def test_write_dataset_list(self): self.io.write_dataset(self.f, DatasetBuilder('test_dataset', a.tolist(), attributes={})) dset = self.f['test_dataset'] self.assertTrue(np.all(dset[:] == a)) + self.assertEqual(get_data_shape(dset), (None, None, None)) def test_write_dataset_list_compress_gzip(self): a = H5DataIO(np.arange(30).reshape(5, 2, 3), @@ -266,9 +268,9 @@ def test_write_dataset_list_fillvalue(self): self.assertEqual(dset.fillvalue, -1) ########################################## - # write_dataset tests: tables + # write_dataset tests: cmpd_dt ########################################## - def test_write_table(self): + def test_write_cmpd_dt(self): cmpd_dt = np.dtype([('a', np.int32), ('b', np.float64)]) data = np.zeros(10, dtype=cmpd_dt) data['a'][1] = 101 @@ -279,8 +281,9 @@ def test_write_table(self): dset = self.f['test_dataset'] self.assertEqual(dset['a'].tolist(), data['a'].tolist()) self.assertEqual(dset['b'].tolist(), data['b'].tolist()) + self.assertEqual(get_data_shape(dset), (None,)) - def test_write_table_nested(self): + def test_write_cmpd_dt_nested(self): b_cmpd_dt = np.dtype([('c', np.int32), ('d', np.float64)]) cmpd_dt = np.dtype([('a', np.int32), ('b', b_cmpd_dt)]) data = np.zeros(10, dtype=cmpd_dt) @@ -739,12 +742,12 @@ def test_copy_h5py_dataset_h5dataio_input(self): self.f['test_copy'][:].tolist()) def test_list_fill_empty(self): - dset = self.io.__list_fill__(self.f, 'empty_dataset', [], options={'dtype': int, 'io_settings': {}}) + dset = self.io.__list_fill__(self.f, 'empty_dataset', [], True, options={'dtype': int, 'io_settings': {}}) self.assertTupleEqual(dset.shape, (0,)) def test_list_fill_empty_no_dtype(self): with self.assertRaisesRegex(Exception, r"cannot add \S+ to [/\S]+ - could not determine type"): - self.io.__list_fill__(self.f, 'empty_dataset', []) + self.io.__list_fill__(self.f, 'empty_dataset', [], True) def test_read_str(self): a = ['a', 'bb', 'ccc', 'dddd', 'e'] @@ -763,6 +766,28 @@ def test_read_str(self): '') +class TestExpand(TestCase): + def setUp(self): + self.manager = get_foo_buildmanager() + self.path = get_temp_filepath() + + def test_expand_false(self): + # Setup all the data we need + foo1 = Foo('foo1', [1, 2, 3, 4, 5], "I am foo1", 17, 3.14) + foobucket = FooBucket('bucket1', [foo1]) + foofile = FooFile(buckets=[foobucket]) + + with HDF5IO(self.path, manager=self.manager, mode='w') as io: + io.write(foofile, expandable=False) + + io = HDF5IO(self.path, manager=self.manager, mode='r') + read_foofile = io.read() + self.assertListEqual(foofile.buckets['bucket1'].foos['foo1'].my_data, + read_foofile.buckets['bucket1'].foos['foo1'].my_data[:].tolist()) + self.assertEqual(get_data_shape(read_foofile.buckets['bucket1'].foos['foo1'].my_data), + (5,)) + + class TestRoundTrip(TestCase): def setUp(self): From 898832b6f4259570fce42a93d93732b14872fca9 Mon Sep 17 00:00:00 2001 From: rly Date: Tue, 14 May 2024 14:12:51 -0700 Subject: [PATCH 24/24] Revert "VectorData Expand by Default via write_dataset (#1093)" This reverts commit 201b8c4f06a8c520cb16692d6789695a05a18d74. --- CHANGELOG.md | 1 - src/hdmf/backends/hdf5/h5tools.py | 34 ++++++----------------------- tests/unit/test_io_hdf5.py | 2 +- tests/unit/test_io_hdf5_h5tools.py | 35 +++++------------------------- 4 files changed, 13 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4390eb8e..909ef5253 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,6 @@ - Updated `TermSetWrapper` to support validating a single field within a compound array. @mavaylon1 [#1061](https://github.com/hdmf-dev/hdmf/pull/1061) - Updated testing to not install in editable mode and not run `coverage` by default. @rly [#1107](https://github.com/hdmf-dev/hdmf/pull/1107) - Add `post_init_method` parameter when generating classes to perform post-init functionality, i.e., validation. @mavaylon1 [#1089](https://github.com/hdmf-dev/hdmf/pull/1089) -- Updated the default behavior for writing HDF5 datasets to be expandandable datasets with chunking enabled by default. This does not override user set chunking parameters. @mavaylon1 [#1093](https://github.com/hdmf-dev/hdmf/pull/1093) ## HDMF 3.13.0 (March 20, 2024) diff --git a/src/hdmf/backends/hdf5/h5tools.py b/src/hdmf/backends/hdf5/h5tools.py index 4444ec486..05ce36e13 100644 --- a/src/hdmf/backends/hdf5/h5tools.py +++ b/src/hdmf/backends/hdf5/h5tools.py @@ -364,10 +364,7 @@ def copy_file(self, **kwargs): 'default': True}, {'name': 'herd', 'type': 'hdmf.common.resources.HERD', 'doc': 'A HERD object to populate with references.', - 'default': None}, - {'name': 'expandable', 'type': bool, 'default': True, - 'doc': ('Bool to set whether datasets are expandable by setting the max shape for all dimensions', - 'of a dataset to None and enabling auto-chunking by default.')}) + 'default': None}) def write(self, **kwargs): """Write the container to an HDF5 file.""" if self.__mode == 'r': @@ -807,16 +804,10 @@ def close_linked_files(self): 'doc': 'exhaust DataChunkIterators one at a time. If False, exhaust them concurrently', 'default': True}, {'name': 'export_source', 'type': str, - 'doc': 'The source of the builders when exporting', 'default': None}, - {'name': 'expandable', 'type': bool, 'default': True, - 'doc': ('Bool to set whether datasets are expandable by setting the max shape for all dimensions', - 'of a dataset to None and enabling auto-chunking by default.')}) + 'doc': 'The source of the builders when exporting', 'default': None}) def write_builder(self, **kwargs): f_builder = popargs('builder', kwargs) - link_data, exhaust_dci, export_source = getargs('link_data', - 'exhaust_dci', - 'export_source', - kwargs) + link_data, exhaust_dci, export_source = getargs('link_data', 'exhaust_dci', 'export_source', kwargs) self.logger.debug("Writing GroupBuilder '%s' to path '%s' with kwargs=%s" % (f_builder.name, self.source, kwargs)) for name, gbldr in f_builder.groups.items(): @@ -987,9 +978,6 @@ def _filler(): 'default': True}, {'name': 'export_source', 'type': str, 'doc': 'The source of the builders when exporting', 'default': None}, - {'name': 'expandable', 'type': bool, 'default': True, - 'doc': ('Bool to set whether datasets are expandable by setting the max shape for all dimensions', - 'of a dataset to None and enabling auto-chunking by default.')}, returns='the Group that was created', rtype=Group) def write_group(self, **kwargs): parent, builder = popargs('parent', 'builder', kwargs) @@ -1090,9 +1078,6 @@ def write_link(self, **kwargs): 'default': True}, {'name': 'export_source', 'type': str, 'doc': 'The source of the builders when exporting', 'default': None}, - {'name': 'expandable', 'type': bool, 'default': True, - 'doc': ('Bool to set whether datasets are expandable by setting the max shape for all dimensions', - 'of a dataset to None and enabling auto-chunking by default.')}, returns='the Dataset that was created', rtype=Dataset) def write_dataset(self, **kwargs): # noqa: C901 """ Write a dataset to HDF5 @@ -1100,7 +1085,7 @@ def write_dataset(self, **kwargs): # noqa: C901 The function uses other dataset-dependent write functions, e.g, ``__scalar_fill__``, ``__list_fill__``, and ``__setup_chunked_dset__`` to write the data. """ - parent, builder, expandable = popargs('parent', 'builder', 'expandable', kwargs) + parent, builder = popargs('parent', 'builder', kwargs) link_data, exhaust_dci, export_source = getargs('link_data', 'exhaust_dci', 'export_source', kwargs) self.logger.debug("Writing DatasetBuilder '%s' to parent group '%s'" % (builder.name, parent.name)) if self.get_written(builder): @@ -1117,7 +1102,6 @@ def write_dataset(self, **kwargs): # noqa: C901 data = data.data else: options['io_settings'] = {} - attributes = builder.attributes options['dtype'] = builder.dtype dset = None @@ -1222,7 +1206,7 @@ def _filler(): return # If the compound data type contains only regular data (i.e., no references) then we can write it as usual else: - dset = self.__list_fill__(parent, name, data, expandable, options) + dset = self.__list_fill__(parent, name, data, options) # Write a dataset containing references, i.e., a region or object reference. # NOTE: we can ignore options['io_settings'] for scalar data elif self.__is_ref(options['dtype']): @@ -1317,7 +1301,7 @@ def _filler(): self.__dci_queue.append(dataset=dset, data=data) # Write a regular in memory array (e.g., numpy array, list etc.) elif hasattr(data, '__len__'): - dset = self.__list_fill__(parent, name, data, expandable, options) + dset = self.__list_fill__(parent, name, data, options) # Write a regular scalar dataset else: dset = self.__scalar_fill__(parent, name, data, options) @@ -1445,7 +1429,7 @@ def __chunked_iter_fill__(cls, parent, name, data, options=None): return dset @classmethod - def __list_fill__(cls, parent, name, data, expandable, options=None): + def __list_fill__(cls, parent, name, data, options=None): # define the io settings and data type if necessary io_settings = {} dtype = None @@ -1467,10 +1451,6 @@ def __list_fill__(cls, parent, name, data, expandable, options=None): data_shape = (len(data),) else: data_shape = get_data_shape(data) - if expandable: - # Don't override existing settings - if 'maxshape' not in io_settings: - io_settings['maxshape'] = tuple([None]*len(data_shape)) # Create the dataset try: diff --git a/tests/unit/test_io_hdf5.py b/tests/unit/test_io_hdf5.py index b4bc72d8d..0dae1fbbe 100644 --- a/tests/unit/test_io_hdf5.py +++ b/tests/unit/test_io_hdf5.py @@ -225,5 +225,5 @@ def test_dataset_shape(self): io.write_builder(self.builder) builder = io.read_builder() dset = builder['test_bucket']['foo_holder']['foo1']['my_data'].data - self.assertEqual(get_data_shape(dset), (None,)) + self.assertEqual(get_data_shape(dset), (10,)) io.close() diff --git a/tests/unit/test_io_hdf5_h5tools.py b/tests/unit/test_io_hdf5_h5tools.py index 2efea40d6..5a4fd5a32 100644 --- a/tests/unit/test_io_hdf5_h5tools.py +++ b/tests/unit/test_io_hdf5_h5tools.py @@ -28,7 +28,6 @@ from hdmf.testing import TestCase, remove_test_file from hdmf.common.resources import HERD from hdmf.term_set import TermSet, TermSetWrapper -from hdmf.utils import get_data_shape from tests.unit.helpers.utils import (Foo, FooBucket, FooFile, get_foo_buildmanager, @@ -164,7 +163,6 @@ def test_write_dataset_list(self): self.io.write_dataset(self.f, DatasetBuilder('test_dataset', a.tolist(), attributes={})) dset = self.f['test_dataset'] self.assertTrue(np.all(dset[:] == a)) - self.assertEqual(get_data_shape(dset), (None, None, None)) def test_write_dataset_list_compress_gzip(self): a = H5DataIO(np.arange(30).reshape(5, 2, 3), @@ -268,9 +266,9 @@ def test_write_dataset_list_fillvalue(self): self.assertEqual(dset.fillvalue, -1) ########################################## - # write_dataset tests: cmpd_dt + # write_dataset tests: tables ########################################## - def test_write_cmpd_dt(self): + def test_write_table(self): cmpd_dt = np.dtype([('a', np.int32), ('b', np.float64)]) data = np.zeros(10, dtype=cmpd_dt) data['a'][1] = 101 @@ -281,9 +279,8 @@ def test_write_cmpd_dt(self): dset = self.f['test_dataset'] self.assertEqual(dset['a'].tolist(), data['a'].tolist()) self.assertEqual(dset['b'].tolist(), data['b'].tolist()) - self.assertEqual(get_data_shape(dset), (None,)) - def test_write_cmpd_dt_nested(self): + def test_write_table_nested(self): b_cmpd_dt = np.dtype([('c', np.int32), ('d', np.float64)]) cmpd_dt = np.dtype([('a', np.int32), ('b', b_cmpd_dt)]) data = np.zeros(10, dtype=cmpd_dt) @@ -742,12 +739,12 @@ def test_copy_h5py_dataset_h5dataio_input(self): self.f['test_copy'][:].tolist()) def test_list_fill_empty(self): - dset = self.io.__list_fill__(self.f, 'empty_dataset', [], True, options={'dtype': int, 'io_settings': {}}) + dset = self.io.__list_fill__(self.f, 'empty_dataset', [], options={'dtype': int, 'io_settings': {}}) self.assertTupleEqual(dset.shape, (0,)) def test_list_fill_empty_no_dtype(self): with self.assertRaisesRegex(Exception, r"cannot add \S+ to [/\S]+ - could not determine type"): - self.io.__list_fill__(self.f, 'empty_dataset', [], True) + self.io.__list_fill__(self.f, 'empty_dataset', []) def test_read_str(self): a = ['a', 'bb', 'ccc', 'dddd', 'e'] @@ -766,28 +763,6 @@ def test_read_str(self): '') -class TestExpand(TestCase): - def setUp(self): - self.manager = get_foo_buildmanager() - self.path = get_temp_filepath() - - def test_expand_false(self): - # Setup all the data we need - foo1 = Foo('foo1', [1, 2, 3, 4, 5], "I am foo1", 17, 3.14) - foobucket = FooBucket('bucket1', [foo1]) - foofile = FooFile(buckets=[foobucket]) - - with HDF5IO(self.path, manager=self.manager, mode='w') as io: - io.write(foofile, expandable=False) - - io = HDF5IO(self.path, manager=self.manager, mode='r') - read_foofile = io.read() - self.assertListEqual(foofile.buckets['bucket1'].foos['foo1'].my_data, - read_foofile.buckets['bucket1'].foos['foo1'].my_data[:].tolist()) - self.assertEqual(get_data_shape(read_foofile.buckets['bucket1'].foos['foo1'].my_data), - (5,)) - - class TestRoundTrip(TestCase): def setUp(self):