Skip to content

Commit

Permalink
feat: add data config checker for property value types
Browse files Browse the repository at this point in the history
  • Loading branch information
@joaoandre
joaoandre committed Dec 13, 2024
1 parent 43c3303 commit c82433d
Show file tree
Hide file tree
Showing 2 changed files with 113 additions and 40 deletions.
59 changes: 19 additions & 40 deletions taipy/core/config/checkers/_data_node_config_checker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,26 +23,6 @@


class _DataNodeConfigChecker(_ConfigChecker):
_PROPERTIES_TYPES: Dict[str, List[Tuple[Any, List[str]]]] = {
DataNodeConfig._STORAGE_TYPE_VALUE_GENERIC: [
(
Callable,
[
DataNodeConfig._OPTIONAL_READ_FUNCTION_GENERIC_PROPERTY,
DataNodeConfig._OPTIONAL_WRITE_FUNCTION_GENERIC_PROPERTY,
],
)
],
DataNodeConfig._STORAGE_TYPE_VALUE_SQL: [
(
Callable,
[
DataNodeConfig._REQUIRED_WRITE_QUERY_BUILDER_SQL_PROPERTY,
DataNodeConfig._OPTIONAL_APPEND_QUERY_BUILDER_SQL_PROPERTY,
],
),
],
}

def __init__(self, config: _Config, collector: IssueCollector):
super().__init__(config, collector)
Expand All @@ -67,7 +47,7 @@ def _check(self) -> IssueCollector:
self._check_scope(data_node_config_id, data_node_config)
self._check_validity_period(data_node_config_id, data_node_config)
self._check_required_properties(data_node_config_id, data_node_config)
self._check_class_type(data_node_config_id, data_node_config)
self._check_property_types(data_node_config_id, data_node_config)
self._check_generic_read_write_fct_and_args(data_node_config_id, data_node_config)
self._check_exposed_type(data_node_config_id, data_node_config)
return self._collector
Expand Down Expand Up @@ -217,25 +197,24 @@ def _check_generic_read_write_fct_and_args(self, data_node_config_id: str, data_
f"DataNodeConfig `{data_node_config_id}` must be populated with a Callable function.",
)

def _check_class_type(self, data_node_config_id: str, data_node_config: DataNodeConfig):
if data_node_config.storage_type in self._PROPERTIES_TYPES.keys():
for class_type, prop_keys in self._PROPERTIES_TYPES[data_node_config.storage_type]:
for prop_key in prop_keys:
prop_value = data_node_config.properties.get(prop_key) if data_node_config.properties else None
if prop_value and not isinstance(prop_value, class_type):
self._error(
prop_key,
prop_value,
f"`{prop_key}` of DataNodeConfig `{data_node_config_id}` must be"
f" populated with a {'Callable' if class_type == Callable else class_type.__name__}.",
)
if class_type == Callable and callable(prop_value) and prop_value.__name__ == "<lambda>":
self._error(
prop_key,
prop_value,
f"`{prop_key}` of DataNodeConfig `{data_node_config_id}` must be"
f" populated with a serializable Callable function but not a lambda.",
)
def _check_property_types(self, data_node_config_id: str, data_node_config: DataNodeConfig):
if property_types := data_node_config._PROPERTIES_TYPES.get(data_node_config.storage_type):
for prop_key, prop_type in property_types.items():
prop_value = data_node_config.properties.get(prop_key) if data_node_config.properties else None
if prop_value and not isinstance(prop_value, prop_type):
self._error(
prop_key,
prop_value,
f"`{prop_key}` of DataNodeConfig `{data_node_config_id}` must be"
f" populated with a {prop_type}.",
)
if prop_type == Callable and callable(prop_value) and prop_value.__name__ == "<lambda>":
self._error(
prop_key,
prop_value,
f"`{prop_key}` of DataNodeConfig `{data_node_config_id}` must be"
f" populated with a serializable Callable function but not a lambda.",
)

def _check_exposed_type(self, data_node_config_id: str, data_node_config: DataNodeConfig):
if not isinstance(data_node_config.exposed_type, str):
Expand Down
94 changes: 94 additions & 0 deletions taipy/core/config/data_node_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,100 @@ class DataNodeConfig(Section):
_OPTIONAL_AWS_S3_GET_OBJECT_PARAMETERS_PROPERTY = "aws_s3_get_object_parameters"
_OPTIONAL_AWS_S3_PUT_OBJECT_PARAMETERS_PROPERTY = "aws_s3_put_object_parameters"

_PROPERTIES_TYPES: Dict[str, Dict[str, Any]] = {
_STORAGE_TYPE_VALUE_GENERIC: {
_OPTIONAL_READ_FUNCTION_GENERIC_PROPERTY: Callable,
_OPTIONAL_WRITE_FUNCTION_GENERIC_PROPERTY: Callable,
_OPTIONAL_READ_FUNCTION_ARGS_GENERIC_PROPERTY: list,
_OPTIONAL_WRITE_FUNCTION_ARGS_GENERIC_PROPERTY: list,
},
_STORAGE_TYPE_VALUE_SQL: {
_REQUIRED_DB_NAME_SQL_PROPERTY: str,
_REQUIRED_DB_ENGINE_SQL_PROPERTY: str,
_REQUIRED_READ_QUERY_SQL_PROPERTY: str,
_REQUIRED_WRITE_QUERY_BUILDER_SQL_PROPERTY: Callable,
_OPTIONAL_APPEND_QUERY_BUILDER_SQL_PROPERTY: Callable,
_OPTIONAL_DB_USERNAME_SQL_PROPERTY: str,
_OPTIONAL_DB_PASSWORD_SQL_PROPERTY: str,
_OPTIONAL_HOST_SQL_PROPERTY: str,
_OPTIONAL_PORT_SQL_PROPERTY: int,
_OPTIONAL_DRIVER_SQL_PROPERTY: str,
_OPTIONAL_FOLDER_PATH_SQLITE_PROPERTY: str,
_OPTIONAL_FILE_EXTENSION_SQLITE_PROPERTY: str,
_OPTIONAL_DB_EXTRA_ARGS_SQL_PROPERTY: dict,
_OPTIONAL_EXPOSED_TYPE_SQL_PROPERTY: str,
},
_STORAGE_TYPE_VALUE_SQL_TABLE: {
_REQUIRED_DB_NAME_SQL_PROPERTY: str,
_REQUIRED_DB_ENGINE_SQL_PROPERTY: str,
_REQUIRED_TABLE_NAME_SQL_TABLE_PROPERTY: str,
_OPTIONAL_DB_USERNAME_SQL_PROPERTY: str,
_OPTIONAL_DB_PASSWORD_SQL_PROPERTY: str,
_OPTIONAL_HOST_SQL_PROPERTY: str,
_OPTIONAL_PORT_SQL_PROPERTY: int,
_OPTIONAL_DRIVER_SQL_PROPERTY: str,
_OPTIONAL_FOLDER_PATH_SQLITE_PROPERTY: str,
_OPTIONAL_FILE_EXTENSION_SQLITE_PROPERTY: str,
_OPTIONAL_DB_EXTRA_ARGS_SQL_PROPERTY: dict,
_OPTIONAL_EXPOSED_TYPE_SQL_PROPERTY: str,
},
_STORAGE_TYPE_VALUE_CSV: {
_OPTIONAL_DEFAULT_PATH_CSV_PROPERTY: str,
_OPTIONAL_ENCODING_PROPERTY: str,
_OPTIONAL_HAS_HEADER_CSV_PROPERTY: bool,
_OPTIONAL_EXPOSED_TYPE_CSV_PROPERTY: str,
},
_STORAGE_TYPE_VALUE_EXCEL: {
_OPTIONAL_DEFAULT_PATH_EXCEL_PROPERTY: str,
_OPTIONAL_HAS_HEADER_EXCEL_PROPERTY: bool,
_OPTIONAL_SHEET_NAME_EXCEL_PROPERTY: Union[str, List[str]],
_OPTIONAL_EXPOSED_TYPE_EXCEL_PROPERTY: str,
},
_STORAGE_TYPE_VALUE_IN_MEMORY: {
_OPTIONAL_DEFAULT_DATA_IN_MEMORY_PROPERTY: Any,
},
_STORAGE_TYPE_VALUE_PICKLE: {
_OPTIONAL_DEFAULT_PATH_PICKLE_PROPERTY: str,
_OPTIONAL_DEFAULT_DATA_PICKLE_PROPERTY: Any,
},
_STORAGE_TYPE_VALUE_JSON: {
_OPTIONAL_DEFAULT_PATH_JSON_PROPERTY: str,
_OPTIONAL_ENCODING_PROPERTY: str,
_OPTIONAL_ENCODER_JSON_PROPERTY: json.JSONEncoder,
_OPTIONAL_DECODER_JSON_PROPERTY: json.JSONDecoder,
},
_STORAGE_TYPE_VALUE_PARQUET: {
_OPTIONAL_DEFAULT_PATH_PARQUET_PROPERTY: str,
_OPTIONAL_ENGINE_PARQUET_PROPERTY: str,
_OPTIONAL_COMPRESSION_PARQUET_PROPERTY: str,
_OPTIONAL_READ_KWARGS_PARQUET_PROPERTY: dict,
_OPTIONAL_WRITE_KWARGS_PARQUET_PROPERTY: dict,
_OPTIONAL_EXPOSED_TYPE_PARQUET_PROPERTY: str,
},
_STORAGE_TYPE_VALUE_MONGO_COLLECTION: {
_REQUIRED_DB_NAME_MONGO_PROPERTY: str,
_REQUIRED_COLLECTION_NAME_MONGO_PROPERTY: str,
_OPTIONAL_CUSTOM_DOCUMENT_MONGO_PROPERTY: Any,
_OPTIONAL_USERNAME_MONGO_PROPERTY: str,
_OPTIONAL_PASSWORD_MONGO_PROPERTY: str,
_OPTIONAL_HOST_MONGO_PROPERTY: str,
_OPTIONAL_PORT_MONGO_PROPERTY: int,
_OPTIONAL_DRIVER_MONGO_PROPERTY: str,
_OPTIONAL_DB_EXTRA_ARGS_MONGO_PROPERTY: dict,
},
_STORAGE_TYPE_VALUE_S3_OBJECT: {
_REQUIRED_AWS_ACCESS_KEY_ID_PROPERTY: str,
_REQUIRED_AWS_SECRET_ACCESS_KEY_PROPERTY: str,
_REQUIRED_AWS_STORAGE_BUCKET_NAME_PROPERTY: str,
_REQUIRED_AWS_S3_OBJECT_KEY_PROPERTY: str,
_OPTIONAL_AWS_REGION_PROPERTY: str,
_OPTIONAL_AWS_S3_CLIENT_PARAMETERS_PROPERTY: dict,
_OPTIONAL_AWS_S3_GET_OBJECT_PARAMETERS_PROPERTY: dict,
_OPTIONAL_AWS_S3_PUT_OBJECT_PARAMETERS_PROPERTY: dict,
},
}


_REQUIRED_PROPERTIES: Dict[str, List] = {
_STORAGE_TYPE_VALUE_PICKLE: [],
_STORAGE_TYPE_VALUE_SQL_TABLE: [
Expand Down

0 comments on commit c82433d

Please sign in to comment.