Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use ParallelQueryMappedValidator from pvframework #5

Merged
merged 2 commits into from
Sep 24, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 10 additions & 41 deletions src/pvtool/customer_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"""
import re
from datetime import date, datetime
from typing import Any, Generator, Iterator, Optional, TypeAlias
from typing import Any, Generator, Optional, TypeAlias

from bo4e.com.adresse import Adresse
from bo4e.com.externereferenz import ExterneReferenz
Expand All @@ -16,9 +16,15 @@
from ibims.datasets import TripicaCustomerLoaderDataSet
from injector import Module, provider
from more_itertools import first_true
from pvframework import PathMappedValidator, Query, QueryMappedValidator, ValidationManager, Validator
from pvframework.mapped_validators.query_map import QueryIterable
from pvframework.types import DataSetT, SyncValidatorFunction, ValidatorFunctionT
from pvframework import (
ParallelQueryMappedValidator,
PathMappedValidator,
Query,
QueryMappedValidator,
ValidationManager,
Validator,
)
from pvframework.types import SyncValidatorFunction
from pvframework.utils import param, required_field
from pytz import timezone
from schwifty import BIC, IBAN
Expand Down Expand Up @@ -326,43 +332,6 @@ def iter_vertragskonten(vertragskonten: list[Vertragskonto]) -> Generator[tuple[
return ((vertragskonto, f"[ouid={vertragskonto.ouid}]") for vertragskonto in vertragskonten)


class ParallelQueryMappedValidator(QueryMappedValidator[DataSetT, ValidatorFunctionT]):
"""
This mapped validator class supplies the parameter combinations not as cartesian product but as parallel.
I.e. each iterator must yield the same number of values or have to be scalar.
=> For every parameter pair p_i, p_j: len(p_i) == len(p_j) or len(p_i) == 1 or len(p_j) == 1
"""

def param_sets(self, param_iterables: dict[str, QueryIterable]) -> Iterator[dict[str, Any] | Exception]:
"""
This method is a generator which yields a dict of parameter sets. Each parameter set is a dict of parameter
names and values. The parameter sets are yielded in parallel. It behaves similar to zip.
"""
for param_iterable in param_iterables.values():
param_iterable.include_exceptions = True
param_sets = {param_name: list(param_iterable) for param_name, param_iterable in param_iterables.items()}
lengths = [len(param_set) for param_set in param_sets.values()]
iter_count = 1
for length in lengths:
if 1 < iter_count != length:
raise ValueError("All parameter sets must have the same length or be scalar")
if length != iter_count:
iter_count = length
for i in range(iter_count):
param_set: Optional[dict[str, Any]] = {
param_name: param_values[0] if len(param_values) == 1 else param_values[i]
for param_name, param_values in param_sets.items()
}
assert param_set is not None
for param_name, param_value in param_set.items():
if param_name in self.validator.required_param_names and isinstance(param_value, Exception):
yield param_value
param_set = None
break
if param_set is not None:
yield param_set


class ValidationManagerProviderCustomer(Module):
"""
This module provides a ValidationManager for customer loader with an injected MigrationConfig
Expand Down
Loading