[WIP] Upgrade to Pydantic v2

setup.cfg

@@ -30,7 +30,8 @@ install_requires =
     ; By default, we only consider core dependencies required to use Ralph as a
     ; library (mostly models).
     langcodes>=3.2.0
-    pydantic[dotenv,email]>=1.10.0, <2.0
+    pydantic[email]>=2.1.1, <3.0
+    pydantic-settings>=2.0.2
     rfc3987>=1.3.0
 package_dir =
     =src

src/ralph/api/auth/oidc.py

@@ -9,7 +9,7 @@
 from fastapi.security import OpenIdConnect
 from jose import ExpiredSignatureError, JWTError, jwt
 from jose.exceptions import JWTClaimsError
-from pydantic import AnyUrl, BaseModel, Extra
+from pydantic import ConfigDict, AnyUrl, BaseModel
 
 from ralph.api.auth.user import AuthenticatedUser
 from ralph.conf import settings
@@ -43,13 +43,11 @@ class IDToken(BaseModel):
 
     iss: str
     sub: str
-    aud: Optional[str]
+    aud: Optional[str] = None
     exp: int
     iat: int
-    scope: Optional[str]
-
-    class Config:  # pylint: disable=missing-class-docstring # noqa: D106
-        extra = Extra.ignore
+    scope: Optional[str] = None
+    model_config = ConfigDict(extra="ignore")
 
 
 @lru_cache()

src/ralph/api/models.py

@@ -6,7 +6,7 @@
 from typing import Optional, Union
 from uuid import UUID
 
-from pydantic import AnyUrl, BaseModel, Extra
+from pydantic import ConfigDict, AnyUrl, BaseModel
 
 from ..models.xapi.base.agents import BaseXapiAgent
 from ..models.xapi.base.groups import BaseXapiGroup
@@ -28,14 +28,7 @@ class BaseModelWithLaxConfig(BaseModel):
     Common base lax model to perform light input validation as
     we receive statements through the API.
     """
-
-    class Config:
-        """Enable extra properties.
-
-        Useful for not having to perform comprehensive validation.
-        """
-
-        extra = Extra.allow
+    model_config = ConfigDict(extra="allow")
 
 
 class LaxObjectField(BaseModelWithLaxConfig):
@@ -64,6 +57,6 @@ class LaxStatement(BaseModelWithLaxConfig):
     """
 
     actor: Union[BaseXapiAgent, BaseXapiGroup]
-    id: Optional[UUID]
+    id: Optional[UUID] = None
     object: LaxObjectField
     verb: LaxVerbField

src/ralph/backends/database/base.py

@@ -9,7 +9,7 @@
 from typing import BinaryIO, List, Literal, Optional, TextIO, Union
 from uuid import UUID
 
-from pydantic import BaseModel
+from pydantic import ConfigDict, BaseModel
 
 from ralph.exceptions import BackendParameterException
 
@@ -18,11 +18,7 @@
 
 class BaseQuery(BaseModel):
     """Base query model."""
-
-    class Config:
-        """Base query model configuration."""
-
-        extra = "forbid"
+    model_config = ConfigDict(extra="forbid")
 
 
 @dataclass

src/ralph/backends/database/clickhouse.py

@@ -36,8 +36,8 @@ class ClickHouseInsert(BaseModel):
 class ClickHouseQuery(BaseQuery):
     """ClickHouse query model."""
 
-    where_clause: Optional[str]
-    return_fields: Optional[List[str]]
+    where_clause: Optional[str] = None
+    return_fields: Optional[List[str]] = None
 
 
 class ClickHouseDatabase(BaseDatabase):  # pylint: disable=too-many-instance-attributes

src/ralph/backends/database/es.py

@@ -40,7 +40,7 @@ class OpType(Enum):
 class ESQuery(BaseQuery):
     """Elasticsearch body query model."""
 
-    query: Optional[dict]
+    query: Optional[dict] = None
 
 
 class ESDatabase(BaseDatabase):

src/ralph/backends/database/mongo.py

@@ -31,8 +31,8 @@
 class MongoQuery(BaseQuery):
     """Mongo query model."""
 
-    filter: Optional[dict]
-    projection: Optional[dict]
+    filter: Optional[dict] = None
+    projection: Optional[dict] = None
 
 
 class MongoDatabase(BaseDatabase):

src/ralph/backends/http/async_lrs.py

@@ -31,13 +31,13 @@ class StatementResponse(BaseModel):
     """Pydantic model for `get` statements response."""
 
     statements: Union[List[dict], dict]
-    more: Optional[str]
+    more: Optional[str] = None
 
 
 class LRSQuery(BaseQuery):
     """LRS body query model."""
 
-    query: Optional[dict]
+    query: Optional[dict] = None
 
 
 class AsyncLRSHTTP(BaseHTTP):

src/ralph/backends/http/base.py

@@ -6,7 +6,7 @@
 from enum import Enum, unique
 from typing import Iterator, List, Optional, Union
 
-from pydantic import BaseModel, ValidationError
+from pydantic import ConfigDict, BaseModel, ValidationError
 
 from ralph.exceptions import BackendParameterException
 
@@ -56,13 +56,9 @@ def wrapper(*args, **kwargs):
 
 class BaseQuery(BaseModel):
     """Base query model."""
+    model_config = ConfigDict(extra="forbid")
 
-    class Config:
-        """Base query model configuration."""
-
-        extra = "forbid"
-
-    query_string: Optional[str]
+    query_string: Optional[str] = None # TODO: validate that this is the behavior we want
 
 
 class BaseHTTP(ABC):

src/ralph/conf.py

@@ -3,12 +3,12 @@
 import io
 from enum import Enum
 from pathlib import Path
-from typing import List, Tuple, Union
+from typing import Any, List, Tuple, Union
 
 try:
-    from typing import Literal
+    from typing import Annotated, Literal, Optional
 except ImportError:
-    from typing_extensions import Literal
+    from typing_extensions import Annotated, Literal, Optional
 
 try:
     from click import get_app_dir
@@ -19,26 +19,38 @@
     from unittest.mock import Mock
 
     get_app_dir = Mock(return_value=".")
-from pydantic import AnyHttpUrl, AnyUrl, BaseModel, BaseSettings, Extra, Field
+from pydantic import BaseModel, ConfigDict, AnyHttpUrl, AnyUrl, BaseModel, Field, GetCoreSchemaHandler, model_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from pydantic.functional_validators import AfterValidator
 
 from .utils import import_string
 
+
 MODEL_PATH_SEPARATOR = "__"
 
 
-class BaseSettingsConfig:
-    """Pydantic model for BaseSettings Configuration."""
+# class BaseSettingsConfig(SettingsConfigDict):
+#     """Pydantic model for BaseSettings Configuration."""
+
+#     case_sensitive = True
+#     env_nested_delimiter = "__"
+#     env_prefix = "RALPH_"
 
-    case_sensitive = True
-    env_nested_delimiter = "__"
-    env_prefix = "RALPH_"
+BASE_SETTINGS_CONFIG = SettingsConfigDict(    
+    case_sensitive = True,
+    env_nested_delimiter = "__",
+    env_prefix = "RALPH_",
+)
 
 
 class CoreSettings(BaseSettings):
-    """Pydantice model for Ralph's core settings."""
+    """Pydantic model for Ralph's core settings."""
 
-    class Config(BaseSettingsConfig):
-        """Pydantic Configuration."""
+    # TODO[pydantic]: The `Config` class inherits from another class, please create the `model_config` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
+    # class Config(BaseSettingsConfig):
+    #     """Pydantic Configuration."""
+    model_config = BASE_SETTINGS_CONFIG
 
     APP_DIR: Path = get_app_dir("ralph")
     LOCALE_ENCODING: str = getattr(io, "LOCALE_ENCODING", "utf8")
@@ -47,29 +59,26 @@ class Config(BaseSettingsConfig):
 core_settings = CoreSettings()
 
 
-class CommaSeparatedTuple(str):
-    """Pydantic field type validating comma separated strings or tuples."""
+def validate_comma_separated_tuple(value: Union[str, Tuple[str, ...]]) -> Tuple[str]:
+    """Checks whether the value is a comma separated string or a tuple."""
 
-    @classmethod
-    def __get_validators__(cls):  # noqa: D105
-        def validate(value: Union[str, Tuple[str]]) -> Tuple[str]:
-            """Checks whether the value is a comma separated string or a tuple."""
-            if isinstance(value, tuple):
-                return value
+    if isinstance(value, tuple):
+        return value
 
-            if isinstance(value, str):
-                return tuple(value.split(","))
+    if isinstance(value, str):
+        return tuple(value.split(","))
 
-            raise TypeError("Invalid comma separated list")
+    raise TypeError("Invalid comma separated list")
 
-        yield validate
+CommaSeparatedTuple = Annotated[Union[str, Tuple[str, ...]], AfterValidator(validate_comma_separated_tuple)]
 
+
 
 class InstantiableSettingsItem(BaseModel):
     """Pydantic model for a settings configuration item that can be instantiated."""
-
-    class Config:  # pylint: disable=missing-class-docstring # noqa: D106
-        underscore_attrs_are_private = True
+    # TODO[pydantic]: The following keys were removed: `underscore_attrs_are_private`.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
+    model_config = SettingsConfigDict(underscore_attrs_are_private=True)
 
     _class_path: str = None
 
@@ -83,9 +92,7 @@ def get_instance(self, **init_parameters):
 
 class ClientOptions(BaseModel):
     """Pydantic model for additionnal client options."""
-
-    class Config:  # pylint: disable=missing-class-docstring # noqa: D106
-        extra = Extra.forbid
+    model_config = ConfigDict(extra="forbid")
 
 
 class ClickhouseClientOptions(ClientOptions):
@@ -124,11 +131,11 @@ class MongoClientOptions(ClientOptions):
 
 
 class ESDatabaseBackendSettings(InstantiableSettingsItem):
-    """Pydantic modelf for Elasticsearch database backend configuration settings."""
+    """Pydantic modelf for Elasticsearch database backend configuration settings."""    
 
     _class_path: str = "ralph.backends.database.es.ESDatabase"
 
-    HOSTS: CommaSeparatedTuple = ("http://localhost:9200",)
+    HOSTS: CommaSeparatedTuple = "http://localhost:9200"
     INDEX: str = "statements"
     CLIENT_OPTIONS: ESClientOptions = ESClientOptions()
     OP_TYPE: Literal["index", "create", "delete", "update"] = "index"
@@ -158,9 +165,7 @@ class DatabaseBackendSettings(BaseModel):
 
 class HeadersParameters(BaseModel):
     """Pydantic model for headers parameters."""
-
-    class Config:  # pylint: disable=missing-class-docstring # noqa: D106
-        extra = Extra.allow
+    model_config = ConfigDict(extra="allow")
 
 
 class LRSHeaders(HeadersParameters):
@@ -305,9 +310,7 @@ class ParserSettings(BaseModel):
 
 class XapiForwardingConfigurationSettings(BaseModel):
     """Pydantic model for xAPI forwarding configuration item."""
-
-    class Config:  # pylint: disable=missing-class-docstring # noqa: D106
-        min_anystr_length = 1
+    model_config = ConfigDict(str_min_length=1)
 
     url: AnyUrl
     is_active: bool
@@ -320,11 +323,16 @@ class Config:  # pylint: disable=missing-class-docstring # noqa: D106
 class Settings(BaseSettings):
     """Pydantic model for Ralph's global environment & configuration settings."""
 
-    class Config(BaseSettingsConfig):
-        """Pydantic Configuration."""
+    # TODO[pydantic]: The `Config` class inherits from another class, please create the `model_config` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
+    # class Config(BaseSettingsConfig):
+    #     """Pydantic Configuration."""
 
-        env_file = ".env"
-        env_file_encoding = core_settings.LOCALE_ENCODING
+    #     env_file = ".env"
+    #     env_file_encoding = core_settings.LOCALE_ENCODING
+
+    #model_config = SettingsConfigDict(env_file = ".env", env_file_encoding = core_settings.LOCALE_ENCODING) | BaseSettingsConfig()
+    model_config = BASE_SETTINGS_CONFIG | SettingsConfigDict(env_file = ".env", env_file_encoding = core_settings.LOCALE_ENCODING, extra="ignore")
 
     class AuthBackends(Enum):
         """Enum of the authentication backends."""
@@ -333,11 +341,15 @@ class AuthBackends(Enum):
         OIDC = "oidc"
 
     _CORE: CoreSettings = core_settings
+
+    # APP_DIR: Path = core_settings.APP_DIR
+    # LOCALE_ENCODING: str = core_settings.LOCALE_ENCODING
+
     AUTH_FILE: Path = _CORE.APP_DIR / "auth.json"
-    AUTH_CACHE_MAX_SIZE = 100
-    AUTH_CACHE_TTL = 3600
+    AUTH_CACHE_MAX_SIZE : int = 100
+    AUTH_CACHE_TTL : int = 3600
     BACKENDS: BackendSettings = BackendSettings()
-    CONVERTER_EDX_XAPI_UUID_NAMESPACE: str = None
+    CONVERTER_EDX_XAPI_UUID_NAMESPACE: Optional[str] = None
     DEFAULT_BACKEND_CHUNK_SIZE: int = 500
     EXECUTION_ENVIRONMENT: str = "development"
     HISTORY_FILE: Path = _CORE.APP_DIR / "history.json"
@@ -374,15 +386,15 @@ class AuthBackends(Enum):
     }
     PARSERS: ParserSettings = ParserSettings()
     RUNSERVER_AUTH_BACKEND: AuthBackends = AuthBackends.BASIC
-    RUNSERVER_AUTH_OIDC_AUDIENCE: str = None
-    RUNSERVER_AUTH_OIDC_ISSUER_URI: AnyHttpUrl = None
+    RUNSERVER_AUTH_OIDC_AUDIENCE: Optional[str] = None
+    RUNSERVER_AUTH_OIDC_ISSUER_URI: Optional[AnyHttpUrl] = None
     RUNSERVER_BACKEND: Literal["clickhouse", "es", "mongo"] = "es"
     RUNSERVER_HOST: str = "0.0.0.0"  # nosec
     RUNSERVER_MAX_SEARCH_HITS_COUNT: int = 100
     RUNSERVER_POINT_IN_TIME_KEEP_ALIVE: str = "1m"
     RUNSERVER_PORT: int = 8100
     SENTRY_CLI_TRACES_SAMPLE_RATE: float = 1.0
-    SENTRY_DSN: str = None
+    SENTRY_DSN: Optional[str] = None
     SENTRY_IGNORE_HEALTH_CHECKS: bool = False
     SENTRY_LRS_TRACES_SAMPLE_RATE: float = 1.0
     XAPI_FORWARDINGS: List[XapiForwardingConfigurationSettings] = []