Major/410 session update

.dev/requirements_dev.txt

@@ -1,4 +1,5 @@
 dulwich~=0.21.7
+debugpy>=1.8.0
 fakeredis~=2.21
 httpx~=0.27.0
 invoke~=2.2

.github/utils/direct_requirements.txt

@@ -1,6 +1,6 @@
 fastapi>=0.110.0
 hypercorn>=0.14.4
-oteapi-core>=0.6.1
+oteapi-core>=0.7.0.dev0
 pydantic>=2.6.2
 pydantic-settings>=2.2.1
 redis>=5.0.1

.github/utils/docker-compose_ci_plugins.yml

@@ -1,4 +1,4 @@
-version: "3"
+version: "3.3"
 
 services:
   oteapi:
@@ -7,13 +7,18 @@ services:
       target: "${DOCKER_OTEAPI_TARGET:-development}"
     ports:
       - "${PORT:-8080}:8080"
+      - "5678:5678" # Debug port
     environment:
       OTEAPI_REDIS_TYPE: redis
       OTEAPI_REDIS_HOST: redis
       OTEAPI_REDIS_PORT: 6379
       OTEAPI_prefix: "${OTEAPI_prefix:-/api/v1}"
-      OTEAPI_PLUGIN_PACKAGES:
+      OTEAPI_INCLUDE_REDISADMIN: "${OTEAPI_INCLUDE_REDISADMIN:-True}"
+      OTEAPI_EXPOSE_SECRETS: "${OTEAPI_EXPOSE_SECRETS:-True}"
       PATH_TO_OTEAPI_CORE: "${PATH_TO_OTEAPI_CORE:-/dev/null}"
+      OTEAPI_PLUGIN_PACKAGES:
+      OTEAPI_AUTHENTICATION_DEPENDENCIES:
+      CI:
     depends_on:
       - redis
     networks:

.pre-commit-config.yaml

@@ -66,7 +66,7 @@ repos:
     - id: pylint-tests
       name: pylint - tests
       entry: pylint
-      args: ["--rcfile=pyproject.toml", "--extension-pkg-whitelist='pydantic'", "--disable=C0415,W0621"]
+      args: ["--rcfile=pyproject.toml", "--extension-pkg-whitelist='pydantic'", "--disable=C0415,W0621,E1101"]
       language: python
       types: [python]
       require_serial: true

.vscode/launch.json

@@ -0,0 +1,20 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Python: Remote Attach",
+      "type": "debugpy",
+      "request": "attach",
+      "connect": {
+        "host": "localhost",
+        "port": 5678
+      },
+      "pathMappings": [
+        {
+          "localRoot": "${workspaceFolder}",
+          "remoteRoot": "/app"
+        }
+      ]
+    }
+  ]
+}

Dockerfile

@@ -27,15 +27,10 @@ COPY .git .git/
 COPY .dev/requirements_dev.txt .pre-commit-config.yaml pyproject.toml ./
 
 # Run static security check, linters, and pytest with code coverage
-RUN pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org -r requirements_dev.txt \
-# Ignore ID 44715 for now.
-# See this NumPy issue for more information: https://github.com/numpy/numpy/issues/19038
-  && pre-commit run --all-files \
-  && safety check -r requirements.txt -r requirements_dev.txt --ignore 44715 \
-  && pytest --cov app
-
+RUN pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org -r requirements_dev.txt
+ENV DEV_ENV=1
 # Run app with reload option
-EXPOSE 8080
+EXPOSE 8080 5678
 CMD if [ "${PATH_TO_OTEAPI_CORE}" != "/dev/null" ] && [ -n "${PATH_TO_OTEAPI_CORE}" ]; then \
   pip install -U --force-reinstall -e /oteapi_core; fi \
   && ./entrypoint.sh --reload --debug --log-level debug

README.md

@@ -333,6 +333,74 @@ networks:
   otenet:
 ```
 
+## Debugging OTEAPI Service in Visual Studio
+
+### Prerequisites
+  * Ensure you have `docker` and `docker-compose` installed on your system.
+  * Install Visual Studio Code along with the Python extension and the Remote - Containers extension.
+  * Have the docker-compose_dev.yml file configured for your OTEAPI Service.
+  * Ensure `debugpy` is installed in your virtual environment
+
+### Configuring Visual Studio Code:
+
+  * Open your project in Visual Studio Code.
+  * Go to the Run and Debug view (Ctrl+Shift+D or ⌘+Shift+D on macOS).
+  * Create a launch.json file in the .vscode folder at the root of your project (if not already present). This file should contain something like this:
+
+  ```json
+    {
+      "version": "0.2.0",
+      "configurations": [
+        {
+          "name": "Python: Remote Attach",
+          "type": "python",
+          "request": "attach",
+          "connect": {
+            "host": "localhost",
+            "port": 5678
+          },
+          "pathMappings": [
+            {
+              "localRoot": "${workspaceFolder}",
+              "remoteRoot": "/app"
+            }
+          ]
+        }
+      ]
+    }
+  ```
+
+### Update the entrypoint.sh
+In order to enable remote debugging, update the file `entrypoint.sh` such that the oteapi is started using the following command:
+
+```
+python -m debugpy --wait-for-client --listen 0.0.0.0:5678 -m hypercorn --bind 0.0.0.0:8080 --reload asgi:app "$@"
+```
+
+This will run the OTEAPI service in remote debug mode. The debugpy debugger will wait until a remote debuggin client is attached to port 5678. This happens when activating the "Run and Debug | Python: Remote Attach" from Visual Studio Code. For other IDEs please follow the relevant documentation.
+
+
+
+### Starting the docker-compose
+  * Open a terminal and navigate to the directory containing your docker-compose_dev.yml file.
+  * Start the service using the command:
+
+  ```sh
+  docker-compose -f docker-compose_dev.yml up -d
+  ```
+
+The `-d` option starts the OTEAPI sevice in detached mode. In order to  access and follow the logs output you can run `docker-compose logs -f`.
+
+### Attaching to the Remote Process:
+
+  * With the `launch.json` configured, go to the Run and Debug view in Visual Studio Code.
+  * Select the "Python: Remote Attach" configuration from the dropdown menu.* Click the green play button or press F5 to start the debugging session.
+  * Visual Studio Code will attach to the remote debugging session running inside the Docker container.
+
+### Debugging the Application:
+
+Set breakpoints in your code as needed. Interact with your FastAPI application as you normally would. Visual Studio Code will pause execution when a breakpoint is hit, allowing you to inspect variables, step through code, and debug your application.
+
 ## Acknowledgment
 
 OTEAPI Core has been supported by the following projects:

app/main.py

@@ -18,6 +18,7 @@
     dataresource,
     function,
     mapping,
+    parser,
     redisadmin,
     session,
     transformation,
@@ -82,9 +83,10 @@ def create_app() -> FastAPI:
     available_routers = [
         session,
         dataresource,
+        parser,
+        mapping,
         datafilter,
         function,
-        mapping,
         transformation,
         triplestore,
     ]

app/models/error.py

@@ -47,7 +47,7 @@ def httpexception_422_resource_id_is_unprocessable(resource_id: str) -> HTTPExce
         detail=[
             {
                 "loc": ["resource_id"],
-                "msg": "Missing downloadUrl/mediaType or "
+                "msg": "Missing resourceType or downloadUrl/mediaType or "
                 f"accessUrl/accessService identifier in {resource_id=}",
                 "type": "Error",
             }

app/models/parser.py

@@ -0,0 +1,43 @@
+"""Parser specific pydantic response models."""
+
+from typing import Annotated
+from uuid import uuid4
+
+from pydantic import Field
+
+from app.models.response import CreateResponse, GetResponse, InitializeResponse
+
+IDPREFIX = "parser"
+
+
+class CreateParserResponse(CreateResponse):
+    """Create a response resource
+
+    Router: `POST /parser`
+    """
+
+    parser_id: Annotated[
+        str,
+        Field(
+            default_factory=lambda: f"{IDPREFIX}-{uuid4()}",
+            description="The parser id.",
+            pattern=(
+                rf"^{IDPREFIX}-[0-9a-f]{{8}}-[0-9a-f]{{4}}-[0-9a-f]{{4}}-[0-9a-f]{{4}}-"
+                r"[0-9a-f]{12}$"
+            ),
+        ),
+    ]
+
+
+class GetParserResponse(GetResponse):
+    """Get a parser resource
+
+    Router: `GET /parser/{parser_id}`
+    """
+
+
+class InitializeParserResponse(InitializeResponse):
+    """Initialize a parser resource
+
+    Router: `POST /parser/{parser_id}/initialize`
+    """

app/models/session.py

@@ -1,6 +1,6 @@
 """Session-specific pydantic response models."""
 
-from typing import Annotated
+from typing import Annotated, Optional
 from uuid import uuid4
 
 from pydantic import Field
@@ -57,6 +57,12 @@ class DeleteAllSessionsResponse(Session):
     number_of_deleted_sessions: Annotated[
         int, Field(description="The number of deleted sessions in the Redis cache.")
     ]
+    message: Annotated[
+        Optional[str],
+        Field(
+            description="Optional message indicating the result of the operation.",
+        ),
+    ] = "All session keys deleted."
 
 
 class DeleteSessionResponse(Session):

app/redis_cache/_cache.py

@@ -0,0 +1,27 @@
+"""Cache operations."""
+
+from typing import Any
+
+from app.models.error import httpexception_404_item_id_does_not_exist
+from app.redis_cache import TRedisPlugin
+
+
+async def _fetch_cache_value(cache: TRedisPlugin, key: str, key_type: str) -> Any:
+    """Fetch key value from Cache and check if its of expected type (str or bytes)."""
+    await _validate_cache_key(cache, key, key_type)
+
+    cache_value = await cache.get(key)
+    if cache_value is None:
+        raise ValueError(f"Cache value for key '{key}' is None.")
+    if not isinstance(cache_value, (str, bytes)):
+        raise TypeError(
+            f"Expected cache value of {key} to be a string or bytes, "
+            f"found it to be of type: `{type(cache_value)!r}`."
+        )
+    return cache_value
+
+
+async def _validate_cache_key(cache: TRedisPlugin, key: str, key_type: str) -> None:
+    """Validate if a key exists in cache and is of expected type (str or bytes)."""
+    if not await cache.exists(key):
+        raise httpexception_404_item_id_does_not_exist(key, key_type)