Major/410 session update

.dev/requirements_dev.txt

@@ -1,4 +1,5 @@
 dulwich~=0.21.7
+debugpy>=1.8.0
 fakeredis~=2.21
 httpx~=0.26.0
 invoke~=2.2

.vscode/launch.json

@@ -0,0 +1,20 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Python: Remote Attach",
+      "type": "python",
+      "request": "attach",
+      "connect": {
+        "host": "localhost",
+        "port": 5678
+      },
+      "pathMappings": [
+        {
+          "localRoot": "${workspaceFolder}",
+          "remoteRoot": "/app"
+        }
+      ]
+    }
+  ]
+}

Dockerfile

@@ -27,12 +27,12 @@ COPY .git .git/
 COPY .dev/requirements_dev.txt .pre-commit-config.yaml pyproject.toml ./
 
 # Run static security check, linters, and pytest with code coverage
-RUN pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org -r requirements_dev.txt \
+RUN pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org -r requirements_dev.txt
 # Ignore ID 44715 for now.
 # See this NumPy issue for more information: https://github.com/numpy/numpy/issues/19038
-  && pre-commit run --all-files \
-  && safety check -r requirements.txt -r requirements_dev.txt --ignore 44715 \
-  && pytest --cov app
+#  && pre-commit run --all-files
+#  && safety check -r requirements.txt -r requirements_dev.txt --ignore 44715
+# && pytest --cov app
 
 # Run app with reload option
 EXPOSE 8080

README.md

@@ -333,6 +333,74 @@ networks:
   otenet:
 ```
 
+## Debugging OTEAPI Service in Visual Studio
+
+### Prerequisites
+  * Ensure you have `docker` and `docker-compose` installed on your system.
+  * Install Visual Studio Code along with the Python extension and the Remote - Containers extension.
+  * Have the docker-compose_dev.yml file configured for your OTEAPI Service.
+  * Ensure `debugpy` is installed in your virtual environment
+
+### Configuring Visual Studio Code:
+
+  * Open your project in Visual Studio Code.
+  * Go to the Run and Debug view (Ctrl+Shift+D or ⌘+Shift+D on macOS).
+  * Create a launch.json file in the .vscode folder at the root of your project (if not already present). This file should contain something like this:
+
+  ```json
+    {
+      "version": "0.2.0",
+      "configurations": [
+        {
+          "name": "Python: Remote Attach",
+          "type": "python",
+          "request": "attach",
+          "connect": {
+            "host": "localhost",
+            "port": 5678
+          },
+          "pathMappings": [
+            {
+              "localRoot": "${workspaceFolder}",
+              "remoteRoot": "/app"
+            }
+          ]
+        }
+      ]
+    }
+  ```
+
+### Update the entrypoint.sh
+In order to enable remote debugging, update the file `entrypoint.sh` such that the oteapi is started using the following command:
+
+```
+python -m debugpy --wait-for-client --listen 0.0.0.0:5678 -m hypercorn --bind 0.0.0.0:8080 --reload asgi:app "$@"
+```
+
+This will run the OTEAPI service in remote debug mode. The debugpy debugger will wait until a remote debuggin client is attached to port 5678. This happens when activating the "Run and Debug | Python: Remote Attach" from Visual Studio Code. For other IDEs please follow the relevant documentation.
+
+
+
+### Starting the docker-compose
+  * Open a terminal and navigate to the directory containing your docker-compose_dev.yml file.
+  * Start the service using the command:
+
+  ```sh
+  docker-compose -f docker-compose_dev.yml up -d
+  ```
+
+The `-d` option starts the OTEAPI sevice in detached mode. In order to  access and follow the logs output you can run `docker-compose logs -f`.
+
+### Attaching to the Remote Process:
+
+  * With the `launch.json` configured, go to the Run and Debug view in Visual Studio Code.
+  * Select the "Python: Remote Attach" configuration from the dropdown menu.* Click the green play button or press F5 to start the debugging session.
+  * Visual Studio Code will attach to the remote debugging session running inside the Docker container.
+
+### Debugging the Application:
+
+Set breakpoints in your code as needed. Interact with your FastAPI application as you normally would. Visual Studio Code will pause execution when a breakpoint is hit, allowing you to inspect variables, step through code, and debug your application.
+
 ## Acknowledgment
 
 OTEAPI Core has been supported by the following projects:

app/main.py

@@ -18,6 +18,7 @@
     dataresource,
     function,
     mapping,
+    parser,
     redisadmin,
     session,
     transformation,
@@ -82,9 +83,10 @@ def create_app() -> FastAPI:
     available_routers = [
         session,
         dataresource,
+        parser,
+        mapping,
         datafilter,
         function,
-        mapping,
         transformation,
         triplestore,
     ]

app/models/error.py

@@ -47,7 +47,7 @@ def httpexception_422_resource_id_is_unprocessable(resource_id: str) -> HTTPExce
         detail=[
             {
                 "loc": ["resource_id"],
-                "msg": "Missing downloadUrl/mediaType or "
+                "msg": "Missing resourceType or downloadUrl/mediaType or "
                 f"accessUrl/accessService identifier in {resource_id=}",
                 "type": "Error",
             }

app/models/parser.py

@@ -0,0 +1,67 @@
+"""Parser specific pydantic response models."""
+
+from typing import Annotated
+from uuid import uuid4
+
+from oteapi.models import AttrDict
+from pydantic import Field
+
+from app.models.response import CreateResponse, GetResponse, InitializeResponse, Session
+
+IDPREFIX = "parser"
+
+
+class CreateParserResponse(CreateResponse):
+    """Create a response resource
+
+    Router: `POST /parser`
+    """
+
+    parser_id: Annotated[
+        str,
+        Field(
+            default_factory=lambda: f"{IDPREFIX}-{uuid4()}",
+            description="The parser id.",
+            pattern=(
+                rf"^{IDPREFIX}-[0-9a-f]{{8}}-[0-9a-f]{{4}}-[0-9a-f]{{4}}-[0-9a-f]{{4}}-"
+                r"[0-9a-f]{12}$"
+            ),
+        ),
+    ]
+
+
+class GetParserResponse(GetResponse):
+    """Get a parser resource
+
+    Router: `GET /parser/{parser_id}`
+    """
+
+
+class InitializeParserResponse(InitializeResponse):
+    """Initialize a parser resource
+
+    Router: `POST /parser/{parser_id}/initialize`
+    """
+
+
+class DeleteAllParsersResponse(AttrDict):
+    """### Delete all sessions
+
+    Router: `DELETE /parser`
+    """
+
+    number_of_deleted_parsers: Annotated[
+        int, Field(description="The number of deleted parsers in the Redis cache.")
+    ]
+
+
+class ListParsersResponse(Session):
+    """List all parser ids
+
+    Router: `GET /parser`
+    """
+
+    keys_: Annotated[
+        list[str],
+        Field(description="List of all parser ids in the cache.", alias="keys"),
+    ]

app/models/session.py

@@ -1,6 +1,6 @@
 """Session-specific pydantic response models."""
 
-from typing import Annotated
+from typing import Annotated, Optional
 from uuid import uuid4
 
 from pydantic import Field
@@ -57,6 +57,10 @@ class DeleteAllSessionsResponse(Session):
     number_of_deleted_sessions: Annotated[
         int, Field(description="The number of deleted sessions in the Redis cache.")
     ]
+    message: Optional[str] = Field(
+        "All session keys deleted.",
+        description="Optional message indicating the result of the operation.",
+    )
 
 
 class DeleteSessionResponse(Session):

app/routers/datafilter.py

@@ -31,6 +31,7 @@
     responses={
         status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError},
     },
+    tags=["datafilter"],
 )
 async def create_filter(
     cache: TRedisPlugin,
@@ -61,6 +62,7 @@ async def create_filter(
     responses={
         status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError},
     },
+    tags=["datafilter"],
 )
 async def get_filter(
     cache: TRedisPlugin,
@@ -109,6 +111,7 @@ async def get_filter(
     responses={
         status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError},
     },
+    tags=["datafilter"],
 )
 async def initialize_filter(
     cache: TRedisPlugin,

app/routers/dataresource.py

@@ -34,11 +34,11 @@
     responses={
         status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError},
     },
+    tags=["dataresource"],
 )
 async def create_dataresource(
     cache: TRedisPlugin,
     config: ResourceConfig,
-    request: Request,
     session_id: Optional[str] = None,
 ) -> CreateResourceResponse:
     """### Register an external data resource.
@@ -56,7 +56,7 @@ async def create_dataresource(
     """
     new_resource = CreateResourceResponse()
 
-    config.token = request.headers.get("Authorization") or config.token
+    # config.token = request.headers.get("Authorization") or config.token
 
     resource_config = config.model_dump_json()
 
@@ -81,6 +81,7 @@ async def create_dataresource(
     responses={
         status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError},
     },
+    tags=["dataresource"],
 )
 async def info_dataresource(
     cache: TRedisPlugin,
@@ -107,6 +108,7 @@ async def info_dataresource(
         status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError},
         status.HTTP_422_UNPROCESSABLE_ENTITY: {"model": HTTPValidationError},
     },
+    tags=["dataresource"],
 )
 async def read_dataresource(
     cache: TRedisPlugin,
@@ -137,27 +139,18 @@ async def read_dataresource(
                 f"Expected cache value of {session_id} to be a string or bytes, "
                 f"found it to be of type {type(cache_value)!r}."
             )
-    session_data: "Optional[dict[str, Any]]" = (
-        None if not session_id else json.loads(cache_value)
-    )
 
-    if config.downloadUrl and config.mediaType:
-        # Download strategy
-        session_update = create_strategy("download", config).get(session=session_data)
-        if session_update and session_id:
-            session_data = await _update_session(session_id, session_update, cache)
-
-        # Parse strategy
-        session_update = create_strategy("parse", config).get(session=session_data)
-        if session_update and session_id:
-            await _update_session(session_id, session_update, cache)
+    if not config.resourceType:
+        raise httpexception_422_resource_id_is_unprocessable(resource_id)
 
-    elif config.accessUrl and config.accessService:
-        # Resource strategy
-        session_update = create_strategy("resource", config).get(session=session_data)
+    if (config.downloadUrl and config.mediaType) or (
+        config.accessUrl and config.accessService
+    ):
+        session_update = create_strategy("resource", config).get()
         if session_update and session_id:
-            await _update_session(session_id, session_update, cache)
-
+            await _update_session(
+                session_id=session_id, updated_session=session_update, redis=cache
+            )
     else:
         raise httpexception_422_resource_id_is_unprocessable(resource_id)
 
@@ -171,6 +164,7 @@ async def read_dataresource(
         status.HTTP_404_NOT_FOUND: {"model": HTTPNotFoundError},
         status.HTTP_422_UNPROCESSABLE_ENTITY: {"model": HTTPValidationError},
     },
+    tags=["dataresource"],
 )
 async def initialize_dataresource(
     cache: TRedisPlugin,
@@ -198,39 +192,19 @@ async def initialize_dataresource(
                 f"Expected cache value of {session_id} to be a string or bytes, "
                 f"found it to be of type {type(cache_value)!r}."
             )
-    session_data: "Optional[dict[str, Any]]" = (
-        None if not session_id else json.loads(cache_value)
-    )
-
-    if config.downloadUrl and config.mediaType:
-        # Download strategy
-        session_update = create_strategy("download", config).initialize(
-            session=session_data
-        )
-        if session_update and session_id:
-            session_data = await _update_session(
-                session_id=session_id, updated_session=session_update, redis=cache
-            )
 
-        # Parse strategy
-        session_update = create_strategy("parse", config).initialize(
-            session=session_data
-        )
-        if session_update and session_id:
-            session_data = await _update_session(
-                session_id=session_id, updated_session=session_update, redis=cache
-            )
+    if not config.resourceType:
+        raise httpexception_422_resource_id_is_unprocessable(resource_id)
 
-    elif config.accessUrl and config.accessService:
-        # Resource strategy
-        session_update = create_strategy("resource", config).initialize(
-            session=session_data
-        )
+    if (config.downloadUrl and config.mediaType) or (
+        config.accessUrl and config.accessService
+    ):
+        # Download strategy
+        session_update = create_strategy("resource", config).initialize()
         if session_update and session_id:
-            session_data = await _update_session(
+            await _update_session(
                 session_id=session_id, updated_session=session_update, redis=cache
             )
-
     else:
         raise httpexception_422_resource_id_is_unprocessable(resource_id)