Merge pull request #7 from unicef/feature/198735-create-public-api

[198735] Create public API

.dockerignore

@@ -1,5 +1,6 @@
 .git
 .*
+!.flake8
 ~*
 db
 docs

pyproject.toml

@@ -37,6 +37,7 @@ dependencies = [
     "django-flags>=5.0.13",
     "django-reversion>=5.0.12",
     "uwsgi>=2.0.25.1",
+    "drf-nested-routers>=0.94.1",
 ]
 
 [tool.pdm.build]
@@ -74,6 +75,7 @@ dev = [
     "unittest-xml-reporting",
     "watchdog",
     "pytest-factoryboy>=2.7.0",
+    "pytest-mock>=3.14.0",
 ]
 [tool.black]
 line-length = 120

src/hope_dedup_engine/apps/api/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class ApiConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "hope_dedup_engine.apps.api"

src/hope_dedup_engine/apps/api/auth.py

@@ -0,0 +1,25 @@
+from rest_framework.authentication import TokenAuthentication
+from rest_framework.permissions import BasePermission
+from rest_framework.request import Request
+from rest_framework.views import View
+
+from hope_dedup_engine.apps.api.models import DeduplicationSet
+from hope_dedup_engine.apps.api.models.auth import HDEToken
+
+
+class AssignedToExternalSystem(BasePermission):
+    def has_permission(self, request: Request, view: View) -> bool:
+        return request.user and request.user.external_system
+
+
+class UserAndDeduplicationSetAreOfTheSameSystem(BasePermission):
+    def has_permission(self, request: Request, view: View) -> bool:
+        if deduplication_set_pk := view.kwargs.get("deduplication_set_pk") or view.kwargs.get("pk"):
+            return DeduplicationSet.objects.filter(
+                external_system=request.user.external_system, pk=deduplication_set_pk
+            ).exists()
+        return True
+
+
+class HDETokenAuthentication(TokenAuthentication):
+    model = HDEToken

src/hope_dedup_engine/apps/api/const.py

@@ -0,0 +1,13 @@
+DEDUPLICATION_SET = "deduplication_set"
+DEDUPLICATION_SET_LIST = f"{DEDUPLICATION_SET}s"
+
+PK = "pk"
+DEDUPLICATION_SET_PARAM = f"{DEDUPLICATION_SET}_{PK}"
+DEDUPLICATION_SET_FILTER = f"{DEDUPLICATION_SET}__{PK}"
+
+IMAGE = "image"
+IMAGE_LIST = f"{IMAGE}s"
+
+BULK = "bulk"
+BULK_IMAGE = f"{IMAGE}_{BULK}"
+BULK_IMAGE_LIST = f"{IMAGE_LIST}_{BULK}"

src/hope_dedup_engine/apps/api/migrations/0001_initial.py

@@ -0,0 +1,123 @@
+# Generated by Django 5.0.6 on 2024-05-20 12:50
+
+import django.db.models.deletion
+import uuid
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ("security", "0001_initial"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="DeduplicationSet",
+            fields=[
+                ("id", models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ("name", models.CharField(max_length=100)),
+                ("reference_pk", models.IntegerField()),
+                (
+                    "state",
+                    models.IntegerField(
+                        choices=[(0, "Clean"), (1, "Dirty"), (2, "Processing"), (3, "Error")], default=0
+                    ),
+                ),
+                ("deleted", models.BooleanField(default=False, verbose_name="deleted")),
+                ("error", models.CharField(blank=True, max_length=255, null=True)),
+                ("created_at", models.DateTimeField(auto_now_add=True, verbose_name="created at")),
+                ("updated_at", models.DateTimeField(auto_now=True, verbose_name="updated at")),
+                (
+                    "created_by",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="+",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+                (
+                    "external_system",
+                    models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="security.externalsystem"),
+                ),
+                (
+                    "updated_by",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="+",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+        ),
+        migrations.CreateModel(
+            name="HDEToken",
+            fields=[
+                ("key", models.CharField(max_length=40, primary_key=True, serialize=False, verbose_name="Key")),
+                ("created", models.DateTimeField(auto_now_add=True, verbose_name="Created")),
+                (
+                    "user",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="auth_tokens",
+                        to=settings.AUTH_USER_MODEL,
+                        verbose_name="User",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "Token",
+                "verbose_name_plural": "Tokens",
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="Duplicate",
+            fields=[
+                ("id", models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ("filename", models.CharField(max_length=255)),
+                (
+                    "deduplication_set",
+                    models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="api.deduplicationset"),
+                ),
+            ],
+            options={
+                "abstract": False,
+                "unique_together": {("deduplication_set", "filename")},
+            },
+        ),
+        migrations.CreateModel(
+            name="Image",
+            fields=[
+                ("id", models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ("filename", models.CharField(max_length=255)),
+                ("created_at", models.DateTimeField(auto_now_add=True, verbose_name="created at")),
+                (
+                    "created_by",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="+",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+                (
+                    "deduplication_set",
+                    models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="api.deduplicationset"),
+                ),
+            ],
+            options={
+                "abstract": False,
+                "unique_together": {("deduplication_set", "filename")},
+            },
+        ),
+    ]

src/hope_dedup_engine/apps/api/models/__init__.py

@@ -0,0 +1,2 @@
+from hope_dedup_engine.apps.api.models.auth import HDEToken  # noqa: F401
+from hope_dedup_engine.apps.api.models.deduplication import DeduplicationSet  # noqa: F401

src/hope_dedup_engine/apps/api/models/auth.py

@@ -0,0 +1,11 @@
+from django.conf import settings
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+
+from rest_framework.authtoken.models import Token
+
+
+class HDEToken(Token):
+    user = models.ForeignKey(
+        settings.AUTH_USER_MODEL, related_name="auth_tokens", on_delete=models.CASCADE, verbose_name=_("User")
+    )

src/hope_dedup_engine/apps/api/models/deduplication.py

@@ -0,0 +1,58 @@
+from uuid import uuid4
+
+from django.conf import settings
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+
+from hope_dedup_engine.apps.security.models import ExternalSystem
+
+
+class DeduplicationSet(models.Model):
+    class State(models.IntegerChoices):
+        CLEAN = 0, _("Clean")  # Deduplication set is created or already processed
+        DIRTY = 1, _("Dirty")  # Images are added to deduplication set, but not yet processed
+        PROCESSING = 2, _("Processing")  # Images are being processed
+        ERROR = 3, _("Error")  # Error occurred
+
+    id = models.UUIDField(primary_key=True, default=uuid4)
+    name = models.CharField(max_length=100)
+    reference_pk = models.IntegerField()
+    state = models.IntegerField(
+        choices=State.choices,
+        default=State.CLEAN,
+    )
+    deleted = models.BooleanField(_("deleted"), null=False, blank=False, default=False)
+    external_system = models.ForeignKey(ExternalSystem, on_delete=models.CASCADE)
+    error = models.CharField(max_length=255, null=True, blank=True)
+    created_by = models.ForeignKey(
+        settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True, related_name="+"
+    )
+    created_at = models.DateTimeField(_("created at"), auto_now_add=True)
+    updated_by = models.ForeignKey(
+        settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True, related_name="+"
+    )
+    updated_at = models.DateTimeField(_("updated at"), auto_now=True)
+
+
+class ImagePath(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid4)
+    deduplication_set = models.ForeignKey(DeduplicationSet, on_delete=models.CASCADE)
+    filename = models.CharField(max_length=255)
+
+    class Meta:
+        abstract = True
+        unique_together = ("deduplication_set", "filename")
+
+
+class Duplicate(ImagePath):
+    pass
+
+
+class Image(ImagePath):
+    created_by = models.ForeignKey(
+        settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True, related_name="+"
+    )
+    created_at = models.DateTimeField(_("created at"), auto_now_add=True)
+    #
+    # class Meta:
+    #     unique_together = ("deduplication_set", "filename")

src/hope_dedup_engine/apps/api/serializers.py

@@ -0,0 +1,20 @@
+from rest_framework import serializers
+
+from hope_dedup_engine.apps.api.models import DeduplicationSet
+from hope_dedup_engine.apps.api.models.deduplication import Image
+
+
+class DeduplicationSetSerializer(serializers.ModelSerializer):
+    state = serializers.CharField(source="get_state_display", read_only=True)
+
+    class Meta:
+        model = DeduplicationSet
+        exclude = ("deleted",)
+        read_only_fields = "external_system", "created_at", "created_by", "deleted", "updated_at", "updated_by"
+
+
+class ImageSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Image
+        fields = "__all__"
+        read_only_fields = "created_by", "created_at"

src/hope_dedup_engine/apps/api/urls.py

@@ -0,0 +1,16 @@
+from django.urls import include, path
+
+from rest_framework import routers
+from rest_framework_nested import routers as nested_routers
+
+from hope_dedup_engine.apps.api.const import BULK_IMAGE_LIST, DEDUPLICATION_SET, DEDUPLICATION_SET_LIST, IMAGE_LIST
+from hope_dedup_engine.apps.api.views import BulkImageViewSet, DeduplicationSetViewSet, ImageViewSet
+
+router = routers.SimpleRouter()
+router.register(DEDUPLICATION_SET_LIST, DeduplicationSetViewSet, basename=DEDUPLICATION_SET_LIST)
+
+deduplication_sets_router = nested_routers.NestedSimpleRouter(router, DEDUPLICATION_SET_LIST, lookup=DEDUPLICATION_SET)
+deduplication_sets_router.register(IMAGE_LIST, ImageViewSet, basename=IMAGE_LIST)
+deduplication_sets_router.register(BULK_IMAGE_LIST, BulkImageViewSet, basename=BULK_IMAGE_LIST)
+
+urlpatterns = [path("", include(router.urls)), path("", include(deduplication_sets_router.urls))]

src/hope_dedup_engine/apps/api/utils.py

@@ -0,0 +1,9 @@
+from hope_dedup_engine.apps.api.models import DeduplicationSet
+
+
+def start_processing(_: DeduplicationSet) -> None:
+    pass
+
+
+def delete_model_data(_: DeduplicationSet) -> None:
+    pass